diff --git a/android/SherpaOnnxWebSocket/.gitignore b/android/SherpaOnnxWebSocket/.gitignore new file mode 100644 index 000000000..aa724b770 --- /dev/null +++ b/android/SherpaOnnxWebSocket/.gitignore @@ -0,0 +1,15 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/android/SherpaOnnxWebSocket/app/.gitignore b/android/SherpaOnnxWebSocket/app/.gitignore new file mode 100644 index 000000000..42afabfd2 --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/android/SherpaOnnxWebSocket/app/build.gradle b/android/SherpaOnnxWebSocket/app/build.gradle new file mode 100644 index 000000000..190241f58 --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/build.gradle @@ -0,0 +1,47 @@ +plugins { + id 'com.android.application' + id 'org.jetbrains.kotlin.android' +} + +android { + namespace 'com.k2fsa.sherpa.onnx' + compileSdk 32 + + defaultConfig { + applicationId "com.k2fsa.sherpa.onnx" + minSdk 21 + targetSdk 32 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = '1.8' + } +} + +dependencies { + + implementation 'androidx.core:core-ktx:1.7.0' + implementation 'androidx.appcompat:appcompat:1.5.1' + implementation 'com.google.android.material:material:1.7.0' + implementation 'androidx.constraintlayout:constraintlayout:2.1.4' + testImplementation 'junit:junit:4.13.2' + androidTestImplementation 'androidx.test.ext:junit:1.1.4' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.0' + + implementation 'org.java-websocket:Java-WebSocket:1.4.0' + implementation 'com.google.code.gson:gson:2.10.1' +} \ No newline at end of file diff --git a/android/SherpaOnnxWebSocket/app/proguard-rules.pro b/android/SherpaOnnxWebSocket/app/proguard-rules.pro new file mode 100644 index 000000000..481bb4348 --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/android/SherpaOnnxWebSocket/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt b/android/SherpaOnnxWebSocket/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt new file mode 100644 index 000000000..183383202 --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package com.k2fsa.sherpa.onnx + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("com.k2fsa.sherpa.onnx", appContext.packageName) + } +} \ No newline at end of file diff --git a/android/SherpaOnnxWebSocket/app/src/main/AndroidManifest.xml b/android/SherpaOnnxWebSocket/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..61f5981ff --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/src/main/AndroidManifest.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + diff --git a/android/SherpaOnnxWebSocket/app/src/main/assets/.gitkeep b/android/SherpaOnnxWebSocket/app/src/main/assets/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt new file mode 100644 index 000000000..58620ad21 --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -0,0 +1,268 @@ +// add by longsm at 2023/10/13 +package com.k2fsa.sherpa.onnx + +import android.Manifest +import android.content.pm.PackageManager +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaRecorder +import android.os.Bundle +import android.text.TextUtils +import android.text.method.ScrollingMovementMethod +import android.util.Log +import android.widget.Button +import android.widget.EditText +import android.widget.TextView +import androidx.appcompat.app.AppCompatActivity +import androidx.core.app.ActivityCompat +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import org.java_websocket.handshake.ServerHandshake +import java.net.URI +import java.net.URISyntaxException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import kotlin.concurrent.thread + +private const val TAG = "sherpa-onnx" +private const val REQUEST_RECORD_AUDIO_PERMISSION = 200 + +class MainActivity : AppCompatActivity(), MyWebsocketClient.WebsocketClientCallback { + private val permissions: Array = arrayOf(Manifest.permission.RECORD_AUDIO) + + private var audioRecord: AudioRecord? = null + private lateinit var recordButton: Button + private lateinit var connectButton: Button + private lateinit var textView: TextView + private lateinit var etUrl: EditText + private var recordingThread: Thread? = null + + private var websocketClient: MyWebsocketClient? = null + + private val audioSource = MediaRecorder.AudioSource.MIC + private val sampleRateInHz = 16000 + private val channelConfig = AudioFormat.CHANNEL_IN_MONO + + // Note: We don't use AudioFormat.ENCODING_PCM_FLOAT + // since the AudioRecord.read(float[]) needs API level >= 23 + // but we are targeting API level >= 21 + private val audioFormat = AudioFormat.ENCODING_PCM_16BIT + private var idx: Long = 0 + private var lastText: String = "" + + @Volatile + private var isRecording: Boolean = false + + @Volatile + private var isConnected: Boolean = false + + override fun onRequestPermissionsResult( + requestCode: Int, permissions: Array, grantResults: IntArray + ) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + val permissionToRecordAccepted = if (requestCode == REQUEST_RECORD_AUDIO_PERMISSION) { + grantResults[0] == PackageManager.PERMISSION_GRANTED + } else { + false + } + + if (!permissionToRecordAccepted) { + Log.e(TAG, "Audio record is disallowed") + finish() + } + + Log.i(TAG, "Audio record is permitted") + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + + ActivityCompat.requestPermissions(this, permissions, REQUEST_RECORD_AUDIO_PERMISSION) + + recordButton = findViewById(R.id.record_button) + recordButton.setOnClickListener { onclick() } + + connectButton = findViewById(R.id.connect_button) + connectButton.setOnClickListener { onclickConnect() } + + textView = findViewById(R.id.my_text) + textView.movementMethod = ScrollingMovementMethod() + + recordButton.isEnabled = false + + etUrl = findViewById(R.id.et_uri) + } + + private fun onclickConnect() { + if (!isConnected) { + val etUrlStr = etUrl.text.toString().trim() + var uriStr = "ws://172.28.13.167:6006" + if (!TextUtils.isEmpty(etUrlStr)) { + uriStr = etUrlStr + } + try { + val uri = URI(uriStr) + websocketClient = MyWebsocketClient(uri) + websocketClient?.setClientCallback(this) + websocketClient?.connect() + } catch (e: URISyntaxException) { + Log.e(TAG, "URISyntaxException === >> $e") + } + } else { + Log.e(TAG, "onclick disconnect") + websocketClient?.close() + websocketClient = null + } + + } + + private fun onclick() { + + if (!isRecording) { + val ret = initMicrophone() + if (!ret) { + Log.e(TAG, "Failed to initialize microphone") + return + } + Log.i(TAG, "state: ${audioRecord?.state}") + audioRecord!!.startRecording() + recordButton.setText(R.string.stop) + isRecording = true + textView.text = "" + lastText = "" + idx = 0 + + recordingThread = thread(true) { + processSamples() + } + connectButton.isEnabled = false + Log.i(TAG, "Started recording") + } else { + isRecording = false + audioRecord!!.stop() + audioRecord!!.release() + audioRecord = null + recordButton.setText(R.string.start) + connectButton.isEnabled = true + Log.i(TAG, "Stopped recording") + } + } + + private fun processSamples() { + Log.i(TAG, "processing samples") + + val interval = 0.1 // i.e., 100 ms + val bufferSize = (interval * sampleRateInHz).toInt() // in samples + val buffer = ShortArray(bufferSize) + + while (isRecording) { + val ret = audioRecord?.read(buffer, 0, buffer.size) + if (ret != null && ret > 0) { + val samples = FloatArray(ret) { buffer[it] / 32768.0f } + + val buffer = ByteBuffer.allocate(4 * samples.size) + .order(ByteOrder.LITTLE_ENDIAN) // float is sizeof 4. allocate enough buffer + + + for (f in samples) { + buffer.putFloat(f) + } + buffer.rewind() + buffer.flip() + buffer.order(ByteOrder.LITTLE_ENDIAN) + + if (isConnected) { + websocketClient?.send(buffer.array()) // send buf to server + } + + } + } + } + + private fun initMicrophone(): Boolean { + if (ActivityCompat.checkSelfPermission( + this, Manifest.permission.RECORD_AUDIO + ) != PackageManager.PERMISSION_GRANTED + ) { + ActivityCompat.requestPermissions(this, permissions, REQUEST_RECORD_AUDIO_PERMISSION) + return false + } + + val numBytes = AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat) + Log.i( + TAG, "buffer size in milliseconds: ${numBytes * 1000.0f / sampleRateInHz}" + ) + + audioRecord = AudioRecord( + audioSource, + sampleRateInHz, + channelConfig, + audioFormat, + numBytes * 2 // a sample has two bytes as we are using 16-bit PCM + ) + return true + } + + override fun onOpen(handshakedata: ServerHandshake?) { + Log.i(TAG, "onOpen === >>") + isConnected = true + runOnUiThread { + recordButton.isEnabled = true + connectButton.text = getString(R.string.disconnect) + } + } + + private val gson = Gson() + private val recognitionText = hashMapOf() + + private fun getDisplayResult(): String { + var i = 0 + var ans = "" + for ((key,value) in recognitionText){ + if (value == ""){ + continue + } + ans += " $i : ${recognitionText[key]}\n" + i += 1 + } + return ans + + } + + override fun onMessage(message: String?) { + Log.i(TAG, "onMessage === >> $message") + val speechContent = gson.fromJson( + message, + object : TypeToken() {}.type + ) + + val text = speechContent.text + val segment = speechContent.segment + Log.i(TAG, "text === >> $text") + + recognitionText[segment] = text + runOnUiThread { + textView.text = getDisplayResult() + } + } + + override fun onClose(code: Int, reason: String?, remote: Boolean?) { + Log.i(TAG, "onClose === >> code$code reason$reason remote$remote") + isConnected = false + runOnUiThread { + recordButton.isEnabled = false + connectButton.text = getString(R.string.connect) + textView.text = getString(R.string.hint) + } + + } + + override fun onError(ex: Exception?) { + Log.i(TAG, "onError === >> $ex") + runOnUiThread { + textView.text = "onError === >> $ex" + } + + } +} diff --git a/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/MyWebsocketClient.kt b/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/MyWebsocketClient.kt new file mode 100644 index 000000000..1226cadc6 --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/MyWebsocketClient.kt @@ -0,0 +1,39 @@ +package com.k2fsa.sherpa.onnx + +import org.java_websocket.client.WebSocketClient +import org.java_websocket.handshake.ServerHandshake +import java.net.URI + +class MyWebsocketClient(serverUri: URI?) : WebSocketClient(serverUri) { + + override fun onOpen(handshakedata: ServerHandshake) { + clientCallback?.onOpen(handshakedata) + + } + override fun onMessage(message: String) { + clientCallback?.onMessage(message) + } + + override fun onClose(code: Int, reason: String, remote: Boolean) { + clientCallback?.onClose(code,reason,remote) + } + + override fun onError(ex: Exception) { + clientCallback?.onError(ex) + } + + private var clientCallback: WebsocketClientCallback? = null + + fun setClientCallback(clientCallback: WebsocketClientCallback?) { + this.clientCallback = clientCallback + } + + interface WebsocketClientCallback { + fun onOpen(handshakedata: ServerHandshake?) + fun onMessage(message: String?) + fun onClose(code: Int, reason: String?, remote: Boolean?) + fun onError(ex: Exception?) + } + + +} \ No newline at end of file diff --git a/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/SpeechContent.kt b/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/SpeechContent.kt new file mode 100644 index 000000000..dc0a4406a --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/SpeechContent.kt @@ -0,0 +1,3 @@ +package com.k2fsa.sherpa.onnx + +data class SpeechContent(val text:String,val segment:Long) diff --git a/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt b/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt new file mode 100644 index 000000000..dca399840 --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt @@ -0,0 +1,29 @@ +// Copyright (c) 2023 Xiaomi Corporation +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +class WaveReader { + companion object { + // Read a mono wave file asset + // The returned array has two entries: + // - the first entry contains an 1-D float array + // - the second entry is the sample rate + external fun readWaveFromAsset( + assetManager: AssetManager, + filename: String, + ): Array + + // Read a mono wave file from disk + // The returned array has two entries: + // - the first entry contains an 1-D float array + // - the second entry is the sample rate + external fun readWaveFromFile( + filename: String, + ): Array + + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} diff --git a/android/SherpaOnnxWebSocket/app/src/main/jniLibs/.gitignore b/android/SherpaOnnxWebSocket/app/src/main/jniLibs/.gitignore new file mode 100644 index 000000000..949c039f1 --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/src/main/jniLibs/.gitignore @@ -0,0 +1,4 @@ +*.so +*.txt +*.onnx +*.wav diff --git a/android/SherpaOnnxWebSocket/app/src/main/jniLibs/arm64-v8a/.gitkeep b/android/SherpaOnnxWebSocket/app/src/main/jniLibs/arm64-v8a/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxWebSocket/app/src/main/jniLibs/armeabi-v7a/.gitkeep b/android/SherpaOnnxWebSocket/app/src/main/jniLibs/armeabi-v7a/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxWebSocket/app/src/main/jniLibs/x86/.gitkeep b/android/SherpaOnnxWebSocket/app/src/main/jniLibs/x86/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxWebSocket/app/src/main/jniLibs/x86_64/.gitkeep b/android/SherpaOnnxWebSocket/app/src/main/jniLibs/x86_64/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/android/SherpaOnnxWebSocket/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/android/SherpaOnnxWebSocket/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 000000000..2b068d114 --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnxWebSocket/app/src/main/res/drawable/ic_launcher_background.xml b/android/SherpaOnnxWebSocket/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..07d5da9cb --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/SherpaOnnxWebSocket/app/src/main/res/layout/activity_main.xml b/android/SherpaOnnxWebSocket/app/src/main/res/layout/activity_main.xml new file mode 100644 index 000000000..ef1f8057d --- /dev/null +++ b/android/SherpaOnnxWebSocket/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,66 @@ + + + + + + +