Adding Flutter controller for Both Android and iOS, with video feed u…

…sing WebRTC (#328)

Creating a flutter controller to Control the Robot via another smartphone (android and ios both). 
This controller can view the robot camera stream in real time using webRTC

android/robot/src/main/java/org/openbot/common/ControlsFragment.java

@@ -242,6 +242,14 @@ private void handlePhoneControllerEvents() {
             commandType = event.getString("command");
           } else if (event.has("driveCmd")) {
             commandType = Constants.CMD_DRIVE;
+          } else if (event.has("server")) {
+            for (int i = 0; i < serverSpinner.getAdapter().getCount(); i++) {
+              if(event.getString("server").equals("noServerFound")){
+                serverSpinner.setSelection(0);
+              } else if(event.getString("server").equals(serverSpinner.getAdapter().getItem(i))){
+                serverSpinner.setSelection(i);
+              }
+            }
           }
 
           switch (commandType) {
@@ -287,7 +295,7 @@ private void handlePhoneControllerEvents() {
         error -> {
           Log.d(null, "Error occurred in ControllerToBotEventBus: " + error);
         },
-        event -> event.has("command") || event.has("driveCmd") // filter out everything else
+        event -> event.has("command") || event.has("driveCmd") || event.has("server") // filter out everything else
         );
   }
 

android/robot/src/main/java/org/openbot/env/NetworkServiceConnection.java

@@ -136,6 +136,7 @@ public void onServiceFound(NsdServiceInfo service) {
         try {
           if (service.getServiceType().equals(SERVICE_TYPE)
               && service.getServiceName().equals(SERVICE_NAME_CONTROLLER)) {
+            Timber.e("found service");
             mNsdManager.resolveService(service, mResolveListener);
           } else if (service.getServiceName().equals(MY_SERVICE_NAME)) {
             Log.d(TAG, "Same machine: " + MY_SERVICE_NAME);
@@ -289,7 +290,6 @@ void runReceiver(Scanner reader) {
       try {
         while (true) {
           String msg = reader.nextLine().trim();
-
           if (!stopped) {
             ((Activity) context).runOnUiThread(() -> dataReceivedCallback.dataReceived(msg));
           }

android/robot/src/main/java/org/openbot/server/ServerCommunication.java

@@ -39,6 +39,7 @@ public void onResolveFailed(NsdServiceInfo serviceInfo, int errorCode) {
 
         @Override
         public void onServiceResolved(NsdServiceInfo serviceInfo) {
+          Timber.e("serviceInfo %s", serviceInfo.getServiceName());
           servers.put(serviceInfo.getServiceName(), serviceInfo);
           try {
             serverListener.onServerListChange(servers.keySet());

controller/.gitignore

@@ -1 +1,2 @@
 __pycache__
+.run

controller/flutter/.gitignore

@@ -0,0 +1,44 @@
+# Miscellaneous
+*.class
+*.log
+*.pyc
+*.swp
+.DS_Store
+.atom/
+.buildlog/
+.history
+.svn/
+migrate_working_dir/
+
+# IntelliJ related
+*.iml
+*.ipr
+*.iws
+.idea/
+
+# The .vscode folder contains launch configuration and tasks you configure in
+# VS Code which you may wish to be included in version control, so this line
+# is commented out by default.
+#.vscode/
+
+# Flutter/Dart/Pub related
+**/doc/api/
+**/ios/Flutter/.last_build_id
+.dart_tool/
+.flutter-plugins
+.flutter-plugins-dependencies
+.packages
+.pub-cache/
+.pub/
+/build/
+
+# Symbolication related
+app.*.symbols
+
+# Obfuscation related
+app.*.map.json
+
+# Android Studio will place build artifacts here
+/android/app/debug
+/android/app/profile
+/android/app/release

controller/flutter/.metadata

@@ -0,0 +1,33 @@
+# This file tracks properties of this Flutter project.
+# Used by Flutter tool to assess capabilities and perform upgrades etc.
+#
+# This file should be version controlled.
+
+version:
+  revision: b8f7f1f9869bb2d116aa6a70dbeac61000b52849
+  channel: stable
+
+project_type: app
+
+# Tracks metadata for the flutter migrate command
+migration:
+  platforms:
+    - platform: root
+      create_revision: b8f7f1f9869bb2d116aa6a70dbeac61000b52849
+      base_revision: b8f7f1f9869bb2d116aa6a70dbeac61000b52849
+    - platform: android
+      create_revision: b8f7f1f9869bb2d116aa6a70dbeac61000b52849
+      base_revision: b8f7f1f9869bb2d116aa6a70dbeac61000b52849
+    - platform: ios
+      create_revision: b8f7f1f9869bb2d116aa6a70dbeac61000b52849
+      base_revision: b8f7f1f9869bb2d116aa6a70dbeac61000b52849
+
+  # User provided section
+
+  # List of Local paths (relative to this file) that should be
+  # ignored by the migrate tool.
+  #
+  # Files that are not part of the templates will be ignored by default.
+  unmanaged_files:
+    - 'lib/main.dart'
+    - 'ios/Runner.xcodeproj/project.pbxproj'

controller/flutter/README.md

@@ -0,0 +1,107 @@
+# Flutter Controller App
+
+This Controller app serves as a `remote controller` for the [OpenBot](https://www.openbot.org) vehicle similar as a BT controller (e.g. PS3/4 or Xbox). It runs on another Android/iOS device and supports live video/audio streaming in addition to control.
+
+ ## Getting Started
+Begin by installing [Flutter](https://flutter.dev/) on your system. Choose the appropriate download for your operating system, which includes options for Windows, macOS, Linux, and ChromeOS. Follow the official Flutter installation guide for detailed instructions: [Flutter Installation Guide](https://docs.flutter.dev/get-started/install)
+
+### Using Terminal
+- Once Flutter is installed successfully, open your **terminal** or **command prompt**.
+- Change your current directory to the location where the OpenBot project is stored and then navigate to `OpenBot/controller/flutter`.
+- Use the following commands to run the Flutter application from the terminal.
+
+  ####  Install Dependencies:
+    ```bash
+     flutter pub get 
+    ```
+    Run the project:
+    ```bash
+     flutter run
+    ```
+    If you encounter any issues, run the following command:
+    ```bash
+     flutter doctor
+    ```
+### Using Editor
+- Follow the official Flutter guide for setting up an editor:  [Set up an editor ](https://docs.flutter.dev/tools/android-studio) 
+- Ensure that your editor is configured for Flutter development. Install any required plugins or extensions, following the editor-specific instructions in the Flutter documentation for the best development experience.
+
+- Once you open your project in the editor after the setup, it will appear as shown in the following image.
+
+  <p float="left">
+    <img src="../../docs/images/android_editor.jpg" width="50%" />
+  </p>
+
+- Please follow the instructions similar to the ones mentioned above for running Flutter in the terminal and directly run using the ``run`` button for future repetitions.
+
+  <p float="left">
+    <img src="../../docs/images/run_editor.jpg" width="50%" />
+  </p>
+
+## Connection 
+
+When the controller app is started, it immediately tries to connect to the robot and shows the following screen:
+
+<p float="left">
+  <img src="../../docs/images/flutter_controller_home.jpg" width="50%" />
+</p>
+
+To connect the controller to the robot, set the robot's control mode to **Phone**.
+For example, in the `FreeRoamFragment` the phone mode is activated like this:
+
+<p float="left">
+  <img src="../../docs/images/phone_selection.gif" width="50%" />
+</p>
+
+Once connected, the controller app will look like this:
+
+<p float="left">
+  <img src="../../docs/images/flutter_controller_connected.jpg" width="50%" />
+</p>
+
+Here you can select to drive the robot by tilting the phone, or by using the on-screen controls.
+
+***Note:*** This should be sufficient to connect, but if the connection cannot be established after 30 seconds, toggle
+the `Control` setting on the bot app to `Gamepad` and then to `Phone` again to re-initiate the connection. If that
+fails, exit the controller app and start it again. Toggle the control mode again on the robot app.
+
+## Operation
+
+### On-screen controls
+
+This mode allows the user to control the robot car via two sliders in `Dual Drive` mode. You can turn left/right by
+moving the slider thumb up and down on each side. The wheels on each side turn forward/backward when moving the thumb
+above/below the center of the slider.
+
+<p float="left">
+  <img src="../../docs/images/flutter_controller_dual_drive_mode.jpg" width="50%" />
+</p>
+
+- ``Indicators``: You can also set the left/right turn indicators <img src="../../docs/images/keyboard_arrow_left-24px.svg" height="24"/> <img src="../../docs/images/keyboard_arrow_right-24px.svg" height="24"/> by clicking on the arrows on the top-left of the screen.
+
+- ``Switch Camera``: switch between the front and back camera modes.
+- ``Mute``: enable/disable audio transmission.
+- ``Mirror view``: mirror the the video feed. 
+
+### Tilt to drive
+
+The controller can also use its accelerometer motion sensor to drive the robot. If you select this option, the
+controller will enter a full-screen (Zen) mode with only the video showing and `brake` and `accelerator` pedals. To
+exit this mode, double-tap on the screen.
+
+Here is a picture of the `tilt mode` screen:
+
+<p float="left">
+  <img src="../../docs/images/flutter_controller_tilt_mode.jpg" width="50%" />
+</p>
+
+Use the `accelerator` and `brake` buttons to move forward/backward.
+
+- Pressing the `accelerator` will accelerate the robot to full speed within 2 seconds. When you release the button, the
+  robot will slow down to a stop (stop speed set to 0% of the maximum speed, can be adjusted).
+- Pressing the `brake` button will immediately stop the robot. If we hold the brake for another second, the robot will
+  start moving backwards until it reaches the maximum reverse speed in one second. When we let go of the brake, the
+  robot will come to a stop.
+- The robot is steered by tilting the controller phone left or right.
+
+Here is a [Technical Overview](../../docs/technical/OpenBotController.pdf) of the controller app.

controller/flutter/analysis_options.yaml

@@ -0,0 +1,29 @@
+# This file configures the analyzer, which statically analyzes Dart code to
+# check for errors, warnings, and lints.
+#
+# The issues identified by the analyzer are surfaced in the UI of Dart-enabled
+# IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be
+# invoked from the command line by running `flutter analyze`.
+
+# The following line activates a set of recommended lints for Flutter apps,
+# packages, and plugins designed to encourage good coding practices.
+include: package:flutter_lints/flutter.yaml
+
+linter:
+  # The lint rules applied to this project can be customized in the
+  # section below to disable rules from the `package:flutter_lints/flutter.yaml`
+  # included above or to enable additional rules. A list of all available lints
+  # and their documentation is published at
+  # https://dart-lang.github.io/linter/lints/index.html.
+  #
+  # Instead of disabling a lint rule for the entire project in the
+  # section below, it can also be suppressed for a single line of code
+  # or a specific dart file by using the `// ignore: name_of_lint` and
+  # `// ignore_for_file: name_of_lint` syntax on the line or in the file
+  # producing the lint.
+  rules:
+    # avoid_print: false  # Uncomment to disable the `avoid_print` rule
+    # prefer_single_quotes: true  # Uncomment to enable the `prefer_single_quotes` rule
+
+# Additional information about this file can be found at
+# https://dart.dev/guides/language/analysis-options

controller/flutter/android/.gitignore

@@ -0,0 +1,13 @@
+gradle-wrapper.jar
+/.gradle
+/captures/
+/gradlew
+/gradlew.bat
+/local.properties
+GeneratedPluginRegistrant.java
+
+# Remember to never publicly share your keystore.
+# See https://flutter.dev/docs/deployment/android#reference-the-keystore-from-the-app
+key.properties
+**/*.keystore
+**/*.jks

controller/flutter/android/app/build.gradle

@@ -0,0 +1,73 @@
+def localProperties = new Properties()
+def localPropertiesFile = rootProject.file('local.properties')
+if (localPropertiesFile.exists()) {
+    localPropertiesFile.withReader('UTF-8') { reader ->
+        localProperties.load(reader)
+    }
+}
+
+def flutterRoot = localProperties.getProperty('flutter.sdk')
+if (flutterRoot == null) {
+    throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file.")
+}
+
+def flutterVersionCode = localProperties.getProperty('flutter.versionCode')
+if (flutterVersionCode == null) {
+    flutterVersionCode = '1'
+}
+
+def flutterVersionName = localProperties.getProperty('flutter.versionName')
+if (flutterVersionName == null) {
+    flutterVersionName = '1.0'
+}
+
+apply plugin: 'com.android.application'
+apply plugin: 'kotlin-android'
+apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"
+
+android {
+    compileSdkVersion flutter.compileSdkVersion
+    ndkVersion flutter.ndkVersion
+
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_11
+        targetCompatibility JavaVersion.VERSION_11
+    }
+
+    kotlinOptions {
+        jvmTarget = '11'
+    }
+
+    sourceSets {
+        main.java.srcDirs += 'src/main/kotlin'
+    }
+
+    defaultConfig {
+        // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html).
+        applicationId "org.openbot.flutter_controller"
+        // You can update the following values to match your application needs.
+        // For more information, see: https://docs.flutter.dev/deployment/android#reviewing-the-build-configuration.
+        minSdkVersion flutter.minSdkVersion
+        targetSdkVersion flutter.targetSdkVersion
+        versionCode flutterVersionCode.toInteger()
+        versionName flutterVersionName
+        minSdkVersion 21
+    }
+
+    buildTypes {
+        release {
+            // TODO: Add your own signing config for the release build.
+            // Signing with the debug keys for now, so `flutter run --release` works.
+            signingConfig signingConfigs.debug
+        }
+    }
+}
+
+flutter {
+    source '../..'
+}
+
+dependencies {
+    implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
+
+}

controller/flutter/android/app/src/debug/AndroidManifest.xml

@@ -0,0 +1,9 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="org.openbot.flutter_controller">
+    <!-- The INTERNET permission is required for development. Specifically,
+         the Flutter tool needs it to communicate with the running application
+         to allow setting breakpoints, to provide hot reload, etc.
+    -->
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.CHANGE_WIFI_MULTICAST_STATE" />
+</manifest>