Implement multitouch (#681)

1. Remove SkikoTouchEvent, we send all pointer events as SkikoPointerEvent now.

2. Add SkikoPointerEvent.Pointer that represents a separate touch/mouse

3. Implement multitouch for iOS/Android

skiko/src/androidMain/kotlin/org/jetbrains/skiko/Convertors.android.kt

@@ -13,7 +13,7 @@ fun toSkikoKeyboardEvent(
         key = SkikoKey.valueOf(keyCode),
         modifiers = toSkikoModifiers(event),
         kind = kind,
-        timestamp = event.getEventTime(),
+        timestamp = event.eventTime,
         platform = event
     )
 }
@@ -48,21 +48,33 @@ private fun toSkikoModifiers(event: KeyEvent): SkikoInputModifiers {
     return SkikoInputModifiers(result)
 }
 
-fun toSkikoTouchEvent(event: MotionEvent, index: Int, density: Float): SkikoTouchEvent {
-    return SkikoTouchEvent(
-        x = (event.getX(index) / density).toDouble(),
-        y = (event.getY(index) / density).toDouble(),
-        timestamp = event.getEventTime(),
-        kind = toSkikoTouchEventKind(event),
-        platform = event
-    )
-}
+fun toSkikoPointerEvent(event: MotionEvent, density: Float): SkikoPointerEvent {
+    val upIndex = when (event.action) {
+        MotionEvent.ACTION_UP -> 0
+        MotionEvent.ACTION_POINTER_UP -> event.actionIndex
+        else -> -1
+    }
 
-fun toSkikoGestureEvent(event: MotionEvent, density: Float): SkikoGestureEvent {
-    return SkikoGestureEvent(
-        x = (event.x / density).toDouble(),
-        y = (event.y / density).toDouble(),
-        kind = toSkikoGestureEventKind(event),
+    val pointers = (0 until event.pointerCount).map {
+        SkikoPointer(
+            x = event.getX(it).toDouble() / density,
+            y = event.getY(it).toDouble() / density,
+            // Same as in Jetpack Compose for Android
+            // https://github.com/androidx/androidx/blob/58597f0eba31b89f57b6605b7ed4977cd48ed38d/compose/ui/ui/src/androidMain/kotlin/androidx/compose/ui/input/pointer/MotionEventAdapter.android.kt#L126
+            // (Mouse isn't supported for Android yet, so hover cannot be checked)
+            pressed = it != upIndex,
+            device = SkikoPointerDevice.TOUCH,
+            id = event.getPointerId(it).toLong(),
+            pressure = event.getPressure(it).toDouble()
+        )
+    }
+
+    return SkikoPointerEvent(
+        x = pointers.centroidX,
+        y = pointers.centroidY,
+        kind = toSkikoPointerEventKind(event),
+        timestamp = event.eventTime,
+        pointers = pointers,
         platform = event
     )
 }
@@ -114,14 +126,14 @@ internal fun toSkikoGestureDirection(
     return SkikoGestureEventDirection.UNKNOWN
 }
 
-private fun toSkikoTouchEventKind(event: MotionEvent): SkikoTouchEventKind {
+internal fun toSkikoPointerEventKind(event: MotionEvent): SkikoPointerEventKind {
     return when (event.action) {
         MotionEvent.ACTION_POINTER_DOWN,
-        MotionEvent.ACTION_DOWN -> SkikoTouchEventKind.STARTED
+        MotionEvent.ACTION_DOWN -> SkikoPointerEventKind.DOWN
         MotionEvent.ACTION_POINTER_UP,
-        MotionEvent.ACTION_UP -> SkikoTouchEventKind.ENDED
-        MotionEvent.ACTION_MOVE -> SkikoTouchEventKind.MOVED
-        else -> SkikoTouchEventKind.UNKNOWN
+        MotionEvent.ACTION_UP -> SkikoPointerEventKind.UP
+        MotionEvent.ACTION_MOVE -> SkikoPointerEventKind.MOVE
+        else -> SkikoPointerEventKind.UNKNOWN
     }
 }
 

skiko/src/androidMain/kotlin/org/jetbrains/skiko/Renderer.android.kt

@@ -47,12 +47,7 @@ class SkikoSurfaceView(context: Context, val layer: SkiaLayer) : GLSurfaceView(c
     internal val gesturesDetector = SkikoGesturesDetector(context, layer)
 
     override fun onTouchEvent(event: MotionEvent): Boolean {
-        val events: MutableList<SkikoTouchEvent> = mutableListOf()
-        val count = event.pointerCount
-        for (index in 0 until count) {
-            events.add(toSkikoTouchEvent(event, index, layer.contentScale))
-        }
-        layer.skikoView?.onTouchEvent(events.toTypedArray())
+        layer.skikoView?.onPointerEvent(toSkikoPointerEvent(event, layer.contentScale))
         return true
     }
 

skiko/src/androidMain/kotlin/org/jetbrains/skiko/SkiaLayer.android.kt

@@ -8,7 +8,6 @@ import org.jetbrains.skia.PixelGeometry
 
 actual typealias SkikoGesturePlatformEvent = MotionEvent
 actual typealias SkikoPlatformPointerEvent = MotionEvent
-actual typealias SkikoTouchPlatformEvent = MotionEvent
 actual typealias SkikoPlatformInputEvent = KeyEvent
 actual typealias SkikoPlatformKeyboardEvent = KeyEvent
 

skiko/src/awtMain/kotlin/org/jetbrains/skiko/SkiaLayer.awt.kt

@@ -676,7 +676,6 @@ internal fun defaultFPSCounter(
 }
 
 // InputEvent is abstract, so we wrap to match modality.
-actual typealias SkikoTouchPlatformEvent = Any
 actual typealias SkikoGesturePlatformEvent = Any
 actual typealias SkikoPlatformInputEvent = Any
 actual typealias SkikoPlatformKeyboardEvent = KeyEvent

skiko/src/commonMain/kotlin/org/jetbrains/skiko/Events.kt

@@ -95,13 +95,13 @@ value class SkikoInputModifiers(val value: Int) {
 }
 
 enum class SkikoGestureEventKind {
-    TAP, DOUBLETAP, LONGPRESS, PAN, PINCH, ROTATION, SWIPE, UNKNOWN
+    UNKNOWN, TAP, DOUBLETAP, LONGPRESS, PAN, PINCH, ROTATION, SWIPE
 }
 enum class SkikoGestureEventDirection {
-    UP, DOWN, LEFT, RIGHT, UNKNOWN
+    UNKNOWN, UP, DOWN, LEFT, RIGHT
 }
 enum class SkikoGestureEventState {
-    PRESSED, STARTED, CHANGED, ENDED, UNKNOWN
+    UNKNOWN, PRESSED, STARTED, CHANGED, ENDED
 }
 expect class SkikoGesturePlatformEvent
 data class SkikoGestureEvent(
@@ -116,18 +116,6 @@ data class SkikoGestureEvent(
     val platform: SkikoGesturePlatformEvent? = null
 )
 
-enum class SkikoTouchEventKind {
-    STARTED, ENDED, MOVED, CANCELLED, UNKNOWN
-}
-expect class SkikoTouchPlatformEvent
-data class SkikoTouchEvent(
-    val x: Double,
-    val y: Double,
-    val kind: SkikoTouchEventKind = SkikoTouchEventKind.UNKNOWN,
-    val timestamp: Long = 0,
-    val platform: SkikoTouchPlatformEvent? = null
-)
-
 expect class SkikoPlatformInputEvent
 data class SkikoInputEvent(
     val input: String,
@@ -138,7 +126,7 @@ data class SkikoInputEvent(
 )
 
 enum class SkikoKeyboardEventKind {
-    UP, DOWN, TYPE, UNKNOWN
+    UNKNOWN, UP, DOWN, TYPE
 }
 expect class SkikoPlatformKeyboardEvent
 data class SkikoKeyboardEvent(
@@ -150,21 +138,45 @@ data class SkikoKeyboardEvent(
 )
 
 enum class SkikoPointerEventKind {
-    UP, DOWN, MOVE, DRAG, SCROLL, ENTER, EXIT, UNKNOWN
+    UNKNOWN, UP, DOWN, MOVE, DRAG, SCROLL, ENTER, EXIT
 }
 
 expect class SkikoPlatformPointerEvent
+
+// TODO(https://github.com/JetBrains/skiko/issues/680) refactor API
 data class SkikoPointerEvent(
+    /**
+     * X position in points (scaled pixels that depend on the scale factor of the current display).
+     *
+     * If the event contains multiple pointers, it represents the center of all pointers.
+     */
     val x: Double,
+    /**
+     * Y position in points (scaled pixels that depend on the scale factor of the current display)
+     *
+     * If the event contains multiple pointers, it represents the center of all pointers.
+     */
     val y: Double,
+    val kind: SkikoPointerEventKind,
+    /**
+     * Scroll delta along the X axis
+     */
     val deltaX: Double = 0.0,
+    /**
+     * Scroll delta along the Y axis
+     */
     val deltaY: Double = 0.0,
     val pressedButtons: SkikoMouseButtons = SkikoMouseButtons.NONE,
     val button: SkikoMouseButtons = SkikoMouseButtons.NONE,
     val modifiers: SkikoInputModifiers = SkikoInputModifiers.EMPTY,
-    val kind: SkikoPointerEventKind,
+    /**
+     * Timestamp in milliseconds
+     */
     val timestamp: Long = 0,
-    val platform: SkikoPlatformPointerEvent?
+    val pointers: List<SkikoPointer> = listOf(
+        SkikoPointer(0, x, y, pressedButtons.has(SkikoMouseButtons.LEFT))
+    ),
+    val platform: SkikoPlatformPointerEvent? = null
 )
 
 val SkikoPointerEvent.isLeftClick: Boolean
@@ -176,3 +188,54 @@ val SkikoPointerEvent.isRightClick: Boolean
 val SkikoPointerEvent.isMiddleClick: Boolean
     get() = button.has(SkikoMouseButtons.MIDDLE) && (kind == SkikoPointerEventKind.UP)
 
+/**
+ * The device type that produces pointer events, such as a mouse or stylus.
+ */
+enum class SkikoPointerDevice {
+    UNKNOWN, MOUSE, TOUCH
+}
+
+/**
+ * Represents pointer such as mouse cursor, or touch/stylus press.
+ * There can be multiple pointers on the screen at the same time.
+ */
+data class SkikoPointer(
+    /**
+     * Unique id associated with the pointer. Used to distinguish between multiple pointers that can exist
+     * at the same time (i.e. multiple pressed touches).
+     *
+     * If there is only on pointer in the system (for example, one mouse), it should always
+     * have the same id across multiple events.
+     */
+    val id: Long,
+
+    /**
+     * X position in points (scaled pixels that depend on the scale factor of the current display)
+     */
+    val x: Double,
+    /**
+     * Y position in points (scaled pixels that depend on the scale factor of the current display)
+     */
+    val y: Double,
+
+    /**
+     * `true` if the pointer event is considered "pressed." For example, finger
+     *  touching the screen or a mouse button is pressed [pressed] would be `true`.
+     *  During the up event, pointer is considered not pressed.
+     */
+    val pressed: Boolean,
+
+    /**
+     * The device type associated with the pointer, such as [mouse][SkikoPointerDevice.MOUSE],
+     * or [touch][SkikoPointerDevice.TOUCH].
+     */
+    val device: SkikoPointerDevice = SkikoPointerDevice.MOUSE,
+
+    /**
+     * Pressure of the pointer. 0.0 - no pressure, 1.0 - average pressure
+     */
+    val pressure: Double = 1.0,
+)
+
+internal val Iterable<SkikoPointer>.centroidX get() = asSequence().map { it.x }.average()
+internal val Iterable<SkikoPointer>.centroidY get() = asSequence().map { it.y }.average()

skiko/src/commonMain/kotlin/org/jetbrains/skiko/SkikoView.kt

@@ -10,7 +10,6 @@ interface SkikoView {
     @Deprecated("This method will be removed. Use override val input: SkikoInput")
     fun onInputEvent(event: SkikoInputEvent) = Unit
     val input: SkikoInput get() = SkikoInput.Empty
-    fun onTouchEvent(events: Array<SkikoTouchEvent>) = Unit
     fun onGestureEvent(event: SkikoGestureEvent) = Unit
 
     // Rendering
@@ -44,10 +43,6 @@ open class GenericSkikoView(
         app.onPointerEvent(event)
     }
 
-    override fun onTouchEvent(events: Array<SkikoTouchEvent>) {
-        app.onTouchEvent(events)
-    }
-
     override fun onGestureEvent(event: SkikoGestureEvent) {
         app.onGestureEvent(event)
     }

skiko/src/iosMain/kotlin/org/jetbrains/skiko/SkiaLayer.ios.kt

@@ -114,7 +114,6 @@ actual open class SkiaLayer {
 }
 
 // TODO: do properly
-actual typealias SkikoTouchPlatformEvent = UITouch
 actual typealias SkikoGesturePlatformEvent = UIEvent
 actual typealias SkikoPlatformInputEvent = UIPress
 actual typealias SkikoPlatformKeyboardEvent = UIPress

skiko/src/iosMain/kotlin/org/jetbrains/skiko/SkikoUIView.kt

@@ -24,6 +24,10 @@ class SkikoUIView : UIView, UIKeyInputProtocol, UITextInputProtocol,
     @OverrideInit
     constructor(coder: NSCoder) : super(coder)
 
+    init {
+        multipleTouchEnabled = true
+    }
+
     private var skiaLayer: SkiaLayer? = null
     private lateinit var _pointInside: (Point, UIEvent?) -> Boolean
     private var _inputDelegate: UITextInputDelegateProtocol? = null
@@ -179,34 +183,54 @@ class SkikoUIView : UIView, UIKeyInputProtocol, UITextInputProtocol,
 
     override fun touchesBegan(touches: Set<*>, withEvent: UIEvent?) {
         super.touchesBegan(touches, withEvent)
-        sendTouchEventToSkikoView(touches, SkikoTouchEventKind.STARTED)
+        sendTouchEventToSkikoView(withEvent!!, SkikoPointerEventKind.DOWN)
     }
 
     override fun touchesEnded(touches: Set<*>, withEvent: UIEvent?) {
         super.touchesEnded(touches, withEvent)
-        sendTouchEventToSkikoView(touches, SkikoTouchEventKind.ENDED)
+        sendTouchEventToSkikoView(withEvent!!, SkikoPointerEventKind.UP)
     }
 
     override fun touchesMoved(touches: Set<*>, withEvent: UIEvent?) {
         super.touchesMoved(touches, withEvent)
-        sendTouchEventToSkikoView(touches, SkikoTouchEventKind.MOVED)
+        sendTouchEventToSkikoView(withEvent!!, SkikoPointerEventKind.MOVE)
     }
 
     override fun touchesCancelled(touches: Set<*>, withEvent: UIEvent?) {
         super.touchesCancelled(touches, withEvent)
-        sendTouchEventToSkikoView(touches, SkikoTouchEventKind.CANCELLED)
-    }
+        sendTouchEventToSkikoView(withEvent!!, SkikoPointerEventKind.UP)
+    }
+
+    private fun sendTouchEventToSkikoView(event: UIEvent, kind: SkikoPointerEventKind) {
+        val pointers = event.touchesForView(this).orEmpty().map {
+            val touch = it as UITouch
+            val (x, y) = touch.locationInView(null).useContents { x to y }
+            SkikoPointer(
+                x = x,
+                y = y,
+                pressed = touch.isPressed,
+                device = SkikoPointerDevice.TOUCH,
+                id = touch.hashCode().toLong(),
+                pressure = touch.force
+            )
+        }
 
-    private fun sendTouchEventToSkikoView(touches: Set<*>, kind: SkikoTouchEventKind) {
-        val events = touches.map {
-            val event = it as UITouch
-            val (x, y) = event.locationInView(null).useContents { x to y }
-            val timestamp = (event.timestamp * 1_000).toLong()
-            SkikoTouchEvent(x, y, kind, timestamp, event)
-        }.toTypedArray()
-        skiaLayer?.skikoView?.onTouchEvent(events)
+        skiaLayer?.skikoView?.onPointerEvent(
+            SkikoPointerEvent(
+                x = pointers.centroidX,
+                y = pointers.centroidY,
+                kind = kind,
+                timestamp = event.timestamp.toLong() * 1_000,
+                pointers = pointers,
+                platform = event
+            )
+        )
     }
 
+    private val UITouch.isPressed get() =
+        phase != UITouchPhase.UITouchPhaseEnded &&
+            phase != UITouchPhase.UITouchPhaseCancelled
+
     override fun inputDelegate(): UITextInputDelegateProtocol? {
         return _inputDelegate
     }

skiko/src/jsMain/kotlin/org/jetbrains/skiko/SkiaLayer.js.kt

@@ -160,7 +160,6 @@ actual open class SkiaLayer {
 
 var onContentScaleChanged: ((Float) -> Unit)? = null
 
-actual typealias SkikoTouchPlatformEvent = Any
 actual typealias SkikoGesturePlatformEvent = Any
 actual typealias SkikoPlatformInputEvent = KeyboardEvent
 actual typealias SkikoPlatformKeyboardEvent = KeyboardEvent

skiko/src/linuxMain/kotlin/org/jetbrains/skiko/SkiaLayer.linux.kt

@@ -37,7 +37,6 @@ actual open class SkiaLayer  {
 }
 
 // TODO: do properly
-actual typealias SkikoTouchPlatformEvent = Any
 actual typealias SkikoGesturePlatformEvent = Any
 actual typealias SkikoPlatformInputEvent = Any
 actual typealias SkikoPlatformKeyboardEvent = Any

skiko/src/macosMain/kotlin/org/jetbrains/skiko/SkiaLayer.macos.kt

@@ -290,7 +290,6 @@ actual open class SkiaLayer {
 }
 
 // TODO: do properly
-actual typealias SkikoTouchPlatformEvent = NSEvent
 actual typealias SkikoGesturePlatformEvent = NSEvent
 actual typealias SkikoPlatformInputEvent = NSEvent
 actual typealias SkikoPlatformKeyboardEvent = NSEvent