Add CTAP2 test suite

.dockerignore

@@ -0,0 +1 @@
+cache

.gitignore

@@ -3,6 +3,7 @@
 
 __pycache__
 bin
+cache
 data
 state
 venv

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "external/CTAP2-test-tool"]
+	path = external/CTAP2-test-tool
+	url = https://github.com/google/CTAP2-test-tool

Dockerfile

@@ -3,8 +3,13 @@
 
 FROM debian:stable
 
-RUN apt-get update
-RUN apt-get install --yes git kmod make openssh-client openssh-server python3 python3-venv usbip usbutils
+RUN apt-get update && apt-get install --yes curl git gnupg kmod make openssh-client openssh-server python3 python3-venv usbip usbutils
+
+# Install bazel for CTAP2-test-tool
+RUN curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor > bazel.gpg
+RUN mv bazel.gpg /etc/apt/trusted.gpg.d/
+RUN echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list
+RUN apt-get update && apt-get install --yes bazel libudev-dev autotools-dev autoconf automake libtool g++
 
 RUN useradd --create-home --user-group user
 

stubs/pexpect.pyi

@@ -1,7 +1,7 @@
 # Copyright (C) 2022 Nitrokey GmbH
 # SPDX-License-Identifier: CC0-1.0
 
-from typing import Union
+from typing import IO, Optional, Union
 
 
 class EOF:
@@ -14,6 +14,9 @@ class spawn:
         cmd: str,
         args: list[str] = [],
         timeout: int = 30,
+        cwd: Optional[str] = None,
+        encoding: Optional[str] = None,
+        logfile: Optional[Union[IO[str], IO[bytes]]] = None,
     ) -> None:
         pass
 

tests/ctap2.py

@@ -0,0 +1,52 @@
+# Copyright (C) 2023 Nitrokey GmbH
+# SPDX-License-Identifier: CC0-1.0
+
+import glob
+import re
+from sys import stderr
+
+import pytest
+from pexpect import spawn
+
+
+PROMPT_REBOOT = "Please replug the device, then hit enter."
+PROMPT_TOUCH = "Please touch your security key!"
+OUTPUT_RESULTS = "RESULTS"
+PATTERNS = [PROMPT_REBOOT, PROMPT_TOUCH, OUTPUT_RESULTS]
+
+
+def detect_tests() -> list[str]:
+    regex = re.compile("BaseTest\\(\\s*\"(\\w+)\"")
+    tests = []
+    for fname in glob.glob("./external/CTAP2-test-tool/src/tests/*.cc"):
+        with open(fname) as f:
+            text = f.read()
+        tests.extend(regex.findall(text))
+    return tests
+
+
+@pytest.mark.parametrize("test", detect_tests())
+@pytest.mark.virtual
+def test_ctap2(touch_device, test) -> None:
+    cmd = "bazel --output_user_root /app/cache/bazel run " \
+        f"//:fido2_conformance -- --token_path=/dev/{touch_device.hidraw} " \
+        f"--test_ids={test}"
+    p = spawn(
+        cmd,
+        cwd="./external/CTAP2-test-tool",
+        encoding="utf-8",
+        logfile=stderr,
+        timeout=60,
+    )
+
+    while True:
+        i = p.expect(PATTERNS)
+        if PATTERNS[i] == PROMPT_REBOOT:
+            touch_device.reboot()
+            p.sendline()
+        elif PATTERNS[i] == PROMPT_TOUCH:
+            touch_device.confirm_user_presence()
+        elif PATTERNS[i] == OUTPUT_RESULTS:
+            break
+        else:
+            raise Exception(f"Unexpected pattern index {i}")

utils/device.py

@@ -154,14 +154,16 @@ def _spawn(binary: str, state: UsbipState) -> tuple[Popen[bytes], str]:
 
     host = "localhost"
     check_call(["usbip", "list", "-r", host])
-    check_call(["usbip", "attach", "-r", host, "-b", "1-1"])
-    check_call(["usbip", "attach", "-r", host, "-b", "1-1"])
 
     for i in range(5):
-        if not find_devices(0x20a0, 0x42b2):
-            time.sleep(1)
-        else:
+        check_call(["usbip", "attach", "-r", host, "-b", "1-1"])
+        check_call(["usbip", "attach", "-r", host, "-b", "1-1"])
+
+        _await_device()
+
+        if find_devices(0x20a0, 0x42b2):
             break
+
     hidraw = find_device(0x20a0, 0x42b2)
 
     for i in range(5):
@@ -175,6 +177,14 @@ def _spawn(binary: str, state: UsbipState) -> tuple[Popen[bytes], str]:
     return (runner, hidraw)
 
 
+def _await_device() -> None:
+    for i in range(5):
+        if not find_devices(0x20a0, 0x42b2):
+            time.sleep(1)
+        else:
+            break
+
+
 device_pin: Optional[str] = None