chore: add Dockerfile to build with neko and ssh

Dockerfile.neko

@@ -0,0 +1,160 @@
+#####
+FROM darkness4/katago:latest as server
+#####
+
+WORKDIR /src
+
+#
+# install dependencies
+RUN set -eux; apt update -y; \
+  apt install -y --no-install-recommends curl build-essential git cmake make libx11-dev libxrandr-dev libxtst-dev \
+  libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly; \
+  # install golang
+  curl -fsSL https://go.dev/dl/go1.21.5.linux-amd64.tar.gz -o go1.21.5.linux-amd64.tar.gz \
+  && tar -C /usr/local -xzf go1.21.5.linux-amd64.tar.gz \
+  && rm -f go1.21.5.linux-amd64.tar.gz; \
+  #
+  # install libclipboard
+  set -eux; \
+  cd /tmp; \
+  git clone --depth=1 https://github.com/jtanx/libclipboard; \
+  cd libclipboard; \
+  cmake .; \
+  make -j4; \
+  make install; \
+  rm -rf /tmp/libclipboard; \
+  #
+  # clean up
+  apt clean -y; \
+  rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+ENV PATH=${PATH}:/usr/local/go/bin
+
+RUN git clone https://github.com/m1k1o/neko.git \
+  && cd ./neko/server \
+  && ./build
+
+# ------------------
+FROM darkness4/katago:latest as runtime
+# ------------------
+
+ARG USERNAME=neko
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+RUN set -eux; \
+  apt update -y; \
+  #
+  # install dependencies
+  apt install -y --no-install-recommends wget ca-certificates supervisor git curl; \
+  apt install -y --no-install-recommends pulseaudio dbus-x11 xserver-xorg-video-dummy; \
+  apt install -y --no-install-recommends libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx7; \
+  #
+  # gst
+  apt install -y --no-install-recommends libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev \
+  gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-pulseaudio; \
+  #
+  # install fonts
+  apt install -y --no-install-recommends \
+  # Google emojis
+  fonts-noto-color-emoji \
+  # Japanese fonts
+  fonts-takao-mincho \
+  # Chinese fonts
+  fonts-wqy-zenhei xfonts-intl-chinese xfonts-wqy \
+  # Korean fonts
+  fonts-wqy-microhei; \
+  #
+  # create a non-root user
+  groupadd --gid $USER_GID $USERNAME; \
+  useradd --uid $USER_UID --gid $USERNAME --shell /bin/bash --create-home $USERNAME; \
+  adduser $USERNAME audio; \
+  adduser $USERNAME video; \
+  adduser $USERNAME pulse; \
+  #
+  # workaround for an X11 problem: http://blog.tigerteufel.de/?p=476
+  mkdir /tmp/.X11-unix; \
+  chmod 1777 /tmp/.X11-unix; \
+  chown $USERNAME /tmp/.X11-unix/; \
+  #
+  # make directories for neko
+  mkdir -p /etc/neko /var/www /var/log/neko \
+  /tmp/runtime-$USERNAME \
+  /home/$USERNAME/.config/pulse  \
+  /home/$USERNAME/.local/share/xorg; \
+  chmod 1777 /var/log/neko; \
+  chown $USERNAME /var/log/neko/ /tmp/runtime-$USERNAME; \
+  chown -R $USERNAME:$USERNAME /home/$USERNAME; \
+  #
+  # clean up
+  apt clean -y; \
+  rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+# copy config files
+RUN git clone https://github.com/m1k1o/neko.git \
+  && cp neko/.docker/base/dbus /usr/bin/dbus \
+  && cp neko/.docker/base/default.pa /etc/pulse/default.pa \
+  && cp neko/.docker/base/supervisord.conf /etc/neko/supervisord.conf \
+  && cp neko/.docker/base/xorg.conf /etc/neko/xorg.conf \
+  && rm -rf neko/
+
+ENV USER=$USERNAME
+ENV DISPLAY=:99.0
+ENV PULSE_SERVER=unix:/tmp/pulseaudio.socket
+ENV XDG_RUNTIME_DIR=/tmp/runtime-$USERNAME
+ENV NEKO_PASSWORD=neko
+ENV NEKO_PASSWORD_ADMIN=admin
+ENV NEKO_BIND=:8080
+
+COPY --from=server /src/neko/server/bin/neko /usr/bin/neko
+COPY --from=m1k1o/neko:latest /var/www/ /var/www
+
+#
+# run neko
+ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/neko/supervisord.conf"]
+
+# ------------------
+FROM darkness4/katago:latest as sabaki
+# ------------------
+
+WORKDIR /src
+
+RUN set -eux; apt update -y; \
+  apt install -y git ca-certificates curl gnupg; \
+  #
+  mkdir -p /etc/apt/keyrings; \
+  curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg; \
+  echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list; \
+  #
+  apt update -y; \
+  apt install nodejs -y; \
+  #
+  apt clean -y; \
+  rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+RUN git clone https://github.com/SabakiHQ/Sabaki \
+  && cd Sabaki \
+  && npm i \
+  && npm run dist:linux
+
+# ------------------
+FROM runtime
+# ------------------
+
+COPY --from=sabaki /src/Sabaki/dist/sabaki-v0.52.2-linux-x64.AppImage /usr/bin/sabaki
+
+RUN set -eux; apt update -y; \
+  apt install -y --no-install-recommends openbox libatk1.0-0 libatk-bridge2.0-0 libgtk-3-0; \
+  #
+  mkdir -p /home/neko/.config/Sabaki; \
+  chown -R neko /home/neko/.config; \
+  #
+  apt clean -y; \
+  rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+#
+# copy configuation files
+COPY neko/supervisord.conf /etc/neko/supervisord/sabaki.conf
+COPY neko/openbox.xml /etc/neko/openbox.xml
+COPY --chown=neko neko/settings.json /home/neko/.config/Sabaki/settings.json
+COPY neko/default_gtp.cfg /app/default_gtp.cfg

Dockerfile.ssh

@@ -0,0 +1,9 @@
+# ------------------
+FROM darkness4/katago:latest
+# ------------------
+
+RUN apt update -y && apt install --no-install-recommends -y \
+  ssh curl wget \
+  && rm -rf /var/lib/apt/lists/*
+
+ENTRYPOINT ["/usr/sbin/sshd", "-D"]

README.md

@@ -176,3 +176,85 @@ You can customize the Dockerfile by setting `--build-arg` parameters like `CUDA_
    # Model name: kata1-b18c384nbt-s5832081920-d3223508649
    # GTP ready, beginning main protocol loop
    ```
+
+## Running on DeepSquare
+
+DeepSquare is a decentralized HPC as a service. You can read more about it on [DeepSquare's official website](https://deepsquare.io) and try the [portal](https://app.deepsquare.run).
+
+![image-20231216180752422](./README.assets/image-20231216180752422.png)
+
+### How to use with Neko
+
+Install the [`dps`](https://docs.deepsquare.run/workflow/cli/getting-started), [the DeepSquare CLI](https://docs.deepsquare.run/workflow/cli/getting-started).
+
+Using with [Neko](https://neko.m1k1o.net/#/) is the easiest way to use Sabaki and Katago.
+
+Just submit the job:
+
+```shell
+dps submit -w -e --no-ts --job-name katago --credits 10000 ./job.katago.yaml
+```
+
+A bore URL will appear in the logs, just connect to it (credentials: user: `admin` and pass: `neko`).
+
+You can start the engine by opening the Engines Sidebar (Engines > Show Engines Sidebar).
+
+![image-20231216170403046](./README.assets/image-20231216170403046.png)
+
+Then, you can toggle the Analysis mode with <kbd>F4</kbd>.
+
+### How to use with SSH
+
+1. Edit the job, and replace the SSH public key.
+
+2. You need to start Sabaki locally.
+
+3. Then, you can submit the job:
+
+   ```shell
+   dps submit -w -e --no-ts --job-name katago --credits 10000 ./job.yaml
+   ```
+
+   The user ID and bore URL will appear. This is the URL to the SSH server. You need to use the TCP one.
+
+4. Write a script on your computer to connect to the server via SSH and launch katago with Sabaki:
+
+   **/projects/katago/script**
+
+   ```shell
+   #!/bin/sh
+   set -e
+
+   SCRIPTPATH="$(dirname "$(realpath "$0")")"
+
+   # Copy config file to remote server
+   scp -P <port> "$SCRIPTPATH/default_gtp.cfg" <user>@bore.deepsquare.run:/tmp/default_gtp.cfg
+
+   ssh -p <port> <user>@bore.deepsquare.run "set -e
+
+   # Change this if you want to use an another model, see https://katagotraining.org
+   RELEASE=kata1-b18c384nbt-s8493331456-d3920571699
+
+   if [ ! -f \$DEEPSQUARE_DISK_TMP/default_model.bin.gz ]; then
+     curl -fsSL https://media.katagotraining.org/uploaded/networks/models/kata1/\$RELEASE.bin.gz -o \$DEEPSQUARE_DISK_TMP/default_model.bin.gz
+   fi
+
+   /app/katago $@ -model \$DEEPSQUARE_DISK_TMP/default_model.bin.gz -config /tmp/default_gtp.cfg"
+
+   ```
+
+   Replace `<user>` and `<port>` with the user ID and bore port.
+
+5. Add an Engine from the Engines Sidebar (Engines > Show Engines Sidebar) with:
+
+   Path: The path to the script (`/projects/katago/script`)
+
+   Arguments: `gtp`
+
+   ![image-20231216172443905](./README.assets/image-20231216172443905.png)
+
+6. Then start the engine:
+
+   ![image-20231216170403046](./README.assets/image-20231216170403046.png)
+
+7. Finally, you can toggle the Analysis mode with <kbd>F4</kbd>.

job.neko.yaml

@@ -0,0 +1,33 @@
+# yaml-language-server: $schema=/home/marc/.cache/.job.schema.json
+## See: https://docs.deepsquare.run/workflow/workflow-api-reference/job
+
+## Allow DeepSquare logging
+enableLogging: true
+
+## Allocate resources
+resources:
+  tasks: 1
+  cpusPerTask: 4
+  memPerCpu: 8000
+  gpus: 1
+
+## The job content
+steps:
+  ## The steps of the jobs which are run sequentially.
+  - name: 'katago'
+    run:
+      container:
+        image: darkness4/katago:latest-neko
+      network: slirp4netns
+      resources:
+        gpusPerTask: 1
+      customNetworkInterfaces:
+        - bore:
+            boreAddress: bore.deepsquare.run:2200
+            targetPort: 8080
+      mapGid: 0
+      mapUid: 0
+      command: |
+        mkdir -p $HOME/.config/Sabaki
+        cp /home/neko/.config/Sabaki/settings.json $HOME/.config/Sabaki/settings.json
+        /usr/bin/supervisord -c /etc/neko/supervisord.conf

job.yaml

@@ -0,0 +1,61 @@
+# yaml-language-server: $schema=/home/marc/.cache/.job.schema.json
+## See: https://docs.deepsquare.run/workflow/workflow-api-reference/job
+
+## Allow DeepSquare logging
+enableLogging: true
+
+## Allocate resources
+resources:
+  tasks: 1
+  cpusPerTask: 1
+  memPerCpu: 8000
+  gpus: 1
+
+## The job content
+steps:
+  ## The steps of the jobs which are run sequentially.
+  - name: 'katago'
+    run:
+      container:
+        image: darkness4/katago:latest-ssh
+      network: slirp4netns
+      resources:
+        gpusPerTask: 1
+      customNetworkInterfaces:
+        - bore:
+            boreAddress: bore.deepsquare.run:2200
+            targetPort: 2200
+      command: |
+        mkdir -p "$HOME/.ssh"
+        echo "<pubkey>" >> "$HOME/.ssh/authorized_keys"
+        chmod 600 "$HOME/.ssh/authorized_keys"
+        chmod 700 "$HOME/.ssh"
+
+        ssh-keygen -A
+
+        echo $USER
+
+        /usr/sbin/sshd -D -p 2200
+
+#### Use a script to interact with it
+
+# #!/bin/sh
+# # katago-remote.sh
+#
+# set -e
+#
+# SCRIPTPATH="$(dirname "$(realpath "$0")")"
+#
+# # Copy config file to remote server
+# scp -P <port> "$SCRIPTPATH/default_gtp.cfg" <user>@bore.deepsquare.run:/tmp/default_gtp.cfg
+#
+# ssh -p <port> <user>@bore.deepsquare.run "set -e
+#
+# # Change this if you want to use an another model, see https://katagotraining.org
+# RELEASE=kata1-b18c384nbt-s8493331456-d3920571699
+#
+# if [ ! -f \$DEEPSQUARE_DISK_TMP/default_model.bin.gz ]; then
+#   curl -fsSL https://media.katagotraining.org/uploaded/networks/models/kata1/\$RELEASE.bin.gz -o \$DEEPSQUARE_DISK_TMP/default_model.bin.gz
+# fi
+#
+# /app/katago $@ -model \$DEEPSQUARE_DISK_TMP/default_model.bin.gz -config /tmp/default_gtp.cfg"