Manage ActivityPub interface to broadcast videos and get external ones

.env.dev-exemple

@@ -10,3 +10,6 @@ REDIS_TAG=redis:alpine3.16
 ### In case of value changing, you have to rebuild and restart your container.
 ### All yours datas will be kept.
 DOCKER_ENV=light
+## PEERTUBE SECRETS
+POSTGRES_PASSWORD=<PWD>
+PEERTUBE_SECRET=<SECRET>

.gitignore

@@ -50,6 +50,7 @@ pod/main/static/custom/img
 !pod/custom/settings_local.py.example
 settings_local.py
 transcription/*
+docker-volume
 
 # Unit test utilities #
 #######################
@@ -73,6 +74,7 @@ compile-model
 *.crt
 *.key
 *.pem
+*.pub
 
 # NPM stuffs #
 ################

Makefile

@@ -174,3 +174,9 @@ endif
 	sudo rm -rf ./pod/db.sqlite3
 	sudo rm -rf ./pod/db_remote.sqlite3
 	sudo rm -rf ./pod/media
+
+# Ouvre un shell avec le contexte Django dans le conteneur pod
+shell:
+	docker exec -it pod-activitypub-with-volumes pip install ipython
+	docker exec -it pod-activitypub-with-volumes env DJANGO_SETTINGS_MODULE=pod.settings ipython --ext=autoreload -c "%autoreload 2" -i
+	# then it is needed to call import django; django.setup()

docker-compose-full-dev-with-volumes.yml

@@ -56,6 +56,18 @@ services:
       - ./.env.dev
     volumes: *pod-volumes
 
+  pod-activitypub:
+    container_name: pod-activitypub-with-volumes
+    build:
+      context: .
+      dockerfile: dockerfile-dev-with-volumes/pod-activitypub/Dockerfile
+    depends_on:
+      - pod-back
+      - redis
+    env_file:
+      - ./.env.dev
+    volumes: *pod-volumes
+
   elasticsearch:
     container_name: elasticsearch-with-volumes
     hostname: elasticsearch.localhost
@@ -79,6 +91,37 @@ services:
     ports:
       - 6379:6379
 
+  peertube:
+    container_name: peertube
+    hostname: peertube.localhost
+    image: chocobozzz/peertube:develop-bookworm
+    ports:
+      - 9000:9000
+      - 3000:3000
+    env_file:
+      - ./dockerfile-dev-with-volumes/peertube/peertube.env
+      - ./.env.dev
+    depends_on:
+      - postgres
+      - redis
+      - postfix
+    command: sh -c "yarn install && npm run dev"
+    restart: "always"
+
+  postgres:
+    image: postgres:13-alpine
+    env_file:
+      - ./dockerfile-dev-with-volumes/peertube/peertube.env
+      - ./.env.dev
+    restart: "always"
+
+  postfix:
+    image: mwader/postfix-relay
+    env_file:
+      - ./dockerfile-dev-with-volumes/peertube/peertube.env
+      - ./.env.dev
+    restart: "always"
+
 #  redis-commander:
 #    container_name: redis-commander
 #    hostname: redis-commander.localhost

dockerfile-dev-with-volumes/peertube/peertube.env

@@ -0,0 +1,60 @@
+NODE_VERSION=21.7.1
+NODE_ENV=dev
+NODE_DB_LOG=false
+
+# Database / Postgres service configuration
+POSTGRES_USER=postgres
+# Postgres database name "peertube"
+POSTGRES_DB=peertube_dev
+# The database name used by PeerTube will be PEERTUBE_DB_NAME (only if set) *OR* 'peertube'+PEERTUBE_DB_SUFFIX
+#PEERTUBE_DB_NAME=<MY POSTGRES DB NAME>
+#PEERTUBE_DB_SUFFIX=_prod
+# Database username and password used by PeerTube must match Postgres', so they are copied:
+PEERTUBE_DB_USERNAME=$POSTGRES_USER
+PEERTUBE_DB_PASSWORD=$POSTGRES_PASSWORD
+PEERTUBE_DB_SSL=false
+# Default to Postgres service name "postgres" in docker-compose.yml
+PEERTUBE_DB_HOSTNAME=postgres
+
+# PeerTube server configuration
+# If you test PeerTube in local: use "peertube.localhost" and add this domain to your host file resolving on 127.0.0.1
+PEERTUBE_WEBSERVER_HOSTNAME=peertube.localhost
+# If you just want to test PeerTube on local
+PEERTUBE_WEBSERVER_PORT=9000
+PEERTUBE_WEBSERVER_HTTPS=false
+# If you need more than one IP as trust_proxy
+# pass them as a comma separated array:
+PEERTUBE_TRUST_PROXY=["127.0.0.1", "loopback", "172.18.0.0/16"]
+
+# E-mail configuration
+# If you use a Custom SMTP server
+#PEERTUBE_SMTP_USERNAME=
+#PEERTUBE_SMTP_PASSWORD=
+# Default to Postfix service name "postfix" in docker-compose.yml
+# May be the hostname of your Custom SMTP server
+PEERTUBE_SMTP_HOSTNAME=postfix
+PEERTUBE_SMTP_PORT=25
+PEERTUBE_SMTP_FROM=[email protected]
+PEERTUBE_SMTP_TLS=false
+PEERTUBE_SMTP_DISABLE_STARTTLS=false
+PEERTUBE_ADMIN_EMAIL=[email protected]
+
+# Postfix service configuration
+POSTFIX_myhostname=example.org
+# If you need to generate a list of sub/DOMAIN keys
+# pass them as a whitespace separated string <DOMAIN>=<selector>
+OPENDKIM_DOMAINS=example.org=peertube
+# see https://github.com/wader/postfix-relay/pull/18
+OPENDKIM_RequireSafeKeys=no
+
+PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PUBLIC="public-read"
+PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PRIVATE="private"
+
+#PEERTUBE_LOG_LEVEL=info
+
+# /!\ Prefer to use the PeerTube admin interface to set the following configurations /!\
+#PEERTUBE_SIGNUP_ENABLED=true
+#PEERTUBE_TRANSCODING_ENABLED=true
+#PEERTUBE_CONTACT_FORM_ENABLED=true
+
+PEERTUBE_REDIS_HOSTNAME=redis-with-volumes

dockerfile-dev-with-volumes/pod-activitypub/Dockerfile

@@ -0,0 +1,35 @@
+#------------------------------------------------------------------------------------------------------------------------------
+#  (\___/)
+#  (='.'=) Dockerfile multi-stages node & python
+#  (")_(")
+#------------------------------------------------------------------------------------------------------------------------------
+# Conteneur node
+ARG PYTHON_VERSION
+# TODO
+#FROM harbor.urba.univ-lille.fr/store/node:19 as source-build-js
+
+#------------------------------------------------------------------------------------------------------------------------------
+# Conteneur python
+FROM $PYTHON_VERSION
+WORKDIR /tmp/pod
+COPY ./pod/ .
+# TODO
+#FROM harbor.urba.univ-lille.fr/store/python:3.7-buster
+
+RUN apt-get clean && apt-get update && apt-get install -y netcat
+
+WORKDIR /usr/src/app
+
+COPY ./requirements.txt .
+COPY ./requirements-conteneur.txt .
+COPY ./requirements-encode.txt .
+COPY ./requirements-dev.txt .
+
+RUN pip3 install --no-cache-dir -r requirements-dev.txt \
+    && pip3 install elasticsearch==7.17.9
+
+# ENTRYPOINT :
+COPY ./dockerfile-dev-with-volumes/pod-activitypub/my-entrypoint-activitypub.sh /tmp/my-entrypoint-activitypub.sh
+RUN chmod 755 /tmp/my-entrypoint-activitypub.sh
+
+ENTRYPOINT ["bash", "/tmp/my-entrypoint-activitypub.sh"]

dockerfile-dev-with-volumes/pod-activitypub/my-entrypoint-activitypub.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+echo "Launching commands into pod-dev"
+until nc -z pod-back 8000; do echo waiting for pod-back; sleep 10; done;
+# Worker ActivityPub
+env DJANGO_SETTINGS_MODULE=pod.settings \
+    python -m watchdog.watchmedo auto-restart --directory pod --pattern '*.py' --recursive --\
+    celery --app pod.activitypub.tasks worker --loglevel INFO --queues activitypub --concurrency 1 --hostname activitypub
+sleep infinity

pod/activitypub/README.md

@@ -0,0 +1,141 @@
+# ActivityPub implementation
+
+Pod implements a minimal set of ActivityPub that allows video sharing between Pod instances.
+The ActivityPub implementation is also compatible with Peertube.
+
+## Federation
+
+Here is what happens when two instances, say *Node A* and *Node B* (being Pod or Peertube) federate with each other, in a one way federation.
+
+### Federation
+
+- An administrator asks for Node A to federate with Node B
+- Node A reaches the [NodeInfo](https://github.com/jhass/nodeinfo/blob/main/PROTOCOL.md) endpoint (`/.well-known/nodeinfo`) on Node B and discover the root application endpoint URL.
+- Node A reaches the root application endpoint (for Pod this is `/ap/`) and get the `inbox` URL.
+- Node A sends a `Create` activity for a `Follow` object on the Node B root application `inbox`.
+- Node B reads the Node A root application endpoint URL in the `Follow` objects, reaches this endpoint and get the Node A root application `inbox` URL.
+- Node B creates a `Follower` objects and stores it locally
+- Node B sends a `Accept` activity for the `Follower` object on Node A root application enpdoint.
+- Later, Node A can send to Node B a `Undo` activity for the `Follow` object to de-federate.
+
+### Video discovery
+
+- Node A reaches the Node B root application `outbox`.
+- Node A browse the pages of the `outbox` and look for announces about `Videos`
+- Node A reaches the `Video` endpoints and store locally the information about the videos.
+
+### Video creation and update sharing
+
+#### Creation
+
+- A user of Node B publishes a `Video`
+- Node B sends a `Announce` activity on the `inbox` of all its `Followers`, including Node A with the ID of the new video.
+- Node A reads the information about the new `Video` on Node B video endpoint.
+
+#### Edition
+
+- A user of Node B edits a `Video`
+- Node B sends a `Update` activity on the `inbox` of all its `Followers`, including Node A with the ID of the new video, containing the details of the `Video`.
+
+#### Deletion
+
+- A user of Node B deletes a `Video`
+- Node B sends a `Delete` activity on the `inbox` of all its `Followers`, including Node A with the ID of the new video.
+
+## Implementation
+
+The ActivityPub implementation tries to replicate the network messages of Peertube.
+There may be things that could have been done differently while still following the ActivityPub specs, but changing the network exchanges would require checking if the Peertube compatibility is not broken.
+This is due to Peertube having a few undocumented behaviors that are not exactly part of the AP specs.
+
+To achieve compatibility with Peertube, Pod implements two specifications to sign ActivityPub exchanges.
+
+- [Signing HTTP Messages, draft 12](https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-12).
+  This specification is replaced by [RFC9421](https://www.rfc-editor.org/rfc/rfc9421.html) but Peertube does not implement the finale spec,
+  and instead lurks on the writing of the [ActivityPub and HTTP Signatures](https://swicg.github.io/activitypub-http-signature/) spec, that is also still a draft.
+  See the [related discussion](https://framacolibri.org/t/rfc9421-replaces-the-signing-http-messages-draft/20911/2).
+  This spec describe how to sign ActivityPub payload with HTTP headers.
+- [Linked Data Signatures 1.0](https://web.archive.org/web/20170717200644/https://w3c-dvcg.github.io/ld-signatures/) draft.
+  This specification is replaced by [Verifiable Credential Data Integrity](https://w3c.github.io/vc-data-integrity/) but Peertube does not implement the finale spec.
+  This spec describe how to sign ActivityPub payload by adding fields in the payload.
+
+The state of the specification support in Peertube is similar to [Mastodon](https://docs.joinmastodon.org/spec/security/), and is probably a mean to keep the two software compatible with each other.
+
+## Limitations
+
+- Peertube instance will only be able to index Pod videos if the video thumbnails are absent.
+- Peertube instance will only be able to index Pod videos if the thumbnails are in JPEG format.
+  png thumbnails are not supported at the moment (but that may come in the future
+  [more details here](https://framacolibri.org/t/comments-and-suggestions-on-the-peertube-activitypub-implementation/21215)).
+  In the meantime, pod fakes the mime-type of all thumbnails to be JPEG, even when they actually are PNGs.
+
+## Configuration
+
+A RSA keypair is needed for ActivityPub to work, and passed as
+`ACTIVITYPUB_PUBLIC_KEY` and `ACTIVITYPUB_PRIVATE_KEY` configuration settings.
+They can be generated from a python console:
+
+```python
+from Crypto.PublicKey import RSA
+
+activitypub_key = RSA.generate(2048)
+
+# Generate the private key
+# Add the content of this command in 'pod/custom/settings_local.py'
+# in a variable named ACTIVITYPUB_PRIVATE_KEY
+with open("pod/activitypub/ap.key", "w") as fd:
+    fd.write(activitypub_key.export_key().decode())
+
+# Generate the public key
+# Add the content of this command in 'pod/custom/settings_local.py'
+# in a variable named ACTIVITYPUB_PUBLIC_KEY
+with open("pod/activitypub/ap.pub", "w") as fd:
+    fd.write(activitypub_key.publickey().export_key().decode())
+```
+
+The federation also needs celery to be configured with `ACTIVITYPUB_CELERY_BROKER_URL`.
+
+Here is a sample working activitypub `pod/custom/settings_local.py`:
+
+```python
+ACTIVITYPUB_CELERY_BROKER_URL = "redis://redis:6379/5"
+
+with open("pod/activitypub/ap.key") as fd:
+    ACTIVITYPUB_PRIVATE_KEY = fd.read()
+
+with open("pod/activitypub/ap.pub") as fd:
+    ACTIVITYPUB_PUBLIC_KEY = fd.read()
+```
+
+## Development
+
+The `DOCKER_ENV` environment var should be set to `full` so a peertube instance and a ActivityPub celery worker are launched.
+Then peertube is available at http://peertube.localhost:9000.
+
+### Federate Peertube with Pod
+
+- Sign in with the `root` account
+- Go to [Main menu > Administration > Federation](http://peertube.localhost:9000/admin/follows/following-list) > Follow
+- Open the *Follow* modal and type `pod.localhost:8000`
+
+### Federate Pod with Peertube
+
+- Sign in with `admin`
+- Go to the [Administration pannel > Followings](http://pod.localhost:8000/admin/activitypub/following/) > Add following
+- Type `http://peertube.localhost:9000` in *Object* and save
+- On the [Followings list](http://pod.localhost:8000/admin/activitypub/following/) select the new object, and select `Send the federation request` in the action list, refresh.
+- If the status is *Following request accepted* then select the object again, and choose `Reindex instance videos` in the action list.
+
+## Shortcuts
+
+### Manual AP request
+
+```shell
+curl -H "Accept: application/activity+json, application/ld+json" -s "http://pod.localhost:8000/accounts/peertube" | jq
+```
+
+### Unit tests
+
+```shell
+python manage.py test --settings=pod.main.test_settings pod.activitypub.tests
+```