Merge pull request #44 from openzim/enhance_errors

Display more useful errors in the UI

.github/workflows/Publish.yml

@@ -0,0 +1,34 @@
+name: Publish
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-deploy:
+    name: Build and deploy
+
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Retrieve source code
+        uses: actions/checkout@v3
+
+      - name: Build and publish Docker Image
+        uses: openzim/docker-publish-action@v10
+        with:
+          image-name: openzim/zimit-ui
+          on-master: latest
+          restrict-to: openzim/zimit-frontend
+          registries: ghcr.io
+          credentials:
+            GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+            GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+
+      - name: Deploy Zimit frontend changes to youzim.it
+        uses: actions-hub/kubectl@master
+        env:
+          KUBE_CONFIG: ${{ secrets.ZIMIT_KUBE_CONFIG }}
+        with:
+          args: rollout restart deployments ui-deployment -n zimit

.github/workflows/ci.yml → .github/workflows/QA.yml

@@ -1,23 +1,29 @@
-name: CI
+name: QA
 
-on: [push]
+on:
+  pull_request:
+  push:
+    branches:
+      - main
 
 jobs:
-  code-formating:
+  check-qa:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v1
-      with:
-        fetch-depth: 1
+    - name: Retrieve source code
+      uses: actions/checkout@v3
+
     - name: Set up Python 3.8
       uses: actions/setup-python@v1
       with:
         python-version: 3.8
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r api/requirements.txt
+
     - name: black code formatting check
       run: |
         pip install -U "black==22.3.0"

api/src/routes/errors.py

@@ -67,44 +67,40 @@ def handler_validationerror(e):
         return make_response(jsonify({"message": e.messages}), HTTPStatus.BAD_REQUEST)
 
 
-# 400
-class BadRequest(Exception):
+class ExceptionWithMessage(Exception):
     def __init__(self, message: str = None):
         self.message = message
 
+    @staticmethod
+    def handler(e, status: HTTPStatus):
+        if isinstance(e, ExceptionWithMessage) and e.message is not None:
+            return make_response(jsonify({"error": e.message}), status)
+        return Response(status=status)
+
+
+# 400
+class BadRequest(ExceptionWithMessage):
     @staticmethod
     def handler(e):
-        if isinstance(e, BadRequest) and e.message is not None:
-            return make_response(jsonify({"error": e.message}), HTTPStatus.BAD_REQUEST)
-        return Response(status=HTTPStatus.BAD_REQUEST)
+        return super().handler(e, HTTPStatus.BAD_REQUEST)
 
 
 # 401
-class Unauthorized(Exception):
-    def __init__(self, message: str = None):
-        self.message = message
-
+class Unauthorized(ExceptionWithMessage):
     @staticmethod
     def handler(e):
-        if isinstance(e, Unauthorized) and e.message is not None:
-            return make_response(jsonify({"error": e.message}), HTTPStatus.UNAUTHORIZED)
-        return Response(status=HTTPStatus.UNAUTHORIZED)
+        return super().handler(e, HTTPStatus.UNAUTHORIZED)
 
 
 # 404
-class NotFound(Exception):
-    def __init__(self, message: str = None):
-        self.message = message
-
+class NotFound(ExceptionWithMessage):
     @staticmethod
     def handler(e):
-        if isinstance(e, NotFound) and e.message is not None:
-            return make_response(jsonify({"error": e.message}), HTTPStatus.NOT_FOUND)
-        return Response(status=HTTPStatus.NOT_FOUND)
+        return super().handler(e, HTTPStatus.NOT_FOUND)
 
 
 # 500
-class InternalError(Exception):
+class InternalError(ExceptionWithMessage):
     @staticmethod
     def handler(e):
-        return Response(status=HTTPStatus.INTERNAL_SERVER_ERROR)
+        return super().handler(e, HTTPStatus.INTERNAL_SERVER_ERROR)

api/src/routes/requests.py

@@ -129,7 +129,14 @@ def post(self, *args, **kwargs):
         success, status, resp = query_api("POST", "/schedules/", payload=payload)
         if not success:
             logger.error(f"Unable to create schedule via HTTP {status}: {resp}")
-            raise InternalError(f"Unable to create schedule via HTTP {status}: {resp}")
+            message = f"Unable to create schedule via HTTP {status}: {resp}"
+            if status == http.HTTPStatus.BAD_REQUEST:
+                # if Zimfarm replied this is a bad request, then this is most probably
+                # a bad request due to user input so we can track it like a bad request
+                raise BadRequest(message)
+            else:
+                # otherwise, this is most probably an internal problem in our systems
+                raise InternalError(message)
 
         # request a task for that newly created schedule
         success, status, resp = query_api(
@@ -138,23 +145,24 @@ def post(self, *args, **kwargs):
             payload={"schedule_names": [schedule_name], "worker": TASK_WORKER},
         )
         if not success:
-            logger.error(f"Unable to request {schedule_name} via HTTP {status}")
-            logger.debug(resp)
-            raise InternalError(f"Unable to request schedule via HTTP {status}: {resp}")
+            logger.error(f"Unable to request {schedule_name} via HTTP {status}: {resp}")
+            raise InternalError(
+                f"Unable to request schedule via HTTP {status}): {resp}"
+            )
 
         try:
             task_id = resp.get("requested").pop()
             if not task_id:
-                raise ValueError("task_id is False")
+                raise InternalError("task_id is False")
         except Exception as exc:
             raise InternalError(f"Couldn't retrieve requested task id: {exc}")
 
         # remove newly created schedule (not needed anymore)
         success, status, resp = query_api("DELETE", f"/schedules/{schedule_name}")
         if not success:
-            logger.error(f"Unable to remove schedule {schedule_name} via HTTP {status}")
-            logger.debug(resp)
-
+            logger.error(
+                f"Unable to remove schedule {schedule_name} via HTTP {status}: {resp}"
+            )
         return make_response(jsonify({"id": str(task_id)}), http.HTTPStatus.CREATED)
 
 

dev/README.md

@@ -0,0 +1,66 @@
+This is a docker-compose configuration to be used **only** for development purpose. There is
+almost zero security in the stack configuration.
+
+It is composed of the Zimit frontend and API (of course), but also a local Zimfarm DB,
+API and UI, so that you can test the whole integration locally.
+
+Zimit UI and API are not deployed as they would be in production to allow hot reload of
+most modifications done to the source code.
+
+Zimfarm UI, API and DB are deployed with official production Docker images.
+
+## List of containers
+
+### zimit_ui
+
+This container is Zimit frontend web server (UI only)
+
+### zimit_api
+
+This container is Zimit API server (API only)
+
+## zimfarm_db
+
+This container is a local Zimfarm database
+
+## zimfarm_api
+
+This container is a local Zimfarm API
+
+## zimfarm_ui
+
+This container is a local Zimfarm UI
+
+## Instructions
+
+First start the Docker-Compose stack:
+
+```sh
+cd dev
+docker compose -p zimit up -d
+```
+
+If it is your first execution of the dev stack, you need to create a "virtual" worker in Zimfarm DB:
+
+```sh
+dev/create_worker.sh
+```
+
+If you have requested a task via Zimit UI and want to simulate a worker starting this task to observe the consequence in Zimit UI, you might use the `dev/start_first_req_task.sh`.
+
+## Restart the backend
+
+Should the API process fail, you might restart it with:
+```sh
+docker restart zimit-zimit_ui-1
+```
+
+## Browse the web UIs
+
+You might open following URLs in your favorite browser:
+
+- [Zimit UI](http://localhost:8001)
+- [Zimfarm API](http://localhost:8002)
+- [Zimfarm UI](http://localhost:8003)
+
+You can login into Zimfarm UI with username `admin` and password `admin`.

dev/create_worker.sh

@@ -0,0 +1,27 @@
+echo "Retrieving access token"
+
+ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \
+    'http://localhost:8002/v1/auth/authorize' \
+    -H 'accept: application/json' \
+    -H 'Content-Type: application/x-www-form-urlencoded' \
+    -d 'username=admin&password=admin' \
+    | jq -r '.access_token')"
+
+echo "Worker check-in (will create if missing)"
+
+curl -s -X 'PUT' \
+  'http://localhost:8002/v1/workers/worker/check-in' \
+  -H 'accept: */*' \
+  -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
+  -d '{
+  "username": "admin",
+  "cpu": 3,
+  "memory": 1024,
+  "disk": 0,
+  "offliners": [
+    "zimit"
+  ]
+}'
+
+echo "DONE"

dev/docker-compose.yml

@@ -0,0 +1,63 @@
+services:
+  zimfarm_db:
+    image: postgres:15.2-bullseye
+    ports:
+      - 127.0.0.1:5432:5432
+    volumes:
+      - zimfarm_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_DB=zimfarm
+      - POSTGRES_USER=zimfarm
+      - POSTGRES_PASSWORD=zimpass
+  zimfarm_api:
+    image: ghcr.io/openzim/zimfarm-dispatcher:latest
+    ports:
+      - 127.0.0.1:8004:80
+    environment:
+      BINDING_HOST: 0.0.0.0
+      JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw
+      POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@zimfarm_db:5432/zimfarm
+      ALEMBIC_UPGRADE_HEAD_ON_START: "1"
+      ZIMIT_USE_RELAXED_SCHEMA: "y"
+    depends_on:
+      - zimfarm_db
+  zimfarm-ui:
+    image: ghcr.io/openzim/zimfarm-ui:latest
+    ports:
+      - 127.0.0.1:8003:80
+    environment:
+      ZIMFARM_WEBAPI: http://localhost:8002/v1
+    depends_on:
+      - zimfarm_api
+  zimit_api:
+    build: ..
+    volumes:
+      - ../api/src:/app
+    command: python main.py
+    ports:
+      - 127.0.0.1:8002:8000
+    environment:
+      BINDING_HOST: 0.0.0.0
+      INTERNAL_ZIMFARM_WEBAPI: http://zimfarm_api:80/v1
+      _ZIMFARM_USERNAME: admin
+      _ZIMFARM_PASSWORD: admin
+      TASK_WORKER: worker
+    depends_on:
+      - zimfarm_api
+  zimit_ui:
+    build:
+      dockerfile: ../dev/zimit_ui_dev/Dockerfile
+      context: ../ui
+    volumes:
+      - ../ui/src:/app/src
+      - ../ui/public:/app/public
+      - ../dev/zimit_ui_dev/environ.json:/app/public/environ.json
+    ports:
+      - 127.0.0.1:8001:80
+    environment:
+      ZIMIT_API_URL: http://localhost:8002
+    depends_on:
+      - zimit_api
+
+volumes:
+  zimfarm_data:

dev/start_first_req_task.sh

@@ -0,0 +1,31 @@
+echo "Retrieving access token"
+
+ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \
+    'http://localhost:8002/v1/auth/authorize' \
+    -H 'accept: application/json' \
+    -H 'Content-Type: application/x-www-form-urlencoded' \
+    -d 'username=admin&password=admin' \
+    | jq -r '.access_token')"
+
+echo "Get last requested task"
+
+LAST_TASK_ID="$(curl -s -X 'GET' \
+  'http://localhost:8002/v1/requested-tasks/' \
+  -H 'accept: application/json' \
+  -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
+  | jq -r '.items[0]._id')"
+
+if [ "$LAST_TASK_ID" = "null" ]; then
+    echo "No pending requested task. Exiting script."
+    exit 1
+fi
+
+echo "Start task"
+
+curl -s -X 'POST' \
+  "http://localhost:8002/v1/tasks/$LAST_TASK_ID?worker_name=worker" \
+  -H 'accept: application/json' \
+  -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
+  -d ''
+
+echo "DONE"