feat: integrate classifier script

.github/workflows/server.yml

@@ -92,3 +92,44 @@ jobs:
           method: POST
           url: ${{ secrets.PORTAINER_WEBHOOK_STAGING }}
           preventFailureOnResponse: true
+  classifier:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r translator/requirements.txt
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build & Push Docker Staging Build
+        uses: docker/build-push-action@v5
+        if: github.ref == 'refs/heads/main'
+        with:
+          context: ./classifier
+          push: true
+          tags: hicsail/gdp-flask-classifier:staging
+          build-args: |
+            NOCO_DB_URL=${{ secrets.NOCO_DB_URL }}
+            NOCO_XC_TOKEN=${{ secrets.NOCO_XC_TOKEN }}
+            DEEPL_API_KEY=${{ secrets.DEEPL_API_KEY }}
+            GOOGLE_API_KEY=${{ secrets.GOOGLE_API_KEY }}
+
+      - name: Push to Staging
+        uses: fjogeleit/http-request-action@v1
+        if: github.ref == 'refs/heads/main'
+        with:
+          method: POST
+          url: ${{ secrets.PORTAINER_WEBHOOK_STAGING }}
+          preventFailureOnResponse: true

classifier/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.11-slim as build
+
+WORKDIR /app
+
+COPY . .
+
+ARG NOCO_DB_URL
+ARG NOCO_XC_TOKEN
+ARG LLM_URL
+
+ENV NOCO_DB_URL=${NOCO_DB_URL}
+ENV NOCO_XC_TOKEN=${NOCO_XC_TOKEN}
+ENV LLM_URL=${LLM_URL}
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+EXPOSE 5001
+
+CMD ["python", "./app.py"]

classifier/app.py

@@ -0,0 +1,67 @@
+from flask import Flask
+from apscheduler.schedulers.background import BackgroundScheduler
+from dotenv import load_dotenv
+
+import os
+import requests
+import json
+
+app = Flask(__name__)
+scheduler = BackgroundScheduler()
+
+@app.route('/health')
+def health_check():
+    return 'healthy'
+
+def classify():
+    scheduler.pause()
+
+    while True:
+        db_url = os.getenv("NOCO_DB_URL")
+        headers = {"xc-token": os.getenv("NOCO_XC_TOKEN")}
+        params = {
+            "where": "(status,eq,unverified)~and(isEnglish,eq,false)",
+            "limit": 5,
+        }
+
+        llm_url = os.getenv("LLM_URL")
+        form_data = {
+            "variables": "headline,body",
+            "string_prompt": "<<SYS>> \n You are an assistant tasked with classifying whether the given headline and body is related to China Development Bank or China Import Export Bank or Debt or Loan From China. \n <</SYS>> \n\n [INST] Generate a SHORT response if the given headline and article body is related to China. The output should be either Yes or No. \n\n Headline: \n\n {headline} \n\n Body: {body}\n\n [/INST]",
+        }
+
+        articles = requests.get(db_url, headers=headers, params=params)
+        if res.json().get("pageInfo").get("totalRows") == 0:
+            print("[MOF Classifier] No articles to classify")
+            scheduler.resume()
+            break
+
+        for article in articles.json().get("list"):
+            attempts = 3
+            while attempts > 0:
+                try:
+                    print("Article to classify: " + article["originalTitle"])
+                    form_data["llm_request"] = json.dumps({
+                        "headline": article["originalTitle"],
+                        "body": article["originalContent"]
+                    })
+
+                    res = requests.post(llm_url, data=form_data)
+
+                    if "yes" in res.json().get("Result").lower()[:5]:
+                        article["status"] = "relevant"
+                    else:
+                        article["status"] = "irrelevant"
+
+                    break
+                except:
+                    attempts -= 1
+                    article["status"] = "undetermined"
+
+            requests.patch(db_url, headers=headers, json=article)
+
+if __name__ == '__main__':
+    load_dotenv()
+    scheduler.add_job(classify, "cron", hour="*", minute=30)
+    scheduler.start()
+    app.run(port=5003)

classifier/requirements.txt

@@ -0,0 +1,4 @@
+APScheduler
+flask
+python-dotenv
+requests

deployment/docker-compose-staging.yml

@@ -15,3 +15,10 @@ services:
       - 5002:80
     environment:
       - FLASK_APP=app.py
+  classifier:
+    image: hicsail/gdp-flask-classifier:staging
+    restart: always
+    ports:
+      - 5003:80
+    environment:
+      - FLASK_APP=app.py