ZDL-96: Integrate Django RQ for asynchronous processing

Procfile

@@ -1 +1,3 @@
-web: gunicorn zadalaAPI.wsgi
+web: gunicorn zadalaAPI.wsgi
+
+worker: python manage.py rqworker high default low

docker-compose.yml

@@ -12,7 +12,7 @@ services:
       timeout: 10s
       retries: 120
     ports:
-      - 5432:5432
+      - "5432:5432"
     expose:
       - 5432
     environment:
@@ -22,18 +22,29 @@ services:
   # Database visualizer
   adminer:
     image: adminer
+    container_name: adminer
     restart: always
     ports:
-      - 8081:8080
+      - "8081:8080"
     depends_on:
       - db
 
+  redis:
+    image: redis:latest
+    container_name: redis
+    restart: always
+    ports:
+      - "6379:6379"
+    expose:
+      - 6379
+
   zadala-api:
+    container_name: zadala
     build: .
     ports:
-      - 8080:8000
+      - "8000:8000"
     expose:
-      - 8080
+      - 8000
     environment:
       - DB_NAME=zadala
       - DB_USER=postgres

orders/models.py

@@ -1,15 +1,7 @@
-from typing import Dict
-
-from django.conf import settings
-from django.core.mail import send_mail
 from django.db import models
 from django.db.models.signals import post_delete
 from django.dispatch import receiver
-from django.template.loader import render_to_string
 from django.utils import timezone
-from django.utils.html import strip_tags
-from pydantic import EmailStr
-from validate_email import validate_email
 
 from authentication.models import User
 from products.models import Product
@@ -51,23 +43,6 @@ def get_cart_items(self):
         total = sum([item.quantity for item in items])
         return total
 
-    @staticmethod
-    def send_email_notification(
-        customer_email: EmailStr, template: str, subject: str, context_data: Dict
-    ):
-        if validate_email(email_address=customer_email, check_smtp=False):
-            html_message = render_to_string(template, context_data)
-            plain_message = strip_tags(html_message)
-
-            send_mail(
-                subject,
-                plain_message,
-                settings.EMAIL_HOST_USER,
-                [customer_email, settings.EMAIL_HOST_USER],
-                fail_silently=False,
-                html_message=html_message,
-            )
-
 
 class OrderItem(models.Model):
     product = models.ForeignKey(Product, on_delete=models.SET_NULL, null=True)

orders/tasks.py

@@ -0,0 +1,24 @@
+from typing import Dict
+
+from django.conf import settings
+from django.core.mail import send_mail
+from django.template.loader import render_to_string
+from django.utils.html import strip_tags
+from validate_email import validate_email
+
+
+def send_email_notification(
+    customer_email: str, template: str, subject: str, context_data: Dict
+):
+    if validate_email(email_address=customer_email, check_smtp=False):
+        html_message = render_to_string(template, context_data)
+        plain_message = strip_tags(html_message)
+
+        send_mail(
+            subject,
+            plain_message,
+            settings.EMAIL_HOST_USER,
+            [customer_email, settings.EMAIL_HOST_USER],
+            fail_silently=False,
+            html_message=html_message,
+        )

orders/tests/test_orders_api.py

@@ -150,8 +150,7 @@ def test_update_cart(logged_in_client):
 
 
 @pytest.mark.django_db
-@patch("django.core.mail.send_mail", lambda **kwargs: kwargs)
-@patch("validate_email.validate_email", lambda z, x: x)
+@patch("django_rq.enqueue", lambda *args: args)
 def test_process_order(logged_in_client, logged_in_user):
     """
     Test process order

orders/tests/test_tasks.py

@@ -0,0 +1,17 @@
+from unittest.mock import patch
+
+from orders.tasks import send_email_notification
+
+
+@patch("django.core.mail.send_mail", lambda **kwargs: kwargs)
+@patch("validate_email.validate_email", lambda z, x: x)
+def test_send_email_notification_task():
+    """
+    Test sending of email task
+    """
+    send_email_notification(
+        "[email protected]",
+        "invoice_email_template.html",
+        "subject",
+        {"context": None},
+    )

orders/views.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from uuid import uuid4
 
+import django_rq
 from rest_condition import Or
 from rest_framework import mixins, status, viewsets
 from rest_framework.decorators import action
@@ -20,6 +21,7 @@
     ShippingAddressSerializer,
     UpdateCartSerializer,
 )
+from orders.tasks import send_email_notification
 from orders.validators import OrdersList
 from products.models import Product
 
@@ -129,9 +131,10 @@ def process_order(self, request):
             state=request_data.validated_data["state"],
             zipcode=request_data.validated_data["zipcode"],
         )
-        shipping.save()
 
-        Order.send_email_notification(
+        shipping.save()
+        django_rq.enqueue(
+            send_email_notification,
             customer.email,
             "invoice_email_template.html",
             f"Order Being Processed: {order.transaction_id}",

requirements.txt

@@ -11,8 +11,10 @@ colorama==0.4.4
 coreapi==2.3.3
 coreschema==0.0.4
 coverage==5.3.1
+Deprecated==1.2.13
 Django==3.0.8
 django-cors-headers==3.7.0
+django-rq==2.5.1
 djangorestframework==3.11.0
 djangorestframework-simplejwt==4.4.0
 dnspython==2.1.0
@@ -51,9 +53,11 @@ pytest-cov==2.10.1
 pytest-django==3.9.0
 python-dateutil==2.8.1
 pytz==2020.1
+redis==4.0.2
 regex==2020.7.14
 requests==2.26.0
 rest-condition==1.0.3
+rq==1.10.1
 ruamel.yaml==0.16.12
 ruamel.yaml.clib==0.2.2
 six==1.15.0
@@ -65,4 +69,5 @@ typing-extensions==3.7.4.2
 uritemplate==3.0.1
 urllib3==1.25.10
 whitenoise==5.2.0
+wrapt==1.13.3
 zipp==3.1.0

zadalaAPI/settings.py

@@ -19,6 +19,7 @@
     EMAIL_HOST_USER,
     ZADALA_SECRET_KEY,
     database,
+    redis_database,
 )
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -53,6 +54,7 @@
     "orders",
     "authentication",
     "rest_framework",
+    "django_rq",
 ]
 
 MIDDLEWARE = [
@@ -196,3 +198,17 @@
 EMAIL_HOST_USER = os.environ.get("EMAIL_HOST_USER", EMAIL_HOST_USER)
 EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_HOST_PASSWORD", EMAIL_HOST_PASSWORD)
 EMAIL_USE_TLS = True
+
+RQ_QUEUES = {
+    "default": {
+        "URL": os.getenv("REDISTOGO_URL", redis_database["REDIS_URL"]),
+        "DEFAULT_TIMEOUT": 360,
+    },
+    "high": {
+        "URL": os.getenv("REDISTOGO_URL", redis_database["REDIS_URL"]),
+        "DEFAULT_TIMEOUT": 500,
+    },
+    "low": {
+        "URL": os.getenv("REDISTOGO_URL", redis_database["REDIS_URL"]),
+    },
+}

zadalaAPI/urls.py

@@ -50,3 +50,4 @@
 ]
 
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+urlpatterns += [path("django-rq/", include("django_rq.urls"))]

zadala_config.py

@@ -7,8 +7,11 @@
     "PORT": "5432",
 }
 
-ZADALA_SECRET_KEY = "my-secret"
+redis_database = {
+    "REDIS_URL": "redis://localhost:6379/0",
+}
 
+ZADALA_SECRET_KEY = "my-secret"
 
 EMAIL_HOST_PROVIDER = "smtp.gmail.com"
 EMAIL_HOST_PORT = 587