upload media to s3

config/settings/common.py

@@ -105,6 +105,7 @@
     'oauth2_jwt_provider',
     'crispy_forms',  # needed to squash warnings around collectstatic with rest_framework
     'post_office',
+    'storages'
 )
 
 SEED_CORE_APPS = (
@@ -163,6 +164,22 @@
 )
 AWS_QUERYSTRING_AUTH = False
 
+USE_S3 = os.getenv('USE_S3') == 'TRUE'
+if USE_S3:
+    # aws settings
+    AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+    AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
+    AWS_STORAGE_BUCKET_NAME = os.getenv('AWS_STORAGE_BUCKET_NAME')
+    AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com"
+    AWS_S3_OBJECT_PARAMETERS = {'CacheControl': 'max-age=86400'}
+    AWS_S3_REGION_NAME = os.getenv('AWS_S3_REGION_NAME')
+    # s3 media settings
+    MEDIA_ROOT = "media"
+    DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
+
+else:
+    MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
+
 # django-longer-username-and-email
 REQUIRE_UNIQUE_EMAIL = False
 

docker-compose.yml

@@ -25,8 +25,11 @@ services:
     image: seedplatform/seed:latest
     build: .
     environment:
+      - USE_S3
       - AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY
+      - AWS_STORAGE_BUCKET_NAME
+      - AWS_S3_REGION_NAME
       - AWS_SES_REGION_NAME
       - AWS_SES_REGION_ENDPOINT
       - BSYNCR_SERVER_HOST
@@ -66,6 +69,11 @@ services:
     build: .
     command: /seed/docker/start_celery_docker.sh
     environment:
+      - USE_S3
+      - AWS_ACCESS_KEY_ID
+      - AWS_SECRET_ACCESS_KEY
+      - AWS_S3_REGION_NAME
+      - AWS_STORAGE_BUCKET_NAME
       - BSYNCR_SERVER_HOST
       - BSYNCR_SERVER_PORT
       - POSTGRES_DB=seed

requirements/aws.txt

@@ -3,3 +3,4 @@ boto3==1.17.36
 django-sslify==0.2.7
 django-ses==1.0.3
 uWSGI==2.0.17.1
+django-storages

seed/data_importer/views.py

@@ -13,7 +13,7 @@
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.core.exceptions import ObjectDoesNotExist
-from django.core.files.storage import FileSystemStorage
+from django.core.files.storage import default_storage
 from django.http import HttpResponse, JsonResponse
 from past.builtins import basestring
 from rest_framework import serializers, status, viewsets
@@ -123,15 +123,11 @@ def create(self, request):
         path = os.path.join(settings.MEDIA_ROOT, "uploads", filename)
 
         # Get a unique filename using the get_available_name method in FileSystemStorage
-        s = FileSystemStorage()
+        s = default_storage
         path = s.get_available_name(path)
 
-        # verify the directory exists
-        if not os.path.exists(os.path.dirname(path)):
-            os.makedirs(os.path.dirname(path))
-
         # save the file
-        with open(path, 'wb+') as temp_file:
+        with default_storage.open(filename, mode='wb+') as temp_file:
             for chunk in the_file.chunks():
                 temp_file.write(chunk)
 
@@ -225,14 +221,10 @@ def create_from_pm_import(self, request):
         # create a folder to keep pm_import files
         path = os.path.join(settings.MEDIA_ROOT, "uploads", "pm_imports", file_name)
 
-        # Get a unique filename using the get_available_name method in FileSystemStorage
-        s = FileSystemStorage()
+        # Get a unique filename using the get_available_name method in deafult_storage
+        s = default_storage
         path = s.get_available_name(path)
 
-        # verify the directory exists
-        if not os.path.exists(os.path.dirname(path)):
-            os.makedirs(os.path.dirname(path))
-
         # This list should cover the core keys coming from PM, ensuring that they map easily
         # We will also look for keys not in this list and just map them to themselves
         # pm_key_to_column_heading_map = {
@@ -344,7 +336,7 @@ def create_from_pm_import(self, request):
             rows.append(this_row)
 
         # Then write the actual data out as csv
-        with open(path, 'w', encoding='utf-8') as csv_file:
+        with default_storage.open(request.FILES['file'], 'w', encoding='utf-8') as csv_file:
             pm_csv_writer = csv.writer(csv_file)
             for row_num, row in enumerate(rows):
                 pm_csv_writer.writerow(row)

seed/views/v3/uploads.py

@@ -6,7 +6,7 @@
 import pint
 import xlrd
 from django.conf import settings
-from django.core.files.storage import FileSystemStorage
+from django.core.files.storage import default_storage
 from django.http import JsonResponse
 from drf_yasg.utils import no_body, swagger_auto_schema
 from past.builtins import basestring
@@ -28,7 +28,7 @@ def get_upload_path(filename):
     path = os.path.join(settings.MEDIA_ROOT, "uploads", filename)
 
     # Get a unique filename using the get_available_name method in FileSystemStorage
-    s = FileSystemStorage()
+    s = default_storage
     return s.get_available_name(path)
 
 
@@ -95,10 +95,6 @@ def create(self, request):
         filename = the_file.name
         path = get_upload_path(filename)
 
-        # verify the directory exists
-        if not os.path.exists(os.path.dirname(path)):
-            os.makedirs(os.path.dirname(path))
-
         extension = the_file.name.split(".")[-1]
         if extension == "xlsx" or extension == "xls":
             workbook = xlrd.open_workbook(file_contents=the_file.read())
@@ -119,7 +115,7 @@ def create(self, request):
                 })
 
         # save the file
-        with open(path, 'wb+') as temp_file:
+        with default_storage.open(path, mode='wb+') as temp_file:
             for chunk in the_file.chunks():
                 temp_file.write(chunk)
         org_id = self.get_organization(request)
@@ -228,14 +224,10 @@ def create_from_pm_import(self, request):
         # create a folder to keep pm_import files
         path = os.path.join(settings.MEDIA_ROOT, "uploads", "pm_imports", file_name)
 
-        # Get a unique filename using the get_available_name method in FileSystemStorage
-        s = FileSystemStorage()
+        # Get a unique filename using the get_available_name method in default_storage
+        s = default_storage
         path = s.get_available_name(path)
 
-        # verify the directory exists
-        if not os.path.exists(os.path.dirname(path)):
-            os.makedirs(os.path.dirname(path))
-
         # This list should cover the core keys coming from PM, ensuring that they map easily
         # We will also look for keys not in this list and just map them to themselves
         # pm_key_to_column_heading_map = {