From ed636ed34b41b5ee9e5529fe416c2b4ae2591764 Mon Sep 17 00:00:00 2001 From: loydbanks Date: Wed, 27 Jul 2022 10:28:13 +0000 Subject: [PATCH 1/2] upload media to s3 --- config/settings/common.py | 24 +++++++++++++++++++++++- docker-compose.yml | 8 ++++++++ requirements/aws.txt | 1 + seed/data_importer/views.py | 20 ++++++-------------- seed/views/v3/uploads.py | 18 +++++------------- 5 files changed, 43 insertions(+), 28 deletions(-) diff --git a/config/settings/common.py b/config/settings/common.py index 6c1980c2db..d9ce4b0b16 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -16,7 +16,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -PROTOCOL = os.environ.get('PROTOCOL', 'https') +# PROTOCOL = os.environ.get('PROTOCOL', 'https') SESSION_COOKIE_DOMAIN = None SESSION_EXPIRE_AT_BROWSER_CLOSE = True @@ -105,6 +105,7 @@ 'oauth2_jwt_provider', 'crispy_forms', # needed to squash warnings around collectstatic with rest_framework 'post_office', + 'storages' ) SEED_CORE_APPS = ( @@ -163,6 +164,27 @@ ) AWS_QUERYSTRING_AUTH = False +USE_S3 = os.getenv('USE_S3') == 'TRUE' +if USE_S3: + print("credentials", os.getenv('AWS_ACCESS_KEY_ID'), + os.getenv('AWS_SECRET_ACCESS_KEY'), + os.getenv('AWS_STORAGE_BUCKET_NAME'), + os.getenv('AWS_S3_REGION_NAME')) + # aws settings + AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') + AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') + AWS_STORAGE_BUCKET_NAME = os.getenv('AWS_STORAGE_BUCKET_NAME') + AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" + print("custom domain", f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com") + AWS_S3_OBJECT_PARAMETERS = {'CacheControl': 'max-age=86400'} + AWS_S3_REGION_NAME = os.getenv('AWS_S3_REGION_NAME') + # s3 media settings + MEDIA_ROOT = "media" + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + +else: + MEDIA_ROOT = os.path.join(BASE_DIR, 'media') + # django-longer-username-and-email REQUIRE_UNIQUE_EMAIL = False diff --git a/docker-compose.yml b/docker-compose.yml index f3bb30efc6..c366ae7ce1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,8 +25,11 @@ services: image: seedplatform/seed:latest build: . environment: + - USE_S3 - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + - AWS_STORAGE_BUCKET_NAME + - AWS_S3_REGION_NAME - AWS_SES_REGION_NAME - AWS_SES_REGION_ENDPOINT - BSYNCR_SERVER_HOST @@ -66,6 +69,11 @@ services: build: . command: /seed/docker/start_celery_docker.sh environment: + - USE_S3 + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_S3_REGION_NAME + - AWS_STORAGE_BUCKET_NAME - BSYNCR_SERVER_HOST - BSYNCR_SERVER_PORT - POSTGRES_DB=seed diff --git a/requirements/aws.txt b/requirements/aws.txt index 0b4962e4a1..5dd4d5d77c 100644 --- a/requirements/aws.txt +++ b/requirements/aws.txt @@ -3,3 +3,4 @@ boto3==1.17.36 django-sslify==0.2.7 django-ses==1.0.3 uWSGI==2.0.17.1 +django-storages \ No newline at end of file diff --git a/seed/data_importer/views.py b/seed/data_importer/views.py index 18799b8e77..342fdae2cb 100644 --- a/seed/data_importer/views.py +++ b/seed/data_importer/views.py @@ -13,7 +13,7 @@ from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.exceptions import ObjectDoesNotExist -from django.core.files.storage import FileSystemStorage +from django.core.files.storage import default_storage from django.http import HttpResponse, JsonResponse from past.builtins import basestring from rest_framework import serializers, status, viewsets @@ -123,15 +123,11 @@ def create(self, request): path = os.path.join(settings.MEDIA_ROOT, "uploads", filename) # Get a unique filename using the get_available_name method in FileSystemStorage - s = FileSystemStorage() + s = default_storage path = s.get_available_name(path) - # verify the directory exists - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - # save the file - with open(path, 'wb+') as temp_file: + with default_storage.open(filename, mode='wb+') as temp_file: for chunk in the_file.chunks(): temp_file.write(chunk) @@ -225,14 +221,10 @@ def create_from_pm_import(self, request): # create a folder to keep pm_import files path = os.path.join(settings.MEDIA_ROOT, "uploads", "pm_imports", file_name) - # Get a unique filename using the get_available_name method in FileSystemStorage - s = FileSystemStorage() + # Get a unique filename using the get_available_name method in deafult_storage + s = default_storage path = s.get_available_name(path) - # verify the directory exists - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - # This list should cover the core keys coming from PM, ensuring that they map easily # We will also look for keys not in this list and just map them to themselves # pm_key_to_column_heading_map = { @@ -344,7 +336,7 @@ def create_from_pm_import(self, request): rows.append(this_row) # Then write the actual data out as csv - with open(path, 'w', encoding='utf-8') as csv_file: + with default_storage.open(request.FILES['file'], 'w', encoding='utf-8') as csv_file: pm_csv_writer = csv.writer(csv_file) for row_num, row in enumerate(rows): pm_csv_writer.writerow(row) diff --git a/seed/views/v3/uploads.py b/seed/views/v3/uploads.py index 52c9ed5ba6..a23768ed54 100644 --- a/seed/views/v3/uploads.py +++ b/seed/views/v3/uploads.py @@ -6,7 +6,7 @@ import pint import xlrd from django.conf import settings -from django.core.files.storage import FileSystemStorage +from django.core.files.storage import default_storage from django.http import JsonResponse from drf_yasg.utils import no_body, swagger_auto_schema from past.builtins import basestring @@ -28,7 +28,7 @@ def get_upload_path(filename): path = os.path.join(settings.MEDIA_ROOT, "uploads", filename) # Get a unique filename using the get_available_name method in FileSystemStorage - s = FileSystemStorage() + s = default_storage return s.get_available_name(path) @@ -95,10 +95,6 @@ def create(self, request): filename = the_file.name path = get_upload_path(filename) - # verify the directory exists - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - extension = the_file.name.split(".")[-1] if extension == "xlsx" or extension == "xls": workbook = xlrd.open_workbook(file_contents=the_file.read()) @@ -119,7 +115,7 @@ def create(self, request): }) # save the file - with open(path, 'wb+') as temp_file: + with default_storage.open(path, mode='wb+') as temp_file: for chunk in the_file.chunks(): temp_file.write(chunk) org_id = self.get_organization(request) @@ -228,14 +224,10 @@ def create_from_pm_import(self, request): # create a folder to keep pm_import files path = os.path.join(settings.MEDIA_ROOT, "uploads", "pm_imports", file_name) - # Get a unique filename using the get_available_name method in FileSystemStorage - s = FileSystemStorage() + # Get a unique filename using the get_available_name method in default_storage + s = default_storage path = s.get_available_name(path) - # verify the directory exists - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - # This list should cover the core keys coming from PM, ensuring that they map easily # We will also look for keys not in this list and just map them to themselves # pm_key_to_column_heading_map = { From 219cd7660067a6234ff704596c6271c9c0633dd6 Mon Sep 17 00:00:00 2001 From: loydbanks Date: Wed, 27 Jul 2022 10:28:13 +0000 Subject: [PATCH 2/2] upload media to s3 --- config/settings/common.py | 17 +++++++++++++++++ docker-compose.yml | 8 ++++++++ requirements/aws.txt | 1 + seed/data_importer/views.py | 20 ++++++-------------- seed/views/v3/uploads.py | 18 +++++------------- 5 files changed, 37 insertions(+), 27 deletions(-) diff --git a/config/settings/common.py b/config/settings/common.py index 6c1980c2db..1a7c3194f7 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -105,6 +105,7 @@ 'oauth2_jwt_provider', 'crispy_forms', # needed to squash warnings around collectstatic with rest_framework 'post_office', + 'storages' ) SEED_CORE_APPS = ( @@ -163,6 +164,22 @@ ) AWS_QUERYSTRING_AUTH = False +USE_S3 = os.getenv('USE_S3') == 'TRUE' +if USE_S3: + # aws settings + AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') + AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') + AWS_STORAGE_BUCKET_NAME = os.getenv('AWS_STORAGE_BUCKET_NAME') + AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" + AWS_S3_OBJECT_PARAMETERS = {'CacheControl': 'max-age=86400'} + AWS_S3_REGION_NAME = os.getenv('AWS_S3_REGION_NAME') + # s3 media settings + MEDIA_ROOT = "media" + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + +else: + MEDIA_ROOT = os.path.join(BASE_DIR, 'media') + # django-longer-username-and-email REQUIRE_UNIQUE_EMAIL = False diff --git a/docker-compose.yml b/docker-compose.yml index f3bb30efc6..c366ae7ce1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,8 +25,11 @@ services: image: seedplatform/seed:latest build: . environment: + - USE_S3 - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + - AWS_STORAGE_BUCKET_NAME + - AWS_S3_REGION_NAME - AWS_SES_REGION_NAME - AWS_SES_REGION_ENDPOINT - BSYNCR_SERVER_HOST @@ -66,6 +69,11 @@ services: build: . command: /seed/docker/start_celery_docker.sh environment: + - USE_S3 + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_S3_REGION_NAME + - AWS_STORAGE_BUCKET_NAME - BSYNCR_SERVER_HOST - BSYNCR_SERVER_PORT - POSTGRES_DB=seed diff --git a/requirements/aws.txt b/requirements/aws.txt index 0b4962e4a1..5dd4d5d77c 100644 --- a/requirements/aws.txt +++ b/requirements/aws.txt @@ -3,3 +3,4 @@ boto3==1.17.36 django-sslify==0.2.7 django-ses==1.0.3 uWSGI==2.0.17.1 +django-storages \ No newline at end of file diff --git a/seed/data_importer/views.py b/seed/data_importer/views.py index 18799b8e77..342fdae2cb 100644 --- a/seed/data_importer/views.py +++ b/seed/data_importer/views.py @@ -13,7 +13,7 @@ from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.exceptions import ObjectDoesNotExist -from django.core.files.storage import FileSystemStorage +from django.core.files.storage import default_storage from django.http import HttpResponse, JsonResponse from past.builtins import basestring from rest_framework import serializers, status, viewsets @@ -123,15 +123,11 @@ def create(self, request): path = os.path.join(settings.MEDIA_ROOT, "uploads", filename) # Get a unique filename using the get_available_name method in FileSystemStorage - s = FileSystemStorage() + s = default_storage path = s.get_available_name(path) - # verify the directory exists - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - # save the file - with open(path, 'wb+') as temp_file: + with default_storage.open(filename, mode='wb+') as temp_file: for chunk in the_file.chunks(): temp_file.write(chunk) @@ -225,14 +221,10 @@ def create_from_pm_import(self, request): # create a folder to keep pm_import files path = os.path.join(settings.MEDIA_ROOT, "uploads", "pm_imports", file_name) - # Get a unique filename using the get_available_name method in FileSystemStorage - s = FileSystemStorage() + # Get a unique filename using the get_available_name method in deafult_storage + s = default_storage path = s.get_available_name(path) - # verify the directory exists - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - # This list should cover the core keys coming from PM, ensuring that they map easily # We will also look for keys not in this list and just map them to themselves # pm_key_to_column_heading_map = { @@ -344,7 +336,7 @@ def create_from_pm_import(self, request): rows.append(this_row) # Then write the actual data out as csv - with open(path, 'w', encoding='utf-8') as csv_file: + with default_storage.open(request.FILES['file'], 'w', encoding='utf-8') as csv_file: pm_csv_writer = csv.writer(csv_file) for row_num, row in enumerate(rows): pm_csv_writer.writerow(row) diff --git a/seed/views/v3/uploads.py b/seed/views/v3/uploads.py index 52c9ed5ba6..a23768ed54 100644 --- a/seed/views/v3/uploads.py +++ b/seed/views/v3/uploads.py @@ -6,7 +6,7 @@ import pint import xlrd from django.conf import settings -from django.core.files.storage import FileSystemStorage +from django.core.files.storage import default_storage from django.http import JsonResponse from drf_yasg.utils import no_body, swagger_auto_schema from past.builtins import basestring @@ -28,7 +28,7 @@ def get_upload_path(filename): path = os.path.join(settings.MEDIA_ROOT, "uploads", filename) # Get a unique filename using the get_available_name method in FileSystemStorage - s = FileSystemStorage() + s = default_storage return s.get_available_name(path) @@ -95,10 +95,6 @@ def create(self, request): filename = the_file.name path = get_upload_path(filename) - # verify the directory exists - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - extension = the_file.name.split(".")[-1] if extension == "xlsx" or extension == "xls": workbook = xlrd.open_workbook(file_contents=the_file.read()) @@ -119,7 +115,7 @@ def create(self, request): }) # save the file - with open(path, 'wb+') as temp_file: + with default_storage.open(path, mode='wb+') as temp_file: for chunk in the_file.chunks(): temp_file.write(chunk) org_id = self.get_organization(request) @@ -228,14 +224,10 @@ def create_from_pm_import(self, request): # create a folder to keep pm_import files path = os.path.join(settings.MEDIA_ROOT, "uploads", "pm_imports", file_name) - # Get a unique filename using the get_available_name method in FileSystemStorage - s = FileSystemStorage() + # Get a unique filename using the get_available_name method in default_storage + s = default_storage path = s.get_available_name(path) - # verify the directory exists - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - # This list should cover the core keys coming from PM, ensuring that they map easily # We will also look for keys not in this list and just map them to themselves # pm_key_to_column_heading_map = {