From 1b54e9284b6e9bfb69f315dd1cd134128f6d3d65 Mon Sep 17 00:00:00 2001 From: Lance Tan Date: Mon, 27 Nov 2023 15:45:27 -0800 Subject: [PATCH] Add AWS S3 Service --- backend/Pipfile | 2 + backend/Pipfile.lock | 120 ++++++++++++++++++++++++------- backend/core/settings.py | 10 +++ backend/sample.env | 7 +- backend/services/AWSS3Service.py | 85 ++++++++++++++++++++++ backend/services/__init__.py | 0 backend/vehicle/views.py | 11 +-- 7 files changed, 199 insertions(+), 36 deletions(-) create mode 100644 backend/services/AWSS3Service.py create mode 100644 backend/services/__init__.py diff --git a/backend/Pipfile b/backend/Pipfile index 40ed5c2..fd1d0a1 100644 --- a/backend/Pipfile +++ b/backend/Pipfile @@ -8,6 +8,8 @@ django = ">=4.2.6" djangorestframework = ">=3.14.0" django-environ = ">=0.11.2" psycopg2 = ">=2.9.9" +boto3 = ">=1.29.6" +django-storages = ">=1.14.2" [dev-packages] flake8 = ">=6.1.0" diff --git a/backend/Pipfile.lock b/backend/Pipfile.lock index 309822d..49d8d5d 100644 --- a/backend/Pipfile.lock +++ b/backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f957e36f17fd5e450c735c18856ee3c9f33219ee524bec76c3796c5b6c190f68" + "sha256": "9a5940b47d695d40c27e7cc02dc771f07c0e03e69ae8f8f03b1403f763e4dd9b" }, "pipfile-spec": 6, "requires": { @@ -24,14 +24,31 @@ "markers": "python_version >= '3.7'", "version": "==3.7.2" }, + "boto3": { + "hashes": [ + "sha256:799fe8399ea132aa5aa868caf78c47ef9ed675d5ef61be97cb7131081bb8a861", + "sha256:ebf6d86217c37986f965dbe35a3bbd0318127d23a65737ab6486667496decb54" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==1.33.0" + }, + "botocore": { + "hashes": [ + "sha256:ccf3d67fd046265ae73bc9862d1618c6e774a61a96beac832edb63d9a21fe1ba", + "sha256:e35526421fe8ee180b6aed3102929594aa51e4d60e3f29366a603707c37c0d52" + ], + "markers": "python_version >= '3.7'", + "version": "==1.33.0" + }, "django": { "hashes": [ - "sha256:08f41f468b63335aea0d904c5729e0250300f6a1907bf293a65499496cdbc68f", - "sha256:a64d2487cdb00ad7461434320ccc38e60af9c404773a2f95ab0093b4453a3215" + "sha256:8e0f1c2c2786b5c0e39fe1afce24c926040fad47c8ea8ad30aaf1188df29fc41", + "sha256:e1d37c51ad26186de355cbcec16613ebdabfa9689bbade9c538835205a8abbe9" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.2.6" + "version": "==4.2.7" }, "django-environ": { "hashes": [ @@ -42,6 +59,15 @@ "markers": "python_version >= '3.6' and python_version < '4'", "version": "==0.11.2" }, + "django-storages": { + "hashes": [ + "sha256:1db759346b52ada6c2efd9f23d8241ecf518813eb31db9e2589207174f58f6ad", + "sha256:51b36af28cc5813b98d5f3dfe7459af638d84428c8df4a03990c7d74d1bea4e5" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==1.14.2" + }, "djangorestframework": { "hashes": [ "sha256:579a333e6256b09489cbe0a067e66abe55c6595d8926be6b99423786334350c8", @@ -51,6 +77,14 @@ "markers": "python_version >= '3.6'", "version": "==3.14.0" }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, "psycopg2": { "hashes": [ "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981", @@ -58,10 +92,12 @@ "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3", "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa", "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a", + "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693", "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372", "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e", "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59", "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156", + "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024", "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913", "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c" ], @@ -69,6 +105,14 @@ "markers": "python_version >= '3.7'", "version": "==2.9.9" }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, "pytz": { "hashes": [ "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b", @@ -76,6 +120,22 @@ ], "version": "==2023.3.post1" }, + "s3transfer": { + "hashes": [ + "sha256:baa479dc2e63e5c2ed51611b4d46cdf0295e2070d8d0b86b22f335ee5b954986", + "sha256:e8d6bd52ffd99841e3a57b34370a54841f12d3aab072af862cdcc50955288002" + ], + "markers": "python_version >= '3.7'", + "version": "==0.8.0" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, "sqlparse": { "hashes": [ "sha256:5430a4fe2ac7d0f93e66f1efc6e1338a41884b7ddf2a350cedd20ccc4d9d28f3", @@ -91,33 +151,41 @@ ], "markers": "python_version < '3.11'", "version": "==4.8.0" + }, + "urllib3": { + "hashes": [ + "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", + "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" + ], + "markers": "python_version >= '3.10'", + "version": "==2.0.7" } }, "develop": { "black": { "hashes": [ - "sha256:0e232f24a337fed7a82c1185ae46c56c4a6167fb0fe37411b43e876892c76699", - "sha256:30b78ac9b54cf87bcb9910ee3d499d2bc893afd52495066c49d9ee6b21eee06e", - "sha256:31946ec6f9c54ed7ba431c38bc81d758970dd734b96b8e8c2b17a367d7908171", - "sha256:31b9f87b277a68d0e99d2905edae08807c007973eaa609da5f0c62def6b7c0bd", - "sha256:47c4510f70ec2e8f9135ba490811c071419c115e46f143e4dce2ac45afdcf4c9", - "sha256:481167c60cd3e6b1cb8ef2aac0f76165843a374346aeeaa9d86765fe0dd0318b", - "sha256:6901631b937acbee93c75537e74f69463adaf34379a04eef32425b88aca88a23", - "sha256:76baba9281e5e5b230c9b7f83a96daf67a95e919c2dfc240d9e6295eab7b9204", - "sha256:7fb5fc36bb65160df21498d5a3dd330af8b6401be3f25af60c6ebfe23753f747", - "sha256:960c21555be135c4b37b7018d63d6248bdae8514e5c55b71e994ad37407f45b8", - "sha256:a3c2ddb35f71976a4cfeca558848c2f2f89abc86b06e8dd89b5a65c1e6c0f22a", - "sha256:c870bee76ad5f7a5ea7bd01dc646028d05568d33b0b09b7ecfc8ec0da3f3f39c", - "sha256:d3d9129ce05b0829730323bdcb00f928a448a124af5acf90aa94d9aba6969604", - "sha256:db451a3363b1e765c172c3fd86213a4ce63fb8524c938ebd82919bf2a6e28c6a", - "sha256:e223b731a0e025f8ef427dd79d8cd69c167da807f5710add30cdf131f13dd62e", - "sha256:f20ff03f3fdd2fd4460b4f631663813e57dc277e37fb216463f3b907aa5a9bdd", - "sha256:f74892b4b836e5162aa0452393112a574dac85e13902c57dfbaaf388e4eda37c", - "sha256:f8dc7d50d94063cdfd13c82368afd8588bac4ce360e4224ac399e769d6704e98" + "sha256:250d7e60f323fcfc8ea6c800d5eba12f7967400eb6c2d21ae85ad31c204fb1f4", + "sha256:2a9acad1451632021ee0d146c8765782a0c3846e0e0ea46659d7c4f89d9b212b", + "sha256:412f56bab20ac85927f3a959230331de5614aecda1ede14b373083f62ec24e6f", + "sha256:421f3e44aa67138ab1b9bfbc22ee3780b22fa5b291e4db8ab7eee95200726b07", + "sha256:45aa1d4675964946e53ab81aeec7a37613c1cb71647b5394779e6efb79d6d187", + "sha256:4c44b7211a3a0570cc097e81135faa5f261264f4dfaa22bd5ee2875a4e773bd6", + "sha256:4c68855825ff432d197229846f971bc4d6666ce90492e5b02013bcaca4d9ab05", + "sha256:5133f5507007ba08d8b7b263c7aa0f931af5ba88a29beacc4b2dc23fcefe9c06", + "sha256:54caaa703227c6e0c87b76326d0862184729a69b73d3b7305b6288e1d830067e", + "sha256:58e5f4d08a205b11800332920e285bd25e1a75c54953e05502052738fe16b3b5", + "sha256:698c1e0d5c43354ec5d6f4d914d0d553a9ada56c85415700b81dc90125aac244", + "sha256:6c1cac07e64433f646a9a838cdc00c9768b3c362805afc3fce341af0e6a9ae9f", + "sha256:760415ccc20f9e8747084169110ef75d545f3b0932ee21368f63ac0fee86b221", + "sha256:7f622b6822f02bfaf2a5cd31fdb7cd86fcf33dab6ced5185c35f5db98260b055", + "sha256:cf57719e581cfd48c4efe28543fea3d139c6b6f1238b3f0102a9c73992cbb479", + "sha256:d136ef5b418c81660ad847efe0e55c58c8208b77a57a28a503a5f345ccf01394", + "sha256:dbea0bb8575c6b6303cc65017b46351dc5953eea5c0a59d7b7e3a2d2f433a911", + "sha256:fc7f6a44d52747e65a02558e1d807c82df1d66ffa80a601862040a43ec2e3142" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==23.10.0" + "version": "==23.11.0" }, "click": { "hashes": [ @@ -179,11 +247,11 @@ }, "platformdirs": { "hashes": [ - "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3", - "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e" + "sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b", + "sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731" ], "markers": "python_version >= '3.7'", - "version": "==3.11.0" + "version": "==4.0.0" }, "pycodestyle": { "hashes": [ diff --git a/backend/core/settings.py b/backend/core/settings.py index b27644a..f00be11 100644 --- a/backend/core/settings.py +++ b/backend/core/settings.py @@ -46,6 +46,7 @@ "django.contrib.messages", "django.contrib.staticfiles", "rest_framework", + "storages", "core", "auction", "user", @@ -53,6 +54,15 @@ "bid", ] +# AWS S3 Configuration +AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID") +AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY") +AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME") +AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME") +AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com" + +DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" + MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", diff --git a/backend/sample.env b/backend/sample.env index 77bf4c3..d9f0d2c 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -1 +1,6 @@ -SECRET_KEY= \ No newline at end of file +SECRET_KEY= +ENVIRONMENT=dev +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_STORAGE_BUCKET_NAME= +AWS_S3_REGION_NAME= \ No newline at end of file diff --git a/backend/services/AWSS3Service.py b/backend/services/AWSS3Service.py new file mode 100644 index 0000000..3e879e8 --- /dev/null +++ b/backend/services/AWSS3Service.py @@ -0,0 +1,85 @@ +import boto3 +from botocore.exceptions import ClientError, NoCredentialsError +from django.conf import settings + + +class AWSS3Service: + EXCEL_FILE_PATH = "inventory-excels/" + IMAGE_FILE_PATH = "vehicle-images/" + + def __init__(self): + self.s3_client = boto3.client( + "s3", + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + region_name=settings.AWS_S3_REGION_NAME, + ) + self.bucket_name = settings.AWS_STORAGE_BUCKET_NAME + + def __upload_file(self, file_path, file_name, file_content): + try: + self.s3_client.put_object( + Bucket=self.bucket_name, + Key=f"{file_path}{file_name}", + Body=file_content, + ) + return True + except NoCredentialsError: + print("Credentials not available") + return False + + def __get_file(self, file_path, file_name): + try: + response = self.s3_client.get_object( + Bucket=self.bucket_name, Key=f"{file_path}{file_name}" + ) + return response["Body"].read() + except ClientError as e: + print(f"Error getting file: {e}") + return None + + def __delete_file(self, file_path, file_name): + try: + self.s3_client.delete_object( + Bucket=self.bucket_name, Key=f"{file_path}{file_name}" + ) + return True + except ClientError as e: + print(f"Error deleting file: {e}") + return False + + def upload_image(self, vehicle_id, file_name, file_content): + vehicle_folder_path = f"{self.IMAGE_FILE_PATH}{vehicle_id}/" + return self.__upload_file(vehicle_folder_path, file_name, file_content) + + def upload_excel_file(self, file_name, file_content): + return self.__upload_file(self.EXCEL_FILE_PATH, file_name, file_content) + + def get_latest_excel_file(self): + try: + response = self.s3_client.list_objects_v2( + Bucket=self.bucket_name, Prefix=self.EXCEL_FILE_PATH + ) + + if "Contents" in response: + excel_files = filter( + lambda x: x["Key"] != self.EXCEL_FILE_PATH, response["Contents"] + ) + + latest_file_key = sorted( + excel_files, key=lambda x: x["LastModified"], reverse=True + )[0]["Key"] + + return self.__get_file("", latest_file_key) + else: + return None + except ClientError as e: + print(f"Error getting latest Excel file: {e}") + return None + + def delete_image(self, vehicle_id, file_name): + vehicle_folder_path = f"{self.IMAGE_FILE_PATH}{vehicle_id}/" + return self.__delete_file(vehicle_folder_path, file_name) + + def delete_excel_file(self, file_name): + return self.__delete_file(self.EXCEL_FILE_PATH, file_name) diff --git a/backend/services/__init__.py b/backend/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/vehicle/views.py b/backend/vehicle/views.py index e2f7b44..7e0c569 100644 --- a/backend/vehicle/views.py +++ b/backend/vehicle/views.py @@ -5,11 +5,8 @@ from rest_framework.response import Response from rest_framework.views import APIView -from .models import ( - Brand, Equipment, Supplier, Trailer, Type, UnitImage, Vehicle) -from .serializers import ( - BrandSerializer, EquipmentSerializer, SupplierSerializer, - TrailerSerializer, TypeSerializer, UnitImageSerializer, VehicleSerializer) +from .models import Vehicle +from .serializers import VehicleSerializer # Create your views here. @@ -70,9 +67,5 @@ def get(self, request, vehicle_id, *args, **kwargs): vehicle = get_object_or_404(Vehicle, id=vehicle_id) serializer = VehicleSerializer(vehicle) return Response(serializer.data, status=status.HTTP_200_OK) - # identifier = request.data.get("id") - # vehicle = Vehicle.objects.get(id=identifier) - # serialized_data = self.serializer_class(vehicle) - # return Response(serialized_data.data, status=status.HTTP_200_OK) except vehicle.DoesNotExist: return Response(status=status.HTTP_404_NOT_FOUND)