From 1b9e32bddadd08dfb1cc523755ab582ac988a461 Mon Sep 17 00:00:00 2001 From: Rajasimman S Date: Wed, 27 Mar 2024 12:22:21 +0530 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=8C=90=20=20cloud=5Fstorage=5Furl=20a?= =?UTF-8?q?s=20aws=20s3=20endpoint=20url=20for=20s3=20compatible=20cloud?= =?UTF-8?q?=20storages=20like=20minio=20and=20ceph?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ansible/roles/aws-cloud-storage/tasks/delete-folder.yml | 3 ++- ansible/roles/aws-cloud-storage/tasks/delete.yml | 3 ++- ansible/roles/aws-cloud-storage/tasks/download.yml | 3 ++- ansible/roles/aws-cloud-storage/tasks/upload-folder.yml | 3 ++- ansible/roles/aws-cloud-storage/tasks/upload.yml | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ansible/roles/aws-cloud-storage/tasks/delete-folder.yml b/ansible/roles/aws-cloud-storage/tasks/delete-folder.yml index c912b14edb..9c1226836b 100644 --- a/ansible/roles/aws-cloud-storage/tasks/delete-folder.yml +++ b/ansible/roles/aws-cloud-storage/tasks/delete-folder.yml @@ -1,9 +1,10 @@ --- - name: delete files and folders recursively environment: + AWS_ENDPOINT_URL: "{{ cloud_storage_url }}" AWS_DEFAULT_REGION: "{{ aws_default_region }}" AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}" AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}" - shell: "aws s3 rm s3://{{ s3_bucket_name }}/{{ s3_path }} --recursive" + shell: "aws s3 rm s3://{{ s3_bucket_name }}/{{ s3_path }} --recursive --endpoint-url {{ cloud_storage_url }}" async: 3600 poll: 10 diff --git a/ansible/roles/aws-cloud-storage/tasks/delete.yml b/ansible/roles/aws-cloud-storage/tasks/delete.yml index 414ea52e6b..81cbf24c5d 100644 --- a/ansible/roles/aws-cloud-storage/tasks/delete.yml +++ b/ansible/roles/aws-cloud-storage/tasks/delete.yml @@ -1,9 +1,10 @@ --- - name: delete files from s3 environment: + AWS_ENDPOINT_URL: "{{ cloud_storage_url }}" AWS_DEFAULT_REGION: "{{ aws_default_region }}" AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}" AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}" - shell: "aws s3 rm s3://{{ s3_bucket_name }}/{{ s3_path }}" + shell: "aws s3 rm s3://{{ s3_bucket_name }}/{{ s3_path }} --endpoint-url {{ cloud_storage_url }}" async: 3600 poll: 10 diff --git a/ansible/roles/aws-cloud-storage/tasks/download.yml b/ansible/roles/aws-cloud-storage/tasks/download.yml index 138024af78..3eeccd9b18 100644 --- a/ansible/roles/aws-cloud-storage/tasks/download.yml +++ b/ansible/roles/aws-cloud-storage/tasks/download.yml @@ -1,9 +1,10 @@ --- - name: download files to s3 environment: + AWS_ENDPOINT_URL: "{{ cloud_storage_url }}" AWS_DEFAULT_REGION: "{{ aws_default_region }}" AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}" AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}" - shell: "aws s3 cp s3://{{ s3_bucket_name }}/{{ s3_path }} {{ local_file_or_folder_path }}" + shell: "aws s3 cp s3://{{ s3_bucket_name }}/{{ s3_path }} {{ local_file_or_folder_path }} --endpoint-url {{ cloud_storage_url }}" async: 3600 poll: 10 diff --git a/ansible/roles/aws-cloud-storage/tasks/upload-folder.yml b/ansible/roles/aws-cloud-storage/tasks/upload-folder.yml index 3e03b068b7..de232076c2 100644 --- a/ansible/roles/aws-cloud-storage/tasks/upload-folder.yml +++ b/ansible/roles/aws-cloud-storage/tasks/upload-folder.yml @@ -1,9 +1,10 @@ --- - name: upload folder to s3 environment: + AWS_ENDPOINT_URL: "{{ cloud_storage_url }}" AWS_DEFAULT_REGION: "{{ aws_default_region }}" AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}" AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}" - shell: "aws s3 cp {{ local_file_or_folder_path }} s3://{{ s3_bucket_name }}/{{ s3_path }} --recursive" + shell: "aws s3 cp {{ local_file_or_folder_path }} s3://{{ s3_bucket_name }}/{{ s3_path }} --recursive --endpoint-url {{ cloud_storage_url }}" async: 3600 poll: 10 diff --git a/ansible/roles/aws-cloud-storage/tasks/upload.yml b/ansible/roles/aws-cloud-storage/tasks/upload.yml index af8de990e2..714e649f1c 100644 --- a/ansible/roles/aws-cloud-storage/tasks/upload.yml +++ b/ansible/roles/aws-cloud-storage/tasks/upload.yml @@ -1,9 +1,10 @@ --- - name: upload files to s3 environment: + AWS_ENDPOINT_URL: "{{ cloud_storage_url }}" AWS_DEFAULT_REGION: "{{ aws_default_region }}" AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}" AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}" - shell: "aws s3 cp {{ local_file_or_folder_path }} s3://{{ s3_bucket_name }}/{{ s3_path }}" + shell: "aws s3 cp {{ local_file_or_folder_path }} s3://{{ s3_bucket_name }}/{{ s3_path }} --endpoint-url {{ cloud_storage_url }}" async: 3600 poll: 10 From 88d68f0a5b17544d7483afc112607037289e9a66 Mon Sep 17 00:00:00 2001 From: Rajasimman S Date: Wed, 27 Mar 2024 14:04:53 +0530 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=90=9B=20cloud=5Fstorage=5Fcassandrab?= =?UTF-8?q?ackup=5Ffoldername=20with=20date?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ansible/roles/cassandra-backup/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/cassandra-backup/defaults/main.yml b/ansible/roles/cassandra-backup/defaults/main.yml index 4481570cc6..694037ec78 100644 --- a/ansible/roles/cassandra-backup/defaults/main.yml +++ b/ansible/roles/cassandra-backup/defaults/main.yml @@ -1,4 +1,4 @@ cassandra_root_dir: '/etc/cassandra' data_dir: '/var/lib/cassandra/data' cloud_storage_cassandrabackup_bucketname: "{{cloud_storage_management_bucketname}}" -cloud_storage_cassandrabackup_foldername: 'cassandra-backup' +cloud_storage_cassandrabackup_foldername: "cassandra-backup/{{ lookup('pipe', 'date +%Y%m%d') }}" From 7d79fa5a25cdc6700f5c76ec89f4e4df8acd583e Mon Sep 17 00:00:00 2001 From: Rajasimman S Date: Wed, 27 Mar 2024 14:13:57 +0530 Subject: [PATCH 3/3] ansible_hostname - cloud_storage_cassandrabackup_foldername --- ansible/roles/cassandra-backup/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/cassandra-backup/defaults/main.yml b/ansible/roles/cassandra-backup/defaults/main.yml index 694037ec78..80e33db277 100644 --- a/ansible/roles/cassandra-backup/defaults/main.yml +++ b/ansible/roles/cassandra-backup/defaults/main.yml @@ -1,4 +1,4 @@ cassandra_root_dir: '/etc/cassandra' data_dir: '/var/lib/cassandra/data' cloud_storage_cassandrabackup_bucketname: "{{cloud_storage_management_bucketname}}" -cloud_storage_cassandrabackup_foldername: "cassandra-backup/{{ lookup('pipe', 'date +%Y%m%d') }}" +cloud_storage_cassandrabackup_foldername: "cassandra-backup/{{ lookup('pipe', 'date +%Y%m%d') }}/{{ ansible_hostname }}"