From 239654deb1576cb4d61d4cd5b241adaf9a950019 Mon Sep 17 00:00:00 2001 From: Rob Swaters Date: Fri, 20 Sep 2024 17:43:49 +0000 Subject: [PATCH 01/13] Update caldp version --- changelog.md | 3 +-- terraform/deploy_vars.sh | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index 8ab2b1a..f92264c 100644 --- a/changelog.md +++ b/changelog.md @@ -1,2 +1 @@ -- default base docker image set to CALDP_20240813_CAL_final -- default crds update to hst_1169.pmap +- update caldp version diff --git a/terraform/deploy_vars.sh b/terraform/deploy_vars.sh index a858f45..2845656 100755 --- a/terraform/deploy_vars.sh +++ b/terraform/deploy_vars.sh @@ -1,6 +1,6 @@ #! /bin/bash -xu -export CALCLOUD_VER="v0.4.44" -export CALDP_VER="v0.2.26" +export CALCLOUD_VER="v0.4.45-rc1" +export CALDP_VER="v0.2.27-rc1" export CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20240813_CAL_final" export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} ` From 5c5ba75395cf875a93239009ab50da40456f77c8 Mon Sep 17 00:00:00 2001 From: Jenny Shih Date: Fri, 16 Aug 2024 17:58:16 -0400 Subject: [PATCH 02/13] fix permission issues --- lambda/JobPredict/Dockerfile | 1 + terraform/deploy_ami_rotation_codebuild_image.sh | 3 +++ terraform/deploy_checkout_repos.sh | 2 ++ 3 files changed, 6 insertions(+) diff --git a/lambda/JobPredict/Dockerfile b/lambda/JobPredict/Dockerfile index ce2da73..bc51e91 100644 --- a/lambda/JobPredict/Dockerfile +++ b/lambda/JobPredict/Dockerfile @@ -22,4 +22,5 @@ RUN yum update -y java-1.8.0-openjdk #RUN python3.7 -m pip install --upgrade pip && python3.7 -m pip install -r requirements.txt && python3.7 -m pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.6.2-cp37-cp37m-manylinux2010_x86_64.whl RUN python3.11 -m pip install --upgrade pip && python3.11 -m pip install -r requirements.txt && python3.11 -m pip install tensorflow-cpu==2.12.1 ADD models ./models/ +RUN chmod -R ugo+r ./models/ CMD ["predict_handler.lambda_handler"] diff --git a/terraform/deploy_ami_rotation_codebuild_image.sh b/terraform/deploy_ami_rotation_codebuild_image.sh index 5364b0e..7eaff82 100755 --- a/terraform/deploy_ami_rotation_codebuild_image.sh +++ b/terraform/deploy_ami_rotation_codebuild_image.sh @@ -29,6 +29,9 @@ docker push ${AMIROTATION_DOCKER_IMAGE_UNSCANNED} cd ${CALCLOUD_BUILD_DIR}/terraform pwd +print("Sleeping for 30 seconds before scanning") +sleep 30 + python3 ami-rotation-image-scan.py amirotation_image_scan_status=$? diff --git a/terraform/deploy_checkout_repos.sh b/terraform/deploy_checkout_repos.sh index d662a5e..03fc03e 100755 --- a/terraform/deploy_checkout_repos.sh +++ b/terraform/deploy_checkout_repos.sh @@ -50,4 +50,6 @@ then fi fi +chmod -R og+r ${TMP_INSTALL_DIR} + cd $cwd \ No newline at end of file From 3dcacb069eeef4e6624372a1a556cac5ed2ad2b4 Mon Sep 17 00:00:00 2001 From: Jenny Shih Date: Fri, 16 Aug 2024 18:01:13 -0400 Subject: [PATCH 03/13] fix typo --- terraform/deploy_ami_rotation_codebuild_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/deploy_ami_rotation_codebuild_image.sh b/terraform/deploy_ami_rotation_codebuild_image.sh index 7eaff82..b45deb6 100755 --- a/terraform/deploy_ami_rotation_codebuild_image.sh +++ b/terraform/deploy_ami_rotation_codebuild_image.sh @@ -29,7 +29,7 @@ docker push ${AMIROTATION_DOCKER_IMAGE_UNSCANNED} cd ${CALCLOUD_BUILD_DIR}/terraform pwd -print("Sleeping for 30 seconds before scanning") +echo "Sleeping for 30 seconds before scanning" sleep 30 python3 ami-rotation-image-scan.py From 41df536c18c360a447093c8b548aa33289a37900 Mon Sep 17 00:00:00 2001 From: Rob Swaters Date: Sat, 21 Sep 2024 16:09:35 +0000 Subject: [PATCH 04/13] Fix permissions --- changelog.md | 1 + terraform/deploy_vars.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index f92264c..badd2e1 100644 --- a/changelog.md +++ b/changelog.md @@ -1 +1,2 @@ - update caldp version +- set permissions to make files readable in AL2023 diff --git a/terraform/deploy_vars.sh b/terraform/deploy_vars.sh index 2845656..bf11388 100755 --- a/terraform/deploy_vars.sh +++ b/terraform/deploy_vars.sh @@ -1,5 +1,5 @@ #! /bin/bash -xu -export CALCLOUD_VER="v0.4.45-rc1" +export CALCLOUD_VER="v0.4.45-rc2" export CALDP_VER="v0.2.27-rc1" export CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20240813_CAL_final" export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} ` From f83e52ca8a659c6cefc8012f973348d539cc518c Mon Sep 17 00:00:00 2001 From: Rob Swaters Date: Tue, 24 Sep 2024 20:02:41 +0000 Subject: [PATCH 05/13] Fix permissions on directories --- lambda/JobPredict/Dockerfile | 1 + terraform/deploy_checkout_repos.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lambda/JobPredict/Dockerfile b/lambda/JobPredict/Dockerfile index bc51e91..b8f043a 100644 --- a/lambda/JobPredict/Dockerfile +++ b/lambda/JobPredict/Dockerfile @@ -23,4 +23,5 @@ RUN yum update -y java-1.8.0-openjdk RUN python3.11 -m pip install --upgrade pip && python3.11 -m pip install -r requirements.txt && python3.11 -m pip install tensorflow-cpu==2.12.1 ADD models ./models/ RUN chmod -R ugo+r ./models/ +RUN find ./models/ -type d -exec chmod og+x {} + CMD ["predict_handler.lambda_handler"] diff --git a/terraform/deploy_checkout_repos.sh b/terraform/deploy_checkout_repos.sh index 03fc03e..ae2ae7a 100755 --- a/terraform/deploy_checkout_repos.sh +++ b/terraform/deploy_checkout_repos.sh @@ -51,5 +51,6 @@ then fi chmod -R og+r ${TMP_INSTALL_DIR} +find ${TMP_INSTALL_DIR} -type d -exec chmod og+x {} + -cd $cwd \ No newline at end of file +cd $cwd From 9aa9e2cd0eef2d76994d9fc0f9f1d79a06b8e613 Mon Sep 17 00:00:00 2001 From: Rob Swaters Date: Tue, 24 Sep 2024 20:14:03 +0000 Subject: [PATCH 06/13] Fix permissions on directories --- changelog.md | 2 +- terraform/deploy_vars.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index badd2e1..7a222e5 100644 --- a/changelog.md +++ b/changelog.md @@ -1,2 +1,2 @@ - update caldp version -- set permissions to make files readable in AL2023 +- set permissions to make files readable and dirs executable in AL2023 diff --git a/terraform/deploy_vars.sh b/terraform/deploy_vars.sh index bf11388..6838f46 100755 --- a/terraform/deploy_vars.sh +++ b/terraform/deploy_vars.sh @@ -1,5 +1,5 @@ #! /bin/bash -xu -export CALCLOUD_VER="v0.4.45-rc2" +export CALCLOUD_VER="v0.4.45-rc3" export CALDP_VER="v0.2.27-rc1" export CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20240813_CAL_final" export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} ` From 07dec2452b4f441e9cba0f15647bc2406d061dc6 Mon Sep 17 00:00:00 2001 From: Rob Swaters Date: Wed, 25 Sep 2024 18:04:00 +0000 Subject: [PATCH 07/13] Update deploy_vars.sh --- terraform/deploy_vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/deploy_vars.sh b/terraform/deploy_vars.sh index 6838f46..33756df 100755 --- a/terraform/deploy_vars.sh +++ b/terraform/deploy_vars.sh @@ -1,6 +1,6 @@ #! /bin/bash -xu export CALCLOUD_VER="v0.4.45-rc3" -export CALDP_VER="v0.2.27-rc1" +export CALDP_VER="v0.2.27-rc2" export CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20240813_CAL_final" export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} ` From e6be9717ac8f2e1f1f0e7d1a1aa0035cbd9b0c22 Mon Sep 17 00:00:00 2001 From: Rob Swaters Date: Wed, 25 Sep 2024 18:04:37 +0000 Subject: [PATCH 08/13] Update deploy_vars.sh --- terraform/deploy_vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/deploy_vars.sh b/terraform/deploy_vars.sh index 33756df..796cb00 100755 --- a/terraform/deploy_vars.sh +++ b/terraform/deploy_vars.sh @@ -1,5 +1,5 @@ #! /bin/bash -xu -export CALCLOUD_VER="v0.4.45-rc3" +export CALCLOUD_VER="v0.4.45-rc4" export CALDP_VER="v0.2.27-rc2" export CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20240813_CAL_final" export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} ` From a7fdf0c7f9f506f73a6731683068f079d2c32429 Mon Sep 17 00:00:00 2001 From: jshihstsci <110426664+jshihstsci@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:58:06 -0400 Subject: [PATCH 09/13] replace old ci ami filter (#207) --- terraform/deploy.sh | 2 +- terraform/deploy_ami_rotate.sh | 2 +- terraform/deploy_ami_rotate_codebuild_script.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/deploy.sh b/terraform/deploy.sh index 8681784..5d50b6d 100755 --- a/terraform/deploy.sh +++ b/terraform/deploy.sh @@ -15,7 +15,7 @@ echo $aws_tfstate # get AMI id(s) cd $CALCLOUD_BUILD_DIR/ami_rotation ami_json=$(echo $(awsudo $ADMIN_ARN aws ec2 describe-images --region us-east-1 --executable-users self)) -ci_ami=`python3 parse_image_json.py "${ami_json}" STSCI-AWS-Linux-2` +ci_ami=`python3 parse_image_json.py "${ami_json}" STSCI-AMAZON-LINUX2023` ecs_ami=`python3 parse_image_json.py "${ami_json}" STSCI-ECS-AL2023` if [[ "$ci_ami" =~ ^ami-[a-z0-9]+$ ]]; then diff --git a/terraform/deploy_ami_rotate.sh b/terraform/deploy_ami_rotate.sh index 66cbf47..ec07fa7 100755 --- a/terraform/deploy_ami_rotate.sh +++ b/terraform/deploy_ami_rotate.sh @@ -76,7 +76,7 @@ echo $aws_tfstate # get AMI id cd $CALCLOUD_BUILD_DIR/ami_rotation ami_json=$(echo $(awsudo $ADMIN_ARN aws ec2 describe-images --region us-east-1 --executable-users self)) -ci_ami=`python3 parse_image_json.py "${ami_json}" STSCI-AWS-Linux-2` +ci_ami=`python3 parse_image_json.py "${ami_json}" STSCI-AMAZON-LINUX2023` ecs_ami=`python3 parse_image_json.py "${ami_json}" STSCI-ECS-AL2023` if [[ "$ci_ami" =~ ^ami-[a-z0-9]+$ ]]; then diff --git a/terraform/deploy_ami_rotate_codebuild_script.sh b/terraform/deploy_ami_rotate_codebuild_script.sh index 7685d63..56c5760 100755 --- a/terraform/deploy_ami_rotate_codebuild_script.sh +++ b/terraform/deploy_ami_rotate_codebuild_script.sh @@ -77,7 +77,7 @@ echo $aws_tfstate # get AMI id cd $CALCLOUD_BUILD_DIR/ami_rotation ami_json=$(echo $(aws ec2 describe-images --region us-east-1 --executable-users self)) -ci_ami=`python3 parse_image_json.py "${ami_json}" STSCI-AWS-Linux-2` +ci_ami=`python3 parse_image_json.py "${ami_json}" STSCI-AMAZON-LINUX2023` ecs_ami=`python3 parse_image_json.py "${ami_json}" STSCI-ECS-AL2023` if [[ "$ci_ami" =~ ^ami-[a-z0-9]+$ ]]; then From e3771d927e07179ee2daab18c63011e90b3f10b3 Mon Sep 17 00:00:00 2001 From: Rob Swaters Date: Tue, 8 Oct 2024 14:22:00 +0000 Subject: [PATCH 10/13] Update deploy_vars.sh --- changelog.md | 1 + terraform/deploy_vars.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 7a222e5..3ec9888 100644 --- a/changelog.md +++ b/changelog.md @@ -1,2 +1,3 @@ - update caldp version - set permissions to make files readable and dirs executable in AL2023 +- additional updates for STSCI-AMAZON-LINUX2023 diff --git a/terraform/deploy_vars.sh b/terraform/deploy_vars.sh index 796cb00..4789f64 100755 --- a/terraform/deploy_vars.sh +++ b/terraform/deploy_vars.sh @@ -1,5 +1,5 @@ #! /bin/bash -xu -export CALCLOUD_VER="v0.4.45-rc4" +export CALCLOUD_VER="v0.4.45-rc5" export CALDP_VER="v0.2.27-rc2" export CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20240813_CAL_final" export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} ` From e742f84518a4108f7c06b28f37265943deac9dd3 Mon Sep 17 00:00:00 2001 From: Rob Swaters Date: Mon, 14 Oct 2024 17:57:38 +0000 Subject: [PATCH 11/13] Update caldp version --- terraform/deploy_vars.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/deploy_vars.sh b/terraform/deploy_vars.sh index 4789f64..12bbe5c 100755 --- a/terraform/deploy_vars.sh +++ b/terraform/deploy_vars.sh @@ -1,7 +1,7 @@ #! /bin/bash -xu -export CALCLOUD_VER="v0.4.45-rc5" +export CALCLOUD_VER="v0.4.45-rc6" export CALDP_VER="v0.2.27-rc2" -export CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20240813_CAL_final" +export CAL_BASE_IMAGE="stsci/hst-pipeline:2024.10.1.1-mostlycos-py311" export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} ` export COMMON_IMAGE_TAG="CALCLOUD_${CALCLOUD_VER}-CALDP_${CALDP_VER}-BASE_${BASE_IMAGE_TAG}" From a4694d2f42111bf6d466d2f44a59f97992623121 Mon Sep 17 00:00:00 2001 From: Rob Swaters Date: Mon, 14 Oct 2024 18:41:02 +0000 Subject: [PATCH 12/13] Update caldp release --- terraform/deploy_vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/deploy_vars.sh b/terraform/deploy_vars.sh index 12bbe5c..1223640 100755 --- a/terraform/deploy_vars.sh +++ b/terraform/deploy_vars.sh @@ -1,6 +1,6 @@ #! /bin/bash -xu export CALCLOUD_VER="v0.4.45-rc6" -export CALDP_VER="v0.2.27-rc2" +export CALDP_VER="v0.2.27-rc3" export CAL_BASE_IMAGE="stsci/hst-pipeline:2024.10.1.1-mostlycos-py311" export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} ` From 62ecc6381f509f2a2b9208fbcb1d70a705a09c88 Mon Sep 17 00:00:00 2001 From: Rob Swaters Date: Tue, 29 Oct 2024 18:34:43 +0000 Subject: [PATCH 13/13] Update deploy_vars.sh --- terraform/deploy_vars.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/deploy_vars.sh b/terraform/deploy_vars.sh index 1223640..a12f0e7 100755 --- a/terraform/deploy_vars.sh +++ b/terraform/deploy_vars.sh @@ -1,6 +1,6 @@ #! /bin/bash -xu -export CALCLOUD_VER="v0.4.45-rc6" -export CALDP_VER="v0.2.27-rc3" +export CALCLOUD_VER="v0.4.45" +export CALDP_VER="v0.2.27" export CAL_BASE_IMAGE="stsci/hst-pipeline:2024.10.1.1-mostlycos-py311" export BASE_IMAGE_TAG=`cut -d ":" -f2- <<< ${CAL_BASE_IMAGE} `