Recursively pre-dl files

batch/scripts/_warp_and_upload.sh

@@ -4,33 +4,24 @@ set -e
 
 # arguments:
 # $0 - The name of this script
-# $1 - remote_src_file
-# $2 - local_src_file
-# $3 - local_warped_file
-# $4 - target_crs
-# $5 - remote target file
+# $1 - local_src_file
+# $2 - local_warped_file
+# $3 - target_crs
+# $4 - remote target file
 
-if aws s3 ls "$5"; then
-  echo "Remote target file $5 already exists, skipping..."
+if aws s3 ls "$4"; then
+  echo "Remote target file $4 already exists, skipping..."
   exit 0
 fi
 
-echo "Now downloading $1 to $2"
-if [[ $1 == gs://* ]]; then
-  gsutil cp "$1" "$2"
-elif [[ $1 == s3://* ]]; then
-  aws s3 cp --no-progress "$1" "$2"
-fi
-echo "Done downloading $1 to $2"
-
-echo "Now warping $2 to $3"
-gdalwarp "$2" "$3" -t_srs "$4" -co COMPRESS=DEFLATE -co TILED=yes
-echo "Done warping $2 to $3"
+echo "Now warping $1 to $2"
+gdalwarp "$1" "$2" -t_srs "$3" -co COMPRESS=DEFLATE -co TILED=yes
+echo "Done warping $1 to $2"
 
-echo "Now uploading $3 to $5"
-aws s3 cp --no-progress "$3" "$5"
-echo "Done uploading $3 to $5"
+echo "Now uploading $2 to $4"
+aws s3 cp --no-progress "$2" "$4"
+echo "Done uploading $2 to $4"
 
-echo "Finally, deleting local files $2 and $3"
-rm "$2" "$3"
-echo "Done deleting local files $2 and $3"
+echo "Finally, deleting local files $1 and $2"
+rm "$1" "$2"
+echo "Done deleting local files $1 and $2"

batch/scripts/unify_projection.sh

@@ -12,27 +12,38 @@ ME=$(basename "$0")
 
 echo "Reproject to a common CRS"
 
-# files=""
-# for i in {2..60}; do files="$files $i.tif"; done
-# Skip 01.tif, 59.tif, and 60.tif for now (problems around the date line)
-files="02.tif 03.tif 04.tif 05.tif 06.tif 07.tif 08.tif 09.tif 10.tif 11.tif 12.tif 13.tif 14.tif 15.tif 16.tif 17.tif 18.tif 19.tif 20.tif 21.tif 22.tif 23.tif 24.tif 25.tif 26.tif 27.tif 28.tif 29.tif 30.tif 31.tif 32.tif 33.tif 34.tif 35.tif 36.tif 37.tif 38.tif 39.tif 40.tif 41.tif 42.tif 43.tif 44.tif 45.tif 46.tif 47.tif 48.tif 49.tif 50.tif 51.tif 52.tif 53.tif 54.tif 55.tif 56.tif 57.tif 58.tif"
-
 src_count=0
-
 CMD_ARGS=()
 
 for s in ${SRC[@]}; do
-  mkdir -p "SRC_${src_count}"
-  mkdir -p "REPROJECTED_${src_count}"
+  source_dir="SRC_${src_count}"
+  mkdir -p "$source_dir"
+
+  echo "Now recursively downloading $s to $source_dir"
+  if [[ $s == gs://* ]]; then
+    gsutil cp -m -r "$s" "$source_dir"
+  elif [[ $s == s3://* ]]; then
+    aws s3 cp --no-progress "$s" "$source_dir"
+  fi
+  echo "Done downloading $s to $source_dir"
+
+  reprojected_dir="REPROJECTED_${src_count}"
+  mkdir -p "$reprojected_dir"
+
+  cd $source_dir
+  for d in $((tree -dfi)); do
+    mkdir -p "../${reprojected_dir}/${d}"
+  done
 
-  for f in ${files}; do
-    remote_src_file=${s}/${f}
-    local_src_file=SRC_${src_count}/${f}
-    local_warped_file=REPROJECTED_${src_count}/${f}
-    remote_target_file=${TARGET}/SRC_${src_count}/${f}
+  for f in $((find . -iname "*.tif")); do
+    local_src_file="${source_dir}/${f}"
+    local_warped_file="${reprojected_dir}/${f}"
+    remote_target_file="${TARGET}/SRC_${src_count}/${f}"
 
-    CMD_ARGS+=("${remote_src_file}" "${local_src_file}" "${local_warped_file}" "${TARGET_CRS}" "${remote_target_file}")
+    CMD_ARGS+=("${local_src_file}" "${local_warped_file}" "${TARGET_CRS}" "${remote_target_file}")
   done
+  cd ..
+
   src_count=$(($src_count+1))
 done