Fix export test esti job (#7635)

treeverse · Apr 9, 2024 · b9a81bc · b9a81bc
1 parent ac29208
commit b9a81bc
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 34 deletions.
diff --git a/.github/workflows/esti.yaml b/.github/workflows/esti.yaml
@@ -525,7 +525,6 @@ jobs:
         run: docker compose logs --tail=2500 lakefs
 
   export:
-    if: false  # TODO (niro): We need to enable this again!!!
     name: Test lakeFS rclone export functionality
     needs: [deploy-image, deploy-rclone-export-image, login-to-amazon-ecr]
     runs-on: ubuntu-22.04

diff --git a/test/rclone_export/docker-compose.yaml b/test/rclone_export/docker-compose.yaml
@@ -1,4 +1,3 @@
-version: '3.9'
 services:
   postgres:
     image: "postgres:11"
@@ -34,9 +33,6 @@ services:
 
   lakefs-export:
     image: "${REPO:-treeverse}/lakefs-rclone-export:${EXPORT_TAG:-latest}"
-    depends_on:
-      - "lakefs"
-      - "postgres"
     profiles: ["client"]
     environment:
       - LAKEFS_ENDPOINT=http://lakefs:8000

diff --git a/test/rclone_export/run-test.sh b/test/rclone_export/run-test.sh
@@ -2,24 +2,31 @@
 
 REPOSITORY=${REPOSITORY:-example}
 
+
+run_cmd_and_validate() {
+    echo $1
+    echo "Running: $2"
+    newVariable=$(eval $2)
+    if [[ $? != "0" ]] 
+    then
+      echo "FAILED!" && exit 1
+    fi
+    echo "Output:"
+    echo $newVariable
+}
+
 # Run Export without previous commit
 echo "Current working directory: ${WORKING_DIRECTORY}"
-echo "upload file_one"
-docker compose exec -T lakefs lakectl fs upload lakefs://${REPOSITORY}/main/a/file_one.txt --source /local/file_one.txt
-echo "res $?"
+run_cmd_and_validate "upload file_one" "docker compose exec -T lakefs lakectl fs upload lakefs://${REPOSITORY}/main/a/file_one.txt --source /local/file_one.txt"
 
-echo "export no previous commit"
-docker compose --project-directory ${WORKING_DIRECTORY} run --rm lakefs-export ${REPOSITORY} ${EXPORT_LOCATION} --branch="main"
-echo "res $?"
+run_cmd_and_validate "export no previous commit" "docker compose --project-directory ${WORKING_DIRECTORY} run --rm lakefs-export ${REPOSITORY} ${EXPORT_LOCATION} --branch=main"
 
 # Validate export
 lakectl_out=$(mktemp)
 s3_out=$(mktemp)
 trap 'rm -f -- $s3_out $lakectl_out' INT TERM EXIT
 
-echo "ls"
 docker compose exec -T lakefs lakectl fs ls --recursive --no-color lakefs://${REPOSITORY}/main/ | awk '{print $8}' | sort > ${lakectl_out}
-echo "res $?"
 
 n=$(grep -o "/" <<< ${EXPORT_LOCATION} | wc -l)
 
@@ -30,37 +37,28 @@ else
   n=$((n-1))
 fi
 
-echo "aws ls"
 aws s3 ls --recursive ${EXPORT_LOCATION} | awk '{print $4}'| cut -d/ -f ${n}-  | grep -v EXPORT_ | sort > ${s3_out}
-echo "res $?"
+
+echo $(cat $lakectl_out)
+echo $(cat $s3_out)
 
 if ! diff ${lakectl_out} ${s3_out}; then
   echo "export location and lakefs should contain same objects"
   exit 1
 fi
 
 # Run Export with previous commit - add file and also delete an existing file
-echo "export previous commit"
-docker compose exec -T lakefs lakectl fs upload lakefs://${REPOSITORY}/main/a/file_two.txt --source /local/file_two.txt
-echo "res $?"
-docker compose exec -T lakefs lakectl fs rm lakefs://${REPOSITORY}/main/a/file_one.txt
-echo "res $?"
-docker compose exec -T lakefs lakectl commit lakefs://${REPOSITORY}/main --message="removed file_one and added file_two"
-echo "res $?"
-
-docker compose --project-directory ${WORKING_DIRECTORY} run --rm lakefs-export ${REPOSITORY} ${EXPORT_LOCATION} --branch="main" --prev_commit_id="some_commit"
-echo "res $?"
+run_cmd_and_validate "upload file_two" "docker compose exec -T lakefs lakectl fs upload lakefs://${REPOSITORY}/main/a/file_two.txt --source /local/file_two.txt"
+run_cmd_and_validate "delete file_one" "docker compose exec -T lakefs lakectl fs rm lakefs://${REPOSITORY}/main/a/file_one.txt"
+run_cmd_and_validate "commit changes" "docker compose exec -T lakefs lakectl commit lakefs://${REPOSITORY}/main --message='removed file_one and added file_two'"
+run_cmd_and_validate "export previous commit" "docker compose --project-directory ${WORKING_DIRECTORY} run --rm lakefs-export ${REPOSITORY} ${EXPORT_LOCATION} --branch=main --prev_commit_id='some_commit'"
 
 # Validate sync
 lakectl_out=$(mktemp)
 s3_out=$(mktemp)
 trap 'rm -f -- $s3_out $lakectl_out' INT TERM EXIT
 
-echo "ls"
 docker compose exec -T lakefs lakectl fs ls --recursive --no-color lakefs://${REPOSITORY}/main/ | awk '{print $8}' | sort > ${lakectl_out}
-echo "res $?"
-
-echo "aws ls"
 aws s3 ls --recursive ${EXPORT_LOCATION} | awk '{print $4}'| cut -d/ -f ${n}-  | grep -v EXPORT_ | sort > ${s3_out}
 
 if ! diff ${lakectl_out} ${s3_out}; then
@@ -69,13 +67,13 @@ if ! diff ${lakectl_out} ${s3_out}; then
 fi
 
 # Run Export with commit_id reference
-echo "export with commit_id"
-docker compose exec -T lakefs lakectl fs upload lakefs://${REPOSITORY}/main/a/file_three.txt --source /local/file_three.txt
-echo "res $?"
+run_cmd_and_validate "upload file_three" "docker compose exec -T lakefs lakectl fs upload lakefs://${REPOSITORY}/main/a/file_three.txt --source /local/file_three.txt"
+
 commit_id=$(docker compose exec -T lakefs lakectl commit lakefs://${REPOSITORY}/main --message="added file_three" | sed -n 4p | awk '{print $2}')
 
-docker compose --project-directory ${WORKING_DIRECTORY} run --rm lakefs-export ${REPOSITORY} ${EXPORT_LOCATION} --commit_id="$commit_id"
-echo "res $?"
+# We should not validate the exit code - since it is erroneous 
+echo "run export"
+docker compose --project-directory ${WORKING_DIRECTORY} run --rm lakefs-export ${REPOSITORY} ${EXPORT_LOCATION} --commit_id=$commit_id
 
 echo "commit_id $commit_id"