Merge pull request #29 from rkaregar/karegar-version-update-3-3-3

chore: use spark 3.3.3 to address CVE and update test script.
GoogleCloudDataproc · Jun 7, 2024 · 7a75b67 · 7a75b67
2 parents 398009b + e5b4731
commit 7a75b67
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
@@ -124,13 +124,15 @@ delete_table() {
 run_load_test() {
     echo "***Running Load test.***"
     BIGTABLE_SPARK_JAR=$(get_bigtable_spark_jar)
+    RESULT_BUCKET_NAME="bigtable-spark-test-resources"
     TEST_SCRIPT="spark-bigtable_2.12/test-pyspark/load_test.py"
     BASE_SCRIPT="spark-bigtable_2.12/test-pyspark/test_base.py"
     TABLE_ID=$(create_table_id "load")
     gcloud dataproc jobs submit pyspark \
         --project=${BIGTABLE_PROJECT_ID} \
         --cluster=${DATAPROC_CLUSTER_NAME} \
         --region=${DATAPROC_CLUSTER_REGION} \
+        --bucket=${RESULT_BUCKET_NAME} \
         --jars=${BIGTABLE_SPARK_JAR} \
         ${TEST_SCRIPT} \
         --py-files=${BASE_SCRIPT} \

diff --git a/pom.xml b/pom.xml
@@ -73,7 +73,7 @@
 
     <scala.version>2.12.14</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
-    <spark.version>3.3.2</spark.version>
+    <spark.version>3.3.3</spark.version>
     <bigtable.java.version>2.38.0</bigtable.java.version>
     <bigtable.java.emulator.version>0.175.0</bigtable.java.emulator.version>