From 6e88ee8ead6c51f439d4977eaf40a90caac5b50c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 14 Nov 2024 10:36:30 +0100 Subject: [PATCH 1/5] Test: Bump Iceberg-Java to 1.7.0 --- dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index 02affa78e2..ea0df40cef 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -38,7 +38,7 @@ WORKDIR ${SPARK_HOME} ENV SPARK_VERSION=3.5.0 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12 -ENV ICEBERG_VERSION=1.6.0 +ENV ICEBERG_VERSION=1.7.0 ENV PYICEBERG_VERSION=0.7.1 RUN curl --retry 3 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ From a5dc5663125e6dc9f5f2d47068ec13e5159fc51c Mon Sep 17 00:00:00 2001 From: Fokko Date: Sun, 8 Dec 2024 16:10:10 +0100 Subject: [PATCH 2/5] Bump to 1.7.1 --- dev/Dockerfile | 2 +- dev/provision.py | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index fe3dfce554..c7f3a9bc72 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -38,7 +38,7 @@ WORKDIR ${SPARK_HOME} ENV SPARK_VERSION=3.5.0 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12 -ENV ICEBERG_VERSION=1.7.0 +ENV ICEBERG_VERSION=1.7.1 ENV PYICEBERG_VERSION=0.8.0 RUN curl --retry 3 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ diff --git a/dev/provision.py b/dev/provision.py index 53360748b6..6d98ce3ff6 100644 --- a/dev/provision.py +++ b/dev/provision.py @@ -120,10 +120,6 @@ """ ) - # Partitioning is not really needed, but there is a bug: - # https://github.com/apache/iceberg/pull/7685 - spark.sql(f"ALTER TABLE {catalog_name}.default.test_positional_mor_deletes ADD PARTITION FIELD years(dt) AS dt_years") - spark.sql( f""" INSERT INTO {catalog_name}.default.test_positional_mor_deletes @@ -168,10 +164,6 @@ """ ) - # Partitioning is not really needed, but there is a bug: - # https://github.com/apache/iceberg/pull/7685 - spark.sql(f"ALTER TABLE {catalog_name}.default.test_positional_mor_double_deletes ADD PARTITION FIELD years(dt) AS dt_years") - spark.sql( f""" INSERT INTO {catalog_name}.default.test_positional_mor_double_deletes From 5c771a437a783ba50a5a3f858048b9375c493581 Mon Sep 17 00:00:00 2001 From: Fokko Date: Mon, 9 Dec 2024 15:25:14 +0100 Subject: [PATCH 3/5] Revert that for now --- dev/provision.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dev/provision.py b/dev/provision.py index 6d98ce3ff6..53360748b6 100644 --- a/dev/provision.py +++ b/dev/provision.py @@ -120,6 +120,10 @@ """ ) + # Partitioning is not really needed, but there is a bug: + # https://github.com/apache/iceberg/pull/7685 + spark.sql(f"ALTER TABLE {catalog_name}.default.test_positional_mor_deletes ADD PARTITION FIELD years(dt) AS dt_years") + spark.sql( f""" INSERT INTO {catalog_name}.default.test_positional_mor_deletes @@ -164,6 +168,10 @@ """ ) + # Partitioning is not really needed, but there is a bug: + # https://github.com/apache/iceberg/pull/7685 + spark.sql(f"ALTER TABLE {catalog_name}.default.test_positional_mor_double_deletes ADD PARTITION FIELD years(dt) AS dt_years") + spark.sql( f""" INSERT INTO {catalog_name}.default.test_positional_mor_double_deletes From 57de760ca2095208b7eeade20b92539cd2260d4e Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 17 Dec 2024 22:11:05 +0100 Subject: [PATCH 4/5] Test --- dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index f0db87d995..e0b0b829a0 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -36,7 +36,7 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$ RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events WORKDIR ${SPARK_HOME} -ENV SPARK_VERSION=3.5.3 +ENV SPARK_VERSION=3.5.2 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12 ENV ICEBERG_VERSION=1.7.1 ENV PYICEBERG_VERSION=0.8.1 From 8330e65cfc682ce66547fa62cc01f6a60c3bc4f2 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 17 Dec 2024 22:14:55 +0100 Subject: [PATCH 5/5] Update dev/Dockerfile --- dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index e0b0b829a0..f0db87d995 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -36,7 +36,7 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$ RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events WORKDIR ${SPARK_HOME} -ENV SPARK_VERSION=3.5.2 +ENV SPARK_VERSION=3.5.3 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12 ENV ICEBERG_VERSION=1.7.1 ENV PYICEBERG_VERSION=0.8.1