From d5998c61b44253372b26ff8be4fa070fc1b0d633 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 22 Dec 2024 10:46:41 -0800 Subject: [PATCH] add reminder --- dev/Dockerfile | 1 + tests/conftest.py | 1 + 2 files changed, 2 insertions(+) diff --git a/dev/Dockerfile b/dev/Dockerfile index d4346bf757..fe0297bd1f 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -36,6 +36,7 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$ RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events WORKDIR ${SPARK_HOME} +# Remember to also update `tests/conftest`'s spark setting ENV SPARK_VERSION=3.5.3 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12 ENV ICEBERG_VERSION=1.6.0 diff --git a/tests/conftest.py b/tests/conftest.py index fb9d34a565..22329b3882 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2240,6 +2240,7 @@ def spark() -> "SparkSession": from pyspark.sql import SparkSession + # Remember to also update `dev/Dockerfile` spark_version = ".".join(importlib.metadata.version("pyspark").split(".")[:2]) scala_version = "2.12" iceberg_version = "1.6.0"