From 8a8e81344bb0730e1657ca664da793b5d3312116 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 9 Aug 2024 10:34:03 +0200 Subject: [PATCH] Allow setting `write.parquet.page-row-limit` (#1017) * Allow setting `write.parquet.page-row-limit` It is being passed down to PyArrow here: https://github.com/apache/iceberg-python/blob/8aeab4951080fa196c0d29c72cba1cbba824ffc4/pyiceberg/io/pyarrow.py#L2374-L2377 * Update test --- pyiceberg/io/pyarrow.py | 1 - tests/integration/test_writes/test_writes.py | 1 - 2 files changed, 2 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 6c5db515c7..52188459c4 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2285,7 +2285,6 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> Dict[str, Any]: for key_pattern in [ TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, - TableProperties.PARQUET_PAGE_ROW_LIMIT, TableProperties.PARQUET_BLOOM_FILTER_MAX_BYTES, f"{TableProperties.PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX}.*", ]: diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index f9ff7d1012..0716862806 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -528,7 +528,6 @@ def test_write_parquet_other_properties( "properties", [ {"write.parquet.row-group-size-bytes": "42"}, - {"write.parquet.page-row-limit": "42"}, {"write.parquet.bloom-filter-enabled.column.bool": "42"}, {"write.parquet.bloom-filter-max-bytes": "42"}, ],