add predicate pushdown

add predicate pushdown formatting fixed filename issue fixed filename issue updated formatting updated formatting added pred push tests added pred push tests2 fixed formatting, removed /data removed /data Force remove tracked files from data/iceberg Remove tracked files from data/iceberg
duckdb · Oct 25, 2024 · eb8c753 · eb8c753
1 parent d62d91d
commit eb8c753
Show file tree

Hide file tree

Showing 125 changed files with 985 additions and 6,286 deletions.
diff --git a/.gitignore b/.gitignore
@@ -9,6 +9,8 @@ test/python/__pycache__/
 .Rhistory
 test/sql/tmp.test
 data/iceberg/generated_*
+data/iceberg/generated_*/
+data/iceberg/
 scripts/metastore_db/
 scripts/derby.log
 scripts/test-script-with-path.sql
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/count.csv b/data/iceberg/generated_spec1_0_001/expected_results/last/count.csv
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/data/._SUCCESS.crc b/data/iceberg/generated_spec1_0_001/expected_results/last/data/._SUCCESS.crc
diff --git a/...esults/last/data/.part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet.crc b/...esults/last/data/.part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet.crc
diff --git a/...ted_results/last/data/part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet b/...ted_results/last/data/part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/query.sql b/data/iceberg/generated_spec1_0_001/expected_results/last/query.sql
diff --git a/...001/pyspark_iceberg_table/data/00000-1-bcb5bbb9-a993-41f7-95e1-09e0c2475f4a-00001.parquet b/...001/pyspark_iceberg_table/data/00000-1-bcb5bbb9-a993-41f7-95e1-09e0c2475f4a-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00000-14-db28d572-b7de-4568-9b00-8c55c69cb179-00001.parquet b/...01/pyspark_iceberg_table/data/00000-14-db28d572-b7de-4568-9b00-8c55c69cb179-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00000-26-f6481588-9ba4-4a7b-b3dd-f188d41fa5b8-00001.parquet b/...01/pyspark_iceberg_table/data/00000-26-f6481588-9ba4-4a7b-b3dd-f188d41fa5b8-00001.parquet
diff --git a/...01/pyspark_iceberg_table/data/00000-36-cf35a788-d8c2-4ded-a9f7-5239797e80b8-00001.parquet b/...01/pyspark_iceberg_table/data/00000-36-cf35a788-d8c2-4ded-a9f7-5239797e80b8-00001.parquet
diff --git a/...001/pyspark_iceberg_table/data/00000-5-bd5417f5-f28c-46b1-b1ab-39ee9c191368-00001.parquet b/...001/pyspark_iceberg_table/data/00000-5-bd5417f5-f28c-46b1-b1ab-39ee9c191368-00001.parquet
diff --git a/...001/pyspark_iceberg_table/data/00000-9-6cdc0135-4256-4772-8c3e-3f4803ded842-00001.parquet b/...001/pyspark_iceberg_table/data/00000-9-6cdc0135-4256-4772-8c3e-3f4803ded842-00001.parquet
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m0.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m1.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m1.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m0.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m1.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m1.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/7723fb1b-ae48-49de-9e77-cd7945667cb9-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/7723fb1b-ae48-49de-9e77-cd7945667cb9-m0.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m0.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m1.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m1.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m0.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m1.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m1.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m2.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m2.avro
diff --git a/...d_spec1_0_001/pyspark_iceberg_table/metadata/e3febcc2-7f11-44b9-80af-571fb1c0463a-m0.avro b/...d_spec1_0_001/pyspark_iceberg_table/metadata/e3febcc2-7f11-44b9-80af-571fb1c0463a-m0.avro
diff --git a/...eberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro b/...eberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro
diff --git a/...eberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro b/...eberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro
diff --git a/...eberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro b/...eberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro
diff --git a/...eberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro b/...eberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro
diff --git a/...eberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro b/...eberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro
diff --git a/...eberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro b/...eberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro
diff --git a/...eberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro b/...eberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json