diff --git a/.gitignore b/.gitignore
index cb608ef..592a7fb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,8 @@ test/python/__pycache__/
 .Rhistory
 test/sql/tmp.test
 data/iceberg/generated_*
+data/iceberg/generated_*/
+data/iceberg/
 scripts/metastore_db/
 scripts/derby.log
 scripts/test-script-with-path.sql
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/count.csv b/data/iceberg/generated_spec1_0_001/expected_results/last/count.csv
deleted file mode 100644
index 3b35899..0000000
--- a/data/iceberg/generated_spec1_0_001/expected_results/last/count.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-count
-7690
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/data/._SUCCESS.crc b/data/iceberg/generated_spec1_0_001/expected_results/last/data/._SUCCESS.crc
deleted file mode 100644
index 3b7b044..0000000
Binary files a/data/iceberg/generated_spec1_0_001/expected_results/last/data/._SUCCESS.crc and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/data/.part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet.crc b/data/iceberg/generated_spec1_0_001/expected_results/last/data/.part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet.crc
deleted file mode 100644
index 62fa3b7..0000000
Binary files a/data/iceberg/generated_spec1_0_001/expected_results/last/data/.part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet.crc and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/data/part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet b/data/iceberg/generated_spec1_0_001/expected_results/last/data/part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet
deleted file mode 100644
index 54fb77d..0000000
Binary files a/data/iceberg/generated_spec1_0_001/expected_results/last/data/part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/query.sql b/data/iceberg/generated_spec1_0_001/expected_results/last/query.sql
deleted file mode 100644
index 6c7f91c..0000000
--- a/data/iceberg/generated_spec1_0_001/expected_results/last/query.sql
+++ /dev/null
@@ -1,3 +0,0 @@
--- The query executed at this step:
-ALTER TABLE iceberg_catalog.pyspark_iceberg_table
-ALTER COLUMN schema_evol_added_col_1 TYPE BIGINT;
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-1-bcb5bbb9-a993-41f7-95e1-09e0c2475f4a-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-1-bcb5bbb9-a993-41f7-95e1-09e0c2475f4a-00001.parquet
deleted file mode 100644
index 97ac7e8..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-1-bcb5bbb9-a993-41f7-95e1-09e0c2475f4a-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-14-db28d572-b7de-4568-9b00-8c55c69cb179-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-14-db28d572-b7de-4568-9b00-8c55c69cb179-00001.parquet
deleted file mode 100644
index 9b8b181..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-14-db28d572-b7de-4568-9b00-8c55c69cb179-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-26-f6481588-9ba4-4a7b-b3dd-f188d41fa5b8-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-26-f6481588-9ba4-4a7b-b3dd-f188d41fa5b8-00001.parquet
deleted file mode 100644
index 3755277..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-26-f6481588-9ba4-4a7b-b3dd-f188d41fa5b8-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-36-cf35a788-d8c2-4ded-a9f7-5239797e80b8-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-36-cf35a788-d8c2-4ded-a9f7-5239797e80b8-00001.parquet
deleted file mode 100644
index facc504..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-36-cf35a788-d8c2-4ded-a9f7-5239797e80b8-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-5-bd5417f5-f28c-46b1-b1ab-39ee9c191368-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-5-bd5417f5-f28c-46b1-b1ab-39ee9c191368-00001.parquet
deleted file mode 100644
index 5998bc2..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-5-bd5417f5-f28c-46b1-b1ab-39ee9c191368-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-9-6cdc0135-4256-4772-8c3e-3f4803ded842-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-9-6cdc0135-4256-4772-8c3e-3f4803ded842-00001.parquet
deleted file mode 100644
index 8f8a093..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-9-6cdc0135-4256-4772-8c3e-3f4803ded842-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m0.avro
deleted file mode 100644
index 90e2897..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m1.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m1.avro
deleted file mode 100644
index 584671d..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m1.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m0.avro
deleted file mode 100644
index 7dd57c8..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m1.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m1.avro
deleted file mode 100644
index 80355e7..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m1.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/7723fb1b-ae48-49de-9e77-cd7945667cb9-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/7723fb1b-ae48-49de-9e77-cd7945667cb9-m0.avro
deleted file mode 100644
index 54b6edf..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/7723fb1b-ae48-49de-9e77-cd7945667cb9-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m0.avro
deleted file mode 100644
index 2974743..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m1.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m1.avro
deleted file mode 100644
index 2ecadc1..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m1.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m0.avro
deleted file mode 100644
index 925a068..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m1.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m1.avro
deleted file mode 100644
index 3666460..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m1.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m2.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m2.avro
deleted file mode 100644
index feb60ea..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m2.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/e3febcc2-7f11-44b9-80af-571fb1c0463a-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/e3febcc2-7f11-44b9-80af-571fb1c0463a-m0.avro
deleted file mode 100644
index b812dca..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/e3febcc2-7f11-44b9-80af-571fb1c0463a-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro
deleted file mode 100644
index 9a7b32f..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro
deleted file mode 100644
index 45cc87a..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro
deleted file mode 100644
index c0715d7..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro
deleted file mode 100644
index e6edc42..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro
deleted file mode 100644
index be74220..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro
deleted file mode 100644
index fde9231..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro
deleted file mode 100644
index 63e2d76..0000000
Binary files a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json
deleted file mode 100644
index ec7f17f..0000000
--- a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json
+++ /dev/null
@@ -1,217 +0,0 @@
-{
-  "format-version" : 1,
-  "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1",
-  "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table",
-  "last-updated-ms" : 1719580919873,
-  "last-column-id" : 15,
-  "schema" : {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  },
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "partition-spec" : [ ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd"
-  },
-  "current-snapshot-id" : 9145725745960929259,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 9145725745960929259,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580919873,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440845",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "snapshot-id" : 9145725745960929259
-  } ],
-  "metadata-log" : [ ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json
deleted file mode 100644
index 8eb6b17..0000000
--- a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json
+++ /dev/null
@@ -1,246 +0,0 @@
-{
-  "format-version" : 1,
-  "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1",
-  "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table",
-  "last-updated-ms" : 1719580920785,
-  "last-column-id" : 15,
-  "schema" : {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  },
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "partition-spec" : [ ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd"
-  },
-  "current-snapshot-id" : 8671490307245765264,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 8671490307245765264,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580919873,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440845",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 8671490307245765264,
-    "parent-snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580920785,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6005",
-      "deleted-records" : "6005",
-      "added-files-size" : "340114",
-      "removed-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "340114",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "snapshot-id" : 9145725745960929259
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "snapshot-id" : 8671490307245765264
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json
deleted file mode 100644
index ae141f5..0000000
--- a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json
+++ /dev/null
@@ -1,272 +0,0 @@
-{
-  "format-version" : 1,
-  "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1",
-  "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table",
-  "last-updated-ms" : 1719580921348,
-  "last-column-id" : 15,
-  "schema" : {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  },
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "partition-spec" : [ ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd"
-  },
-  "current-snapshot-id" : 4543110679664799316,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 4543110679664799316,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580919873,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440845",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 8671490307245765264,
-    "parent-snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580920785,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6005",
-      "deleted-records" : "6005",
-      "added-files-size" : "340114",
-      "removed-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "340114",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 4543110679664799316,
-    "parent-snapshot-id" : 8671490307245765264,
-    "timestamp-ms" : 1719580921348,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133331",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "473445",
-      "total-data-files" : "2",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "snapshot-id" : 9145725745960929259
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "snapshot-id" : 8671490307245765264
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "snapshot-id" : 4543110679664799316
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json
deleted file mode 100644
index b44942a..0000000
--- a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json
+++ /dev/null
@@ -1,301 +0,0 @@
-{
-  "format-version" : 1,
-  "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1",
-  "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table",
-  "last-updated-ms" : 1719580921764,
-  "last-column-id" : 15,
-  "schema" : {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  },
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "partition-spec" : [ ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd"
-  },
-  "current-snapshot-id" : 6238750566879819059,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 6238750566879819059,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580919873,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440845",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 8671490307245765264,
-    "parent-snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580920785,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6005",
-      "deleted-records" : "6005",
-      "added-files-size" : "340114",
-      "removed-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "340114",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 4543110679664799316,
-    "parent-snapshot-id" : 8671490307245765264,
-    "timestamp-ms" : 1719580921348,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133331",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "473445",
-      "total-data-files" : "2",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 6238750566879819059,
-    "parent-snapshot-id" : 4543110679664799316,
-    "timestamp-ms" : 1719580921764,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "2",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "399010",
-      "removed-files-size" : "473445",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "snapshot-id" : 9145725745960929259
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "snapshot-id" : 8671490307245765264
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "snapshot-id" : 4543110679664799316
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "snapshot-id" : 6238750566879819059
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json
deleted file mode 100644
index 1aabc15..0000000
--- a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json
+++ /dev/null
@@ -1,324 +0,0 @@
-{
-  "format-version" : 1,
-  "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1",
-  "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table",
-  "last-updated-ms" : 1719580922113,
-  "last-column-id" : 15,
-  "schema" : {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  },
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "partition-spec" : [ ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd"
-  },
-  "current-snapshot-id" : 2276968461870063565,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 2276968461870063565,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580919873,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440845",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 8671490307245765264,
-    "parent-snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580920785,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6005",
-      "deleted-records" : "6005",
-      "added-files-size" : "340114",
-      "removed-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "340114",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 4543110679664799316,
-    "parent-snapshot-id" : 8671490307245765264,
-    "timestamp-ms" : 1719580921348,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133331",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "473445",
-      "total-data-files" : "2",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 6238750566879819059,
-    "parent-snapshot-id" : 4543110679664799316,
-    "timestamp-ms" : 1719580921764,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "2",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "399010",
-      "removed-files-size" : "473445",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 2276968461870063565,
-    "parent-snapshot-id" : 6238750566879819059,
-    "timestamp-ms" : 1719580922113,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "changed-partition-count" : "0",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "snapshot-id" : 9145725745960929259
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "snapshot-id" : 8671490307245765264
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "snapshot-id" : 4543110679664799316
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "snapshot-id" : 6238750566879819059
-  }, {
-    "timestamp-ms" : 1719580922113,
-    "snapshot-id" : 2276968461870063565
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json
deleted file mode 100644
index 9162f1b..0000000
--- a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json
+++ /dev/null
@@ -1,353 +0,0 @@
-{
-  "format-version" : 1,
-  "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1",
-  "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table",
-  "last-updated-ms" : 1719580922559,
-  "last-column-id" : 15,
-  "schema" : {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  },
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "partition-spec" : [ ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd"
-  },
-  "current-snapshot-id" : 1692767036460164714,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 1692767036460164714,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580919873,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440845",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 8671490307245765264,
-    "parent-snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580920785,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6005",
-      "deleted-records" : "6005",
-      "added-files-size" : "340114",
-      "removed-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "340114",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 4543110679664799316,
-    "parent-snapshot-id" : 8671490307245765264,
-    "timestamp-ms" : 1719580921348,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133331",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "473445",
-      "total-data-files" : "2",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 6238750566879819059,
-    "parent-snapshot-id" : 4543110679664799316,
-    "timestamp-ms" : 1719580921764,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "2",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "399010",
-      "removed-files-size" : "473445",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 2276968461870063565,
-    "parent-snapshot-id" : 6238750566879819059,
-    "timestamp-ms" : 1719580922113,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "changed-partition-count" : "0",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 1692767036460164714,
-    "parent-snapshot-id" : 2276968461870063565,
-    "timestamp-ms" : 1719580922559,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "399010",
-      "removed-files-size" : "399010",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "snapshot-id" : 9145725745960929259
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "snapshot-id" : 8671490307245765264
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "snapshot-id" : 4543110679664799316
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "snapshot-id" : 6238750566879819059
-  }, {
-    "timestamp-ms" : 1719580922113,
-    "snapshot-id" : 2276968461870063565
-  }, {
-    "timestamp-ms" : 1719580922559,
-    "snapshot-id" : 1692767036460164714
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580922113,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v7.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v7.metadata.json
deleted file mode 100644
index 744af3d..0000000
--- a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v7.metadata.json
+++ /dev/null
@@ -1,445 +0,0 @@
-{
-  "format-version" : 1,
-  "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1",
-  "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table",
-  "last-updated-ms" : 1719580922734,
-  "last-column-id" : 16,
-  "schema" : {
-    "type" : "struct",
-    "schema-id" : 1,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "int"
-    } ]
-  },
-  "current-schema-id" : 1,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  }, {
-    "type" : "struct",
-    "schema-id" : 1,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "int"
-    } ]
-  } ],
-  "partition-spec" : [ ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd"
-  },
-  "current-snapshot-id" : 1692767036460164714,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 1692767036460164714,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580919873,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440845",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 8671490307245765264,
-    "parent-snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580920785,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6005",
-      "deleted-records" : "6005",
-      "added-files-size" : "340114",
-      "removed-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "340114",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 4543110679664799316,
-    "parent-snapshot-id" : 8671490307245765264,
-    "timestamp-ms" : 1719580921348,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133331",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "473445",
-      "total-data-files" : "2",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 6238750566879819059,
-    "parent-snapshot-id" : 4543110679664799316,
-    "timestamp-ms" : 1719580921764,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "2",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "399010",
-      "removed-files-size" : "473445",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 2276968461870063565,
-    "parent-snapshot-id" : 6238750566879819059,
-    "timestamp-ms" : 1719580922113,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "changed-partition-count" : "0",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 1692767036460164714,
-    "parent-snapshot-id" : 2276968461870063565,
-    "timestamp-ms" : 1719580922559,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "399010",
-      "removed-files-size" : "399010",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "snapshot-id" : 9145725745960929259
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "snapshot-id" : 8671490307245765264
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "snapshot-id" : 4543110679664799316
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "snapshot-id" : 6238750566879819059
-  }, {
-    "timestamp-ms" : 1719580922113,
-    "snapshot-id" : 2276968461870063565
-  }, {
-    "timestamp-ms" : 1719580922559,
-    "snapshot-id" : 1692767036460164714
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580922113,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580922559,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v8.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v8.metadata.json
deleted file mode 100644
index ede9cca..0000000
--- a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v8.metadata.json
+++ /dev/null
@@ -1,474 +0,0 @@
-{
-  "format-version" : 1,
-  "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1",
-  "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table",
-  "last-updated-ms" : 1719580923120,
-  "last-column-id" : 16,
-  "schema" : {
-    "type" : "struct",
-    "schema-id" : 1,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "int"
-    } ]
-  },
-  "current-schema-id" : 1,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  }, {
-    "type" : "struct",
-    "schema-id" : 1,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "int"
-    } ]
-  } ],
-  "partition-spec" : [ ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd"
-  },
-  "current-snapshot-id" : 4407328776463037310,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 4407328776463037310,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580919873,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440845",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 8671490307245765264,
-    "parent-snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580920785,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6005",
-      "deleted-records" : "6005",
-      "added-files-size" : "340114",
-      "removed-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "340114",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 4543110679664799316,
-    "parent-snapshot-id" : 8671490307245765264,
-    "timestamp-ms" : 1719580921348,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133331",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "473445",
-      "total-data-files" : "2",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 6238750566879819059,
-    "parent-snapshot-id" : 4543110679664799316,
-    "timestamp-ms" : 1719580921764,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "2",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "399010",
-      "removed-files-size" : "473445",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 2276968461870063565,
-    "parent-snapshot-id" : 6238750566879819059,
-    "timestamp-ms" : 1719580922113,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "changed-partition-count" : "0",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 1692767036460164714,
-    "parent-snapshot-id" : 2276968461870063565,
-    "timestamp-ms" : 1719580922559,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "399010",
-      "removed-files-size" : "399010",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 4407328776463037310,
-    "parent-snapshot-id" : 1692767036460164714,
-    "timestamp-ms" : 1719580923120,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "400831",
-      "removed-files-size" : "399010",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "400831",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro",
-    "schema-id" : 1
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "snapshot-id" : 9145725745960929259
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "snapshot-id" : 8671490307245765264
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "snapshot-id" : 4543110679664799316
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "snapshot-id" : 6238750566879819059
-  }, {
-    "timestamp-ms" : 1719580922113,
-    "snapshot-id" : 2276968461870063565
-  }, {
-    "timestamp-ms" : 1719580922559,
-    "snapshot-id" : 1692767036460164714
-  }, {
-    "timestamp-ms" : 1719580923120,
-    "snapshot-id" : 4407328776463037310
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580922113,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580922559,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580922734,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v7.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v9.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v9.metadata.json
deleted file mode 100644
index 6d60244..0000000
--- a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v9.metadata.json
+++ /dev/null
@@ -1,561 +0,0 @@
-{
-  "format-version" : 1,
-  "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1",
-  "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table",
-  "last-updated-ms" : 1719580923295,
-  "last-column-id" : 16,
-  "schema" : {
-    "type" : "struct",
-    "schema-id" : 2,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "long"
-    } ]
-  },
-  "current-schema-id" : 2,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  }, {
-    "type" : "struct",
-    "schema-id" : 1,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "int"
-    } ]
-  }, {
-    "type" : "struct",
-    "schema-id" : 2,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "long"
-    } ]
-  } ],
-  "partition-spec" : [ ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd"
-  },
-  "current-snapshot-id" : 4407328776463037310,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 4407328776463037310,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580919873,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440845",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 8671490307245765264,
-    "parent-snapshot-id" : 9145725745960929259,
-    "timestamp-ms" : 1719580920785,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6005",
-      "deleted-records" : "6005",
-      "added-files-size" : "340114",
-      "removed-files-size" : "440845",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "340114",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 4543110679664799316,
-    "parent-snapshot-id" : 8671490307245765264,
-    "timestamp-ms" : 1719580921348,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133331",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "473445",
-      "total-data-files" : "2",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 6238750566879819059,
-    "parent-snapshot-id" : 4543110679664799316,
-    "timestamp-ms" : 1719580921764,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "2",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "399010",
-      "removed-files-size" : "473445",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 2276968461870063565,
-    "parent-snapshot-id" : 6238750566879819059,
-    "timestamp-ms" : 1719580922113,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "changed-partition-count" : "0",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 1692767036460164714,
-    "parent-snapshot-id" : 2276968461870063565,
-    "timestamp-ms" : 1719580922559,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "399010",
-      "removed-files-size" : "399010",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "399010",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro",
-    "schema-id" : 0
-  }, {
-    "snapshot-id" : 4407328776463037310,
-    "parent-snapshot-id" : 1692767036460164714,
-    "timestamp-ms" : 1719580923120,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580917302",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "7690",
-      "deleted-records" : "7690",
-      "added-files-size" : "400831",
-      "removed-files-size" : "399010",
-      "changed-partition-count" : "1",
-      "total-records" : "7690",
-      "total-files-size" : "400831",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro",
-    "schema-id" : 1
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "snapshot-id" : 9145725745960929259
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "snapshot-id" : 8671490307245765264
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "snapshot-id" : 4543110679664799316
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "snapshot-id" : 6238750566879819059
-  }, {
-    "timestamp-ms" : 1719580922113,
-    "snapshot-id" : 2276968461870063565
-  }, {
-    "timestamp-ms" : 1719580922559,
-    "snapshot-id" : 1692767036460164714
-  }, {
-    "timestamp-ms" : 1719580923120,
-    "snapshot-id" : 4407328776463037310
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580919873,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580920785,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921348,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580921764,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580922113,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580922559,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580922734,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v7.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580923120,
-    "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v8.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/version-hint.text b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/version-hint.text
deleted file mode 100644
index f11c82a..0000000
--- a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/version-hint.text
+++ /dev/null
@@ -1 +0,0 @@
-9
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/count.csv b/data/iceberg/generated_spec2_0_001/expected_results/last/count.csv
deleted file mode 100644
index 1221195..0000000
--- a/data/iceberg/generated_spec2_0_001/expected_results/last/count.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-count
-6592
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/data/._SUCCESS.crc b/data/iceberg/generated_spec2_0_001/expected_results/last/data/._SUCCESS.crc
deleted file mode 100644
index 3b7b044..0000000
Binary files a/data/iceberg/generated_spec2_0_001/expected_results/last/data/._SUCCESS.crc and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc b/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc
deleted file mode 100644
index a0d706d..0000000
Binary files a/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc b/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc
deleted file mode 100644
index 2f43db6..0000000
Binary files a/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet b/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet
deleted file mode 100644
index 53283bf..0000000
Binary files a/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet b/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet
deleted file mode 100644
index 6d6fe6d..0000000
Binary files a/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/query.sql b/data/iceberg/generated_spec2_0_001/expected_results/last/query.sql
deleted file mode 100644
index 6c7f91c..0000000
--- a/data/iceberg/generated_spec2_0_001/expected_results/last/query.sql
+++ /dev/null
@@ -1,3 +0,0 @@
--- The query executed at this step:
-ALTER TABLE iceberg_catalog.pyspark_iceberg_table
-ALTER COLUMN schema_evol_added_col_1 TYPE BIGINT;
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-1-3e88ec3a-0596-440f-9ce6-3debf172be49-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-1-3e88ec3a-0596-440f-9ce6-3debf172be49-00001.parquet
deleted file mode 100644
index 6837339..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-1-3e88ec3a-0596-440f-9ce6-3debf172be49-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001-deletes.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001-deletes.parquet
deleted file mode 100644
index 4d44760..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001-deletes.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001.parquet
deleted file mode 100644
index b3c12c7..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-24-3a7a66b3-bd3a-4417-b6a9-45cb309eddc2-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-24-3a7a66b3-bd3a-4417-b6a9-45cb309eddc2-00001.parquet
deleted file mode 100644
index 98c2d49..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-24-3a7a66b3-bd3a-4417-b6a9-45cb309eddc2-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001-deletes.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001-deletes.parquet
deleted file mode 100644
index 47329b2..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001-deletes.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001.parquet
deleted file mode 100644
index 979ad92..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001-deletes.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001-deletes.parquet
deleted file mode 100644
index 99aafa1..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001-deletes.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001.parquet
deleted file mode 100644
index c242949..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-7-3be35a72-224f-475b-a0eb-34cea92784b4-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-7-3be35a72-224f-475b-a0eb-34cea92784b4-00001.parquet
deleted file mode 100644
index 285069d..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-7-3be35a72-224f-475b-a0eb-34cea92784b4-00001.parquet and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/26871791-3133-4757-9cbc-b356c613c83a-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/26871791-3133-4757-9cbc-b356c613c83a-m0.avro
deleted file mode 100644
index a922251..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/26871791-3133-4757-9cbc-b356c613c83a-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m0.avro
deleted file mode 100644
index f0b73f1..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m1.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m1.avro
deleted file mode 100644
index d7b9a64..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m1.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m0.avro
deleted file mode 100644
index 19d600f..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m1.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m1.avro
deleted file mode 100644
index c8d664d..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m1.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/9ae37730-f1aa-4609-8b39-3f0ded6f78cf-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/9ae37730-f1aa-4609-8b39-3f0ded6f78cf-m0.avro
deleted file mode 100644
index 92c4470..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/9ae37730-f1aa-4609-8b39-3f0ded6f78cf-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m0.avro
deleted file mode 100644
index e28081d..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m1.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m1.avro
deleted file mode 100644
index 9101316..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m1.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m0.avro
deleted file mode 100644
index 9a6b6d5..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m0.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m1.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m1.avro
deleted file mode 100644
index 31716e1..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m1.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro
deleted file mode 100644
index fafc511..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro
deleted file mode 100644
index 3df6c65..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro
deleted file mode 100644
index f4da2fc..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4786266686210019019-1-7c6f85be-3a33-4e3a-817d-7839fa44ff07.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4786266686210019019-1-7c6f85be-3a33-4e3a-817d-7839fa44ff07.avro
deleted file mode 100644
index 0da1497..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4786266686210019019-1-7c6f85be-3a33-4e3a-817d-7839fa44ff07.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro
deleted file mode 100644
index 8f34f30..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro
deleted file mode 100644
index 3e9ee6b..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro
deleted file mode 100644
index aab9ece..0000000
Binary files a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro and /dev/null differ
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json
deleted file mode 100644
index 3c9acfc..0000000
--- a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json
+++ /dev/null
@@ -1,139 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7",
-  "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table",
-  "last-sequence-number" : 1,
-  "last-updated-ms" : 1719580927570,
-  "last-column-id" : 15,
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 764624380497366583,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 764624380497366583,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580927570,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440835",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440835",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "snapshot-id" : 764624380497366583
-  } ],
-  "metadata-log" : [ ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json
deleted file mode 100644
index bae2e24..0000000
--- a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json
+++ /dev/null
@@ -1,169 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7",
-  "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table",
-  "last-sequence-number" : 2,
-  "last-updated-ms" : 1719580928275,
-  "last-column-id" : 15,
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 4037069315291880534,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 4037069315291880534,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580927570,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440835",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440835",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 2,
-    "snapshot-id" : 4037069315291880534,
-    "parent-snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580928275,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "3077",
-      "added-files-size" : "114786",
-      "added-position-deletes" : "3077",
-      "changed-partition-count" : "1",
-      "total-records" : "9082",
-      "total-files-size" : "555621",
-      "total-data-files" : "2",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "snapshot-id" : 764624380497366583
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "snapshot-id" : 4037069315291880534
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json
deleted file mode 100644
index 7db94f1..0000000
--- a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json
+++ /dev/null
@@ -1,196 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7",
-  "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table",
-  "last-sequence-number" : 3,
-  "last-updated-ms" : 1719580929047,
-  "last-column-id" : 15,
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 6287117141668015642,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 6287117141668015642,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580927570,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440835",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440835",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 2,
-    "snapshot-id" : 4037069315291880534,
-    "parent-snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580928275,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "3077",
-      "added-files-size" : "114786",
-      "added-position-deletes" : "3077",
-      "changed-partition-count" : "1",
-      "total-records" : "9082",
-      "total-files-size" : "555621",
-      "total-data-files" : "2",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 3,
-    "snapshot-id" : 6287117141668015642,
-    "parent-snapshot-id" : 4037069315291880534,
-    "timestamp-ms" : 1719580929047,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133314",
-      "changed-partition-count" : "1",
-      "total-records" : "10767",
-      "total-files-size" : "688935",
-      "total-data-files" : "3",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "snapshot-id" : 764624380497366583
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "snapshot-id" : 4037069315291880534
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "snapshot-id" : 6287117141668015642
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json
deleted file mode 100644
index edcbb4b..0000000
--- a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json
+++ /dev/null
@@ -1,226 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7",
-  "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table",
-  "last-sequence-number" : 4,
-  "last-updated-ms" : 1719580929661,
-  "last-column-id" : 15,
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 6585012225877417653,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 6585012225877417653,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580927570,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440835",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440835",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 2,
-    "snapshot-id" : 4037069315291880534,
-    "parent-snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580928275,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "3077",
-      "added-files-size" : "114786",
-      "added-position-deletes" : "3077",
-      "changed-partition-count" : "1",
-      "total-records" : "9082",
-      "total-files-size" : "555621",
-      "total-data-files" : "2",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 3,
-    "snapshot-id" : 6287117141668015642,
-    "parent-snapshot-id" : 4037069315291880534,
-    "timestamp-ms" : 1719580929047,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133314",
-      "changed-partition-count" : "1",
-      "total-records" : "10767",
-      "total-files-size" : "688935",
-      "total-data-files" : "3",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 4,
-    "snapshot-id" : 6585012225877417653,
-    "parent-snapshot-id" : 6287117141668015642,
-    "timestamp-ms" : 1719580929661,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "7690",
-      "added-files-size" : "410506",
-      "added-position-deletes" : "7690",
-      "changed-partition-count" : "1",
-      "total-records" : "18457",
-      "total-files-size" : "1099441",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "snapshot-id" : 764624380497366583
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "snapshot-id" : 4037069315291880534
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "snapshot-id" : 6287117141668015642
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "snapshot-id" : 6585012225877417653
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json
deleted file mode 100644
index 61b94fc..0000000
--- a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json
+++ /dev/null
@@ -1,256 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7",
-  "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table",
-  "last-sequence-number" : 5,
-  "last-updated-ms" : 1719580930402,
-  "last-column-id" : 15,
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 4440319347650982524,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 4440319347650982524,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580927570,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440835",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440835",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 2,
-    "snapshot-id" : 4037069315291880534,
-    "parent-snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580928275,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "3077",
-      "added-files-size" : "114786",
-      "added-position-deletes" : "3077",
-      "changed-partition-count" : "1",
-      "total-records" : "9082",
-      "total-files-size" : "555621",
-      "total-data-files" : "2",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 3,
-    "snapshot-id" : 6287117141668015642,
-    "parent-snapshot-id" : 4037069315291880534,
-    "timestamp-ms" : 1719580929047,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133314",
-      "changed-partition-count" : "1",
-      "total-records" : "10767",
-      "total-files-size" : "688935",
-      "total-data-files" : "3",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 4,
-    "snapshot-id" : 6585012225877417653,
-    "parent-snapshot-id" : 6287117141668015642,
-    "timestamp-ms" : 1719580929661,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "7690",
-      "added-files-size" : "410506",
-      "added-position-deletes" : "7690",
-      "changed-partition-count" : "1",
-      "total-records" : "18457",
-      "total-files-size" : "1099441",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 5,
-    "snapshot-id" : 4440319347650982524,
-    "parent-snapshot-id" : 6585012225877417653,
-    "timestamp-ms" : 1719580930402,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6592",
-      "deleted-records" : "7690",
-      "added-files-size" : "333848",
-      "removed-files-size" : "388851",
-      "changed-partition-count" : "1",
-      "total-records" : "17359",
-      "total-files-size" : "1044438",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "snapshot-id" : 764624380497366583
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "snapshot-id" : 4037069315291880534
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "snapshot-id" : 6287117141668015642
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "snapshot-id" : 6585012225877417653
-  }, {
-    "timestamp-ms" : 1719580930402,
-    "snapshot-id" : 4440319347650982524
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json
deleted file mode 100644
index 2f8a8fb..0000000
--- a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json
+++ /dev/null
@@ -1,280 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7",
-  "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table",
-  "last-sequence-number" : 6,
-  "last-updated-ms" : 1719580930749,
-  "last-column-id" : 15,
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 3119545726281138740,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 3119545726281138740,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580927570,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440835",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440835",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 2,
-    "snapshot-id" : 4037069315291880534,
-    "parent-snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580928275,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "3077",
-      "added-files-size" : "114786",
-      "added-position-deletes" : "3077",
-      "changed-partition-count" : "1",
-      "total-records" : "9082",
-      "total-files-size" : "555621",
-      "total-data-files" : "2",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 3,
-    "snapshot-id" : 6287117141668015642,
-    "parent-snapshot-id" : 4037069315291880534,
-    "timestamp-ms" : 1719580929047,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133314",
-      "changed-partition-count" : "1",
-      "total-records" : "10767",
-      "total-files-size" : "688935",
-      "total-data-files" : "3",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 4,
-    "snapshot-id" : 6585012225877417653,
-    "parent-snapshot-id" : 6287117141668015642,
-    "timestamp-ms" : 1719580929661,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "7690",
-      "added-files-size" : "410506",
-      "added-position-deletes" : "7690",
-      "changed-partition-count" : "1",
-      "total-records" : "18457",
-      "total-files-size" : "1099441",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 5,
-    "snapshot-id" : 4440319347650982524,
-    "parent-snapshot-id" : 6585012225877417653,
-    "timestamp-ms" : 1719580930402,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6592",
-      "deleted-records" : "7690",
-      "added-files-size" : "333848",
-      "removed-files-size" : "388851",
-      "changed-partition-count" : "1",
-      "total-records" : "17359",
-      "total-files-size" : "1044438",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 6,
-    "snapshot-id" : 3119545726281138740,
-    "parent-snapshot-id" : 4440319347650982524,
-    "timestamp-ms" : 1719580930749,
-    "summary" : {
-      "operation" : "delete",
-      "spark.app.id" : "local-1719580924876",
-      "changed-partition-count" : "0",
-      "total-records" : "17359",
-      "total-files-size" : "1044438",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "snapshot-id" : 764624380497366583
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "snapshot-id" : 4037069315291880534
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "snapshot-id" : 6287117141668015642
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "snapshot-id" : 6585012225877417653
-  }, {
-    "timestamp-ms" : 1719580930402,
-    "snapshot-id" : 4440319347650982524
-  }, {
-    "timestamp-ms" : 1719580930749,
-    "snapshot-id" : 3119545726281138740
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580930402,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v7.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v7.metadata.json
deleted file mode 100644
index 7dd8062..0000000
--- a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v7.metadata.json
+++ /dev/null
@@ -1,367 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7",
-  "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table",
-  "last-sequence-number" : 6,
-  "last-updated-ms" : 1719580930997,
-  "last-column-id" : 16,
-  "current-schema-id" : 1,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  }, {
-    "type" : "struct",
-    "schema-id" : 1,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "int"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 3119545726281138740,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 3119545726281138740,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580927570,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440835",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440835",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 2,
-    "snapshot-id" : 4037069315291880534,
-    "parent-snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580928275,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "3077",
-      "added-files-size" : "114786",
-      "added-position-deletes" : "3077",
-      "changed-partition-count" : "1",
-      "total-records" : "9082",
-      "total-files-size" : "555621",
-      "total-data-files" : "2",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 3,
-    "snapshot-id" : 6287117141668015642,
-    "parent-snapshot-id" : 4037069315291880534,
-    "timestamp-ms" : 1719580929047,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133314",
-      "changed-partition-count" : "1",
-      "total-records" : "10767",
-      "total-files-size" : "688935",
-      "total-data-files" : "3",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 4,
-    "snapshot-id" : 6585012225877417653,
-    "parent-snapshot-id" : 6287117141668015642,
-    "timestamp-ms" : 1719580929661,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "7690",
-      "added-files-size" : "410506",
-      "added-position-deletes" : "7690",
-      "changed-partition-count" : "1",
-      "total-records" : "18457",
-      "total-files-size" : "1099441",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 5,
-    "snapshot-id" : 4440319347650982524,
-    "parent-snapshot-id" : 6585012225877417653,
-    "timestamp-ms" : 1719580930402,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6592",
-      "deleted-records" : "7690",
-      "added-files-size" : "333848",
-      "removed-files-size" : "388851",
-      "changed-partition-count" : "1",
-      "total-records" : "17359",
-      "total-files-size" : "1044438",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 6,
-    "snapshot-id" : 3119545726281138740,
-    "parent-snapshot-id" : 4440319347650982524,
-    "timestamp-ms" : 1719580930749,
-    "summary" : {
-      "operation" : "delete",
-      "spark.app.id" : "local-1719580924876",
-      "changed-partition-count" : "0",
-      "total-records" : "17359",
-      "total-files-size" : "1044438",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro",
-    "schema-id" : 0
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "snapshot-id" : 764624380497366583
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "snapshot-id" : 4037069315291880534
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "snapshot-id" : 6287117141668015642
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "snapshot-id" : 6585012225877417653
-  }, {
-    "timestamp-ms" : 1719580930402,
-    "snapshot-id" : 4440319347650982524
-  }, {
-    "timestamp-ms" : 1719580930749,
-    "snapshot-id" : 3119545726281138740
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580930402,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580930749,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v8.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v8.metadata.json
deleted file mode 100644
index 56d07a9..0000000
--- a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v8.metadata.json
+++ /dev/null
@@ -1,397 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7",
-  "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table",
-  "last-sequence-number" : 7,
-  "last-updated-ms" : 1719580931465,
-  "last-column-id" : 16,
-  "current-schema-id" : 1,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  }, {
-    "type" : "struct",
-    "schema-id" : 1,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "int"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 4786266686210019019,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 4786266686210019019,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580927570,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440835",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440835",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 2,
-    "snapshot-id" : 4037069315291880534,
-    "parent-snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580928275,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "3077",
-      "added-files-size" : "114786",
-      "added-position-deletes" : "3077",
-      "changed-partition-count" : "1",
-      "total-records" : "9082",
-      "total-files-size" : "555621",
-      "total-data-files" : "2",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 3,
-    "snapshot-id" : 6287117141668015642,
-    "parent-snapshot-id" : 4037069315291880534,
-    "timestamp-ms" : 1719580929047,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133314",
-      "changed-partition-count" : "1",
-      "total-records" : "10767",
-      "total-files-size" : "688935",
-      "total-data-files" : "3",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 4,
-    "snapshot-id" : 6585012225877417653,
-    "parent-snapshot-id" : 6287117141668015642,
-    "timestamp-ms" : 1719580929661,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "7690",
-      "added-files-size" : "410506",
-      "added-position-deletes" : "7690",
-      "changed-partition-count" : "1",
-      "total-records" : "18457",
-      "total-files-size" : "1099441",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 5,
-    "snapshot-id" : 4440319347650982524,
-    "parent-snapshot-id" : 6585012225877417653,
-    "timestamp-ms" : 1719580930402,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6592",
-      "deleted-records" : "7690",
-      "added-files-size" : "333848",
-      "removed-files-size" : "388851",
-      "changed-partition-count" : "1",
-      "total-records" : "17359",
-      "total-files-size" : "1044438",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 6,
-    "snapshot-id" : 3119545726281138740,
-    "parent-snapshot-id" : 4440319347650982524,
-    "timestamp-ms" : 1719580930749,
-    "summary" : {
-      "operation" : "delete",
-      "spark.app.id" : "local-1719580924876",
-      "changed-partition-count" : "0",
-      "total-records" : "17359",
-      "total-files-size" : "1044438",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 7,
-    "snapshot-id" : 4786266686210019019,
-    "parent-snapshot-id" : 3119545726281138740,
-    "timestamp-ms" : 1719580931465,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "685",
-      "added-files-size" : "51653",
-      "added-position-deletes" : "685",
-      "changed-partition-count" : "1",
-      "total-records" : "18044",
-      "total-files-size" : "1096091",
-      "total-data-files" : "5",
-      "total-delete-files" : "3",
-      "total-position-deletes" : "11452",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4786266686210019019-1-7c6f85be-3a33-4e3a-817d-7839fa44ff07.avro",
-    "schema-id" : 1
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "snapshot-id" : 764624380497366583
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "snapshot-id" : 4037069315291880534
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "snapshot-id" : 6287117141668015642
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "snapshot-id" : 6585012225877417653
-  }, {
-    "timestamp-ms" : 1719580930402,
-    "snapshot-id" : 4440319347650982524
-  }, {
-    "timestamp-ms" : 1719580930749,
-    "snapshot-id" : 3119545726281138740
-  }, {
-    "timestamp-ms" : 1719580931465,
-    "snapshot-id" : 4786266686210019019
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580930402,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580930749,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580930997,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v7.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v9.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v9.metadata.json
deleted file mode 100644
index 85c01bd..0000000
--- a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v9.metadata.json
+++ /dev/null
@@ -1,484 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7",
-  "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table",
-  "last-sequence-number" : 7,
-  "last-updated-ms" : 1719580931691,
-  "last-column-id" : 16,
-  "current-schema-id" : 2,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    } ]
-  }, {
-    "type" : "struct",
-    "schema-id" : 1,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "int"
-    } ]
-  }, {
-    "type" : "struct",
-    "schema-id" : 2,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey_bool",
-      "required" : false,
-      "type" : "boolean"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey_int",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey_long",
-      "required" : false,
-      "type" : "long"
-    }, {
-      "id" : 4,
-      "name" : "l_extendedprice_float",
-      "required" : false,
-      "type" : "float"
-    }, {
-      "id" : 5,
-      "name" : "l_extendedprice_double",
-      "required" : false,
-      "type" : "double"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice_dec9_2",
-      "required" : false,
-      "type" : "decimal(9, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_extendedprice_dec18_6",
-      "required" : false,
-      "type" : "decimal(18, 6)"
-    }, {
-      "id" : 8,
-      "name" : "l_extendedprice_dec38_10",
-      "required" : false,
-      "type" : "decimal(38, 10)"
-    }, {
-      "id" : 9,
-      "name" : "l_shipdate_date",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 10,
-      "name" : "l_partkey_time",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 11,
-      "name" : "l_commitdate_timestamp",
-      "required" : false,
-      "type" : "timestamp"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate_timestamp_tz",
-      "required" : false,
-      "type" : "timestamptz"
-    }, {
-      "id" : 13,
-      "name" : "l_comment_string",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 14,
-      "name" : "uuid",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_comment_blob",
-      "required" : false,
-      "type" : "binary"
-    }, {
-      "id" : 16,
-      "name" : "schema_evol_added_col_1",
-      "required" : false,
-      "type" : "long"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "peter",
-    "write.parquet.compression-codec" : "zstd",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 4786266686210019019,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 4786266686210019019,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580927570,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "6005",
-      "added-files-size" : "440835",
-      "changed-partition-count" : "1",
-      "total-records" : "6005",
-      "total-files-size" : "440835",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 2,
-    "snapshot-id" : 4037069315291880534,
-    "parent-snapshot-id" : 764624380497366583,
-    "timestamp-ms" : 1719580928275,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "3077",
-      "added-files-size" : "114786",
-      "added-position-deletes" : "3077",
-      "changed-partition-count" : "1",
-      "total-records" : "9082",
-      "total-files-size" : "555621",
-      "total-data-files" : "2",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 3,
-    "snapshot-id" : 6287117141668015642,
-    "parent-snapshot-id" : 4037069315291880534,
-    "timestamp-ms" : 1719580929047,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-records" : "1685",
-      "added-files-size" : "133314",
-      "changed-partition-count" : "1",
-      "total-records" : "10767",
-      "total-files-size" : "688935",
-      "total-data-files" : "3",
-      "total-delete-files" : "1",
-      "total-position-deletes" : "3077",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 4,
-    "snapshot-id" : 6585012225877417653,
-    "parent-snapshot-id" : 6287117141668015642,
-    "timestamp-ms" : 1719580929661,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "7690",
-      "added-files-size" : "410506",
-      "added-position-deletes" : "7690",
-      "changed-partition-count" : "1",
-      "total-records" : "18457",
-      "total-files-size" : "1099441",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 5,
-    "snapshot-id" : 4440319347650982524,
-    "parent-snapshot-id" : 6585012225877417653,
-    "timestamp-ms" : 1719580930402,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "6592",
-      "deleted-records" : "7690",
-      "added-files-size" : "333848",
-      "removed-files-size" : "388851",
-      "changed-partition-count" : "1",
-      "total-records" : "17359",
-      "total-files-size" : "1044438",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 6,
-    "snapshot-id" : 3119545726281138740,
-    "parent-snapshot-id" : 4440319347650982524,
-    "timestamp-ms" : 1719580930749,
-    "summary" : {
-      "operation" : "delete",
-      "spark.app.id" : "local-1719580924876",
-      "changed-partition-count" : "0",
-      "total-records" : "17359",
-      "total-files-size" : "1044438",
-      "total-data-files" : "4",
-      "total-delete-files" : "2",
-      "total-position-deletes" : "10767",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 7,
-    "snapshot-id" : 4786266686210019019,
-    "parent-snapshot-id" : 3119545726281138740,
-    "timestamp-ms" : 1719580931465,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1719580924876",
-      "added-data-files" : "1",
-      "added-position-delete-files" : "1",
-      "added-delete-files" : "1",
-      "added-records" : "685",
-      "added-files-size" : "51653",
-      "added-position-deletes" : "685",
-      "changed-partition-count" : "1",
-      "total-records" : "18044",
-      "total-files-size" : "1096091",
-      "total-data-files" : "5",
-      "total-delete-files" : "3",
-      "total-position-deletes" : "11452",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4786266686210019019-1-7c6f85be-3a33-4e3a-817d-7839fa44ff07.avro",
-    "schema-id" : 1
-  } ],
-  "statistics" : [ ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "snapshot-id" : 764624380497366583
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "snapshot-id" : 4037069315291880534
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "snapshot-id" : 6287117141668015642
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "snapshot-id" : 6585012225877417653
-  }, {
-    "timestamp-ms" : 1719580930402,
-    "snapshot-id" : 4440319347650982524
-  }, {
-    "timestamp-ms" : 1719580930749,
-    "snapshot-id" : 3119545726281138740
-  }, {
-    "timestamp-ms" : 1719580931465,
-    "snapshot-id" : 4786266686210019019
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1719580927570,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580928275,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929047,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580929661,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580930402,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580930749,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580930997,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v7.metadata.json"
-  }, {
-    "timestamp-ms" : 1719580931465,
-    "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v8.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/version-hint.text b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/version-hint.text
deleted file mode 100644
index f11c82a..0000000
--- a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/version-hint.text
+++ /dev/null
@@ -1 +0,0 @@
-9
\ No newline at end of file
diff --git a/data/iceberg/lineitem_iceberg/README.md b/data/iceberg/lineitem_iceberg/README.md
deleted file mode 100644
index d564b5f..0000000
--- a/data/iceberg/lineitem_iceberg/README.md
+++ /dev/null
@@ -1,20 +0,0 @@
-# README
-this iceberg table is generated by using DuckDB (v0.7.0) to generated TPC-H lineitem
-SF0.01 then storing that to a parquet file.
-
-Then pyspark (3.3.1) was used with the iceberg extension from https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.3_2.12/1.0.0/iceberg-spark-runtime-3.3_2.12-1.0.0.jar
-to write the iceberg table.
-
-finally, using pyspark, a delete query was performed on this iceberg table:
-
-```
-DELETE FROM iceberg_catalog.lineitem_iceberg where l_extendedprice < 10000
-```
-
-The result for Q06 of TPC-H on this table according to pyspark is now:
-```
-[Row(revenue=Decimal('1077536.9101'))]
-```
-
-Note: it appears that there are no deletes present in this iceberg table, the whole thing was rewritten.
-this is likely due to the fact that the table is so small?
\ No newline at end of file
diff --git a/data/iceberg/lineitem_iceberg/data/.00000-411-0792dcfe-4e25-4ca3-8ada-175286069a47-00001.parquet.crc b/data/iceberg/lineitem_iceberg/data/.00000-411-0792dcfe-4e25-4ca3-8ada-175286069a47-00001.parquet.crc
deleted file mode 100644
index 9c4bdc8..0000000
Binary files a/data/iceberg/lineitem_iceberg/data/.00000-411-0792dcfe-4e25-4ca3-8ada-175286069a47-00001.parquet.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/data/.00041-414-f3c73457-bbd6-4b92-9c15-17b241171b16-00001.parquet.crc b/data/iceberg/lineitem_iceberg/data/.00041-414-f3c73457-bbd6-4b92-9c15-17b241171b16-00001.parquet.crc
deleted file mode 100644
index d00535f..0000000
Binary files a/data/iceberg/lineitem_iceberg/data/.00041-414-f3c73457-bbd6-4b92-9c15-17b241171b16-00001.parquet.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/data/00000-411-0792dcfe-4e25-4ca3-8ada-175286069a47-00001.parquet b/data/iceberg/lineitem_iceberg/data/00000-411-0792dcfe-4e25-4ca3-8ada-175286069a47-00001.parquet
deleted file mode 100644
index 0042379..0000000
Binary files a/data/iceberg/lineitem_iceberg/data/00000-411-0792dcfe-4e25-4ca3-8ada-175286069a47-00001.parquet and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/data/00041-414-f3c73457-bbd6-4b92-9c15-17b241171b16-00001.parquet b/data/iceberg/lineitem_iceberg/data/00041-414-f3c73457-bbd6-4b92-9c15-17b241171b16-00001.parquet
deleted file mode 100644
index d254ec4..0000000
Binary files a/data/iceberg/lineitem_iceberg/data/00041-414-f3c73457-bbd6-4b92-9c15-17b241171b16-00001.parquet and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/.10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m0.avro.crc b/data/iceberg/lineitem_iceberg/metadata/.10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m0.avro.crc
deleted file mode 100644
index bf060e8..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/.10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m0.avro.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/.10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m1.avro.crc b/data/iceberg/lineitem_iceberg/metadata/.10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m1.avro.crc
deleted file mode 100644
index 671a7d1..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/.10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m1.avro.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/.cf3d0be5-cf70-453d-ad8f-48fdc412e608-m0.avro.crc b/data/iceberg/lineitem_iceberg/metadata/.cf3d0be5-cf70-453d-ad8f-48fdc412e608-m0.avro.crc
deleted file mode 100644
index e9dc69a..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/.cf3d0be5-cf70-453d-ad8f-48fdc412e608-m0.avro.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/.snap-3776207205136740581-1-cf3d0be5-cf70-453d-ad8f-48fdc412e608.avro.crc b/data/iceberg/lineitem_iceberg/metadata/.snap-3776207205136740581-1-cf3d0be5-cf70-453d-ad8f-48fdc412e608.avro.crc
deleted file mode 100644
index ed617a4..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/.snap-3776207205136740581-1-cf3d0be5-cf70-453d-ad8f-48fdc412e608.avro.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/.snap-7635660646343998149-1-10eaca8a-1e1c-421e-ad6d-b232e5ee23d3.avro.crc b/data/iceberg/lineitem_iceberg/metadata/.snap-7635660646343998149-1-10eaca8a-1e1c-421e-ad6d-b232e5ee23d3.avro.crc
deleted file mode 100644
index a74ad2f..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/.snap-7635660646343998149-1-10eaca8a-1e1c-421e-ad6d-b232e5ee23d3.avro.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/.v1.metadata.json.crc b/data/iceberg/lineitem_iceberg/metadata/.v1.metadata.json.crc
deleted file mode 100644
index af5bbe4..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/.v1.metadata.json.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/.v2.metadata.json.crc b/data/iceberg/lineitem_iceberg/metadata/.v2.metadata.json.crc
deleted file mode 100644
index 856dc24..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/.v2.metadata.json.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/.version-hint.text.crc b/data/iceberg/lineitem_iceberg/metadata/.version-hint.text.crc
deleted file mode 100644
index 2003120..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/.version-hint.text.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m0.avro b/data/iceberg/lineitem_iceberg/metadata/10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m0.avro
deleted file mode 100644
index 5cd8724..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m0.avro and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m1.avro b/data/iceberg/lineitem_iceberg/metadata/10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m1.avro
deleted file mode 100644
index 24aabbb..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/10eaca8a-1e1c-421e-ad6d-b232e5ee23d3-m1.avro and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/cf3d0be5-cf70-453d-ad8f-48fdc412e608-m0.avro b/data/iceberg/lineitem_iceberg/metadata/cf3d0be5-cf70-453d-ad8f-48fdc412e608-m0.avro
deleted file mode 100644
index 0322f43..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/cf3d0be5-cf70-453d-ad8f-48fdc412e608-m0.avro and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/snap-3776207205136740581-1-cf3d0be5-cf70-453d-ad8f-48fdc412e608.avro b/data/iceberg/lineitem_iceberg/metadata/snap-3776207205136740581-1-cf3d0be5-cf70-453d-ad8f-48fdc412e608.avro
deleted file mode 100644
index 42ac35b..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/snap-3776207205136740581-1-cf3d0be5-cf70-453d-ad8f-48fdc412e608.avro and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/snap-7635660646343998149-1-10eaca8a-1e1c-421e-ad6d-b232e5ee23d3.avro b/data/iceberg/lineitem_iceberg/metadata/snap-7635660646343998149-1-10eaca8a-1e1c-421e-ad6d-b232e5ee23d3.avro
deleted file mode 100644
index bfc7489..0000000
Binary files a/data/iceberg/lineitem_iceberg/metadata/snap-7635660646343998149-1-10eaca8a-1e1c-421e-ad6d-b232e5ee23d3.avro and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg/metadata/v1.metadata.json b/data/iceberg/lineitem_iceberg/metadata/v1.metadata.json
deleted file mode 100644
index 0439057..0000000
--- a/data/iceberg/lineitem_iceberg/metadata/v1.metadata.json
+++ /dev/null
@@ -1,142 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "a319422b-6f8c-44d0-90ba-96242d9a1d7b",
-  "location" : "./lineitem_iceberg",
-  "last-sequence-number" : 1,
-  "last-updated-ms" : 1676473674504,
-  "last-column-id" : 16,
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 4,
-      "name" : "l_linenumber",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 5,
-      "name" : "l_quantity",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice",
-      "required" : false,
-      "type" : "decimal(15, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_discount",
-      "required" : false,
-      "type" : "decimal(15, 2)"
-    }, {
-      "id" : 8,
-      "name" : "l_tax",
-      "required" : false,
-      "type" : "decimal(15, 2)"
-    }, {
-      "id" : 9,
-      "name" : "l_returnflag",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 10,
-      "name" : "l_linestatus",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 11,
-      "name" : "l_shipdate",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 13,
-      "name" : "l_receiptdate",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 14,
-      "name" : "l_shipinstruct",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_shipmode",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 16,
-      "name" : "l_comment",
-      "required" : false,
-      "type" : "string"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "root",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 3776207205136740581,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 3776207205136740581,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 3776207205136740581,
-    "timestamp-ms" : 1676473674504,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1676472783435",
-      "added-data-files" : "1",
-      "added-records" : "60175",
-      "added-files-size" : "1390176",
-      "changed-partition-count" : "1",
-      "total-records" : "60175",
-      "total-files-size" : "1390176",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "lineitem_iceberg/metadata/snap-3776207205136740581-1-cf3d0be5-cf70-453d-ad8f-48fdc412e608.avro",
-    "schema-id" : 0
-  } ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1676473674504,
-    "snapshot-id" : 3776207205136740581
-  } ],
-  "metadata-log" : [ ]
-}
\ No newline at end of file
diff --git a/data/iceberg/lineitem_iceberg/metadata/v2.metadata.json b/data/iceberg/lineitem_iceberg/metadata/v2.metadata.json
deleted file mode 100644
index ffd0d60..0000000
--- a/data/iceberg/lineitem_iceberg/metadata/v2.metadata.json
+++ /dev/null
@@ -1,172 +0,0 @@
-{
-  "format-version" : 2,
-  "table-uuid" : "a319422b-6f8c-44d0-90ba-96242d9a1d7b",
-  "location" : "./lineitem_iceberg",
-  "last-sequence-number" : 2,
-  "last-updated-ms" : 1676473694730,
-  "last-column-id" : 16,
-  "current-schema-id" : 0,
-  "schemas" : [ {
-    "type" : "struct",
-    "schema-id" : 0,
-    "fields" : [ {
-      "id" : 1,
-      "name" : "l_orderkey",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 2,
-      "name" : "l_partkey",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 3,
-      "name" : "l_suppkey",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 4,
-      "name" : "l_linenumber",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 5,
-      "name" : "l_quantity",
-      "required" : false,
-      "type" : "int"
-    }, {
-      "id" : 6,
-      "name" : "l_extendedprice",
-      "required" : false,
-      "type" : "decimal(15, 2)"
-    }, {
-      "id" : 7,
-      "name" : "l_discount",
-      "required" : false,
-      "type" : "decimal(15, 2)"
-    }, {
-      "id" : 8,
-      "name" : "l_tax",
-      "required" : false,
-      "type" : "decimal(15, 2)"
-    }, {
-      "id" : 9,
-      "name" : "l_returnflag",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 10,
-      "name" : "l_linestatus",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 11,
-      "name" : "l_shipdate",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 12,
-      "name" : "l_commitdate",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 13,
-      "name" : "l_receiptdate",
-      "required" : false,
-      "type" : "date"
-    }, {
-      "id" : 14,
-      "name" : "l_shipinstruct",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 15,
-      "name" : "l_shipmode",
-      "required" : false,
-      "type" : "string"
-    }, {
-      "id" : 16,
-      "name" : "l_comment",
-      "required" : false,
-      "type" : "string"
-    } ]
-  } ],
-  "default-spec-id" : 0,
-  "partition-specs" : [ {
-    "spec-id" : 0,
-    "fields" : [ ]
-  } ],
-  "last-partition-id" : 999,
-  "default-sort-order-id" : 0,
-  "sort-orders" : [ {
-    "order-id" : 0,
-    "fields" : [ ]
-  } ],
-  "properties" : {
-    "owner" : "root",
-    "write.update.mode" : "merge-on-read"
-  },
-  "current-snapshot-id" : 7635660646343998149,
-  "refs" : {
-    "main" : {
-      "snapshot-id" : 7635660646343998149,
-      "type" : "branch"
-    }
-  },
-  "snapshots" : [ {
-    "sequence-number" : 1,
-    "snapshot-id" : 3776207205136740581,
-    "timestamp-ms" : 1676473674504,
-    "summary" : {
-      "operation" : "append",
-      "spark.app.id" : "local-1676472783435",
-      "added-data-files" : "1",
-      "added-records" : "60175",
-      "added-files-size" : "1390176",
-      "changed-partition-count" : "1",
-      "total-records" : "60175",
-      "total-files-size" : "1390176",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "lineitem_iceberg/metadata/snap-3776207205136740581-1-cf3d0be5-cf70-453d-ad8f-48fdc412e608.avro",
-    "schema-id" : 0
-  }, {
-    "sequence-number" : 2,
-    "snapshot-id" : 7635660646343998149,
-    "parent-snapshot-id" : 3776207205136740581,
-    "timestamp-ms" : 1676473694730,
-    "summary" : {
-      "operation" : "overwrite",
-      "spark.app.id" : "local-1676472783435",
-      "added-data-files" : "1",
-      "deleted-data-files" : "1",
-      "added-records" : "51793",
-      "deleted-records" : "60175",
-      "added-files-size" : "1208539",
-      "removed-files-size" : "1390176",
-      "changed-partition-count" : "1",
-      "total-records" : "51793",
-      "total-files-size" : "1208539",
-      "total-data-files" : "1",
-      "total-delete-files" : "0",
-      "total-position-deletes" : "0",
-      "total-equality-deletes" : "0"
-    },
-    "manifest-list" : "lineitem_iceberg/metadata/snap-7635660646343998149-1-10eaca8a-1e1c-421e-ad6d-b232e5ee23d3.avro",
-    "schema-id" : 0
-  } ],
-  "snapshot-log" : [ {
-    "timestamp-ms" : 1676473674504,
-    "snapshot-id" : 3776207205136740581
-  }, {
-    "timestamp-ms" : 1676473694730,
-    "snapshot-id" : 7635660646343998149
-  } ],
-  "metadata-log" : [ {
-    "timestamp-ms" : 1676473674504,
-    "metadata-file" : "lineitem_iceberg/metadata/v1.metadata.json"
-  } ]
-}
\ No newline at end of file
diff --git a/data/iceberg/lineitem_iceberg/metadata/version-hint.text b/data/iceberg/lineitem_iceberg/metadata/version-hint.text
deleted file mode 100644
index d8263ee..0000000
--- a/data/iceberg/lineitem_iceberg/metadata/version-hint.text
+++ /dev/null
@@ -1 +0,0 @@
-2
\ No newline at end of file
diff --git a/data/iceberg/lineitem_iceberg_gz/data/.00000-2-371a340c-ded5-4e85-aa49-9c788d6f21cd-00001.parquet.crc b/data/iceberg/lineitem_iceberg_gz/data/.00000-2-371a340c-ded5-4e85-aa49-9c788d6f21cd-00001.parquet.crc
deleted file mode 100644
index f88d18d..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/data/.00000-2-371a340c-ded5-4e85-aa49-9c788d6f21cd-00001.parquet.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/data/00000-2-371a340c-ded5-4e85-aa49-9c788d6f21cd-00001.parquet b/data/iceberg/lineitem_iceberg_gz/data/00000-2-371a340c-ded5-4e85-aa49-9c788d6f21cd-00001.parquet
deleted file mode 100644
index 63b23d3..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/data/00000-2-371a340c-ded5-4e85-aa49-9c788d6f21cd-00001.parquet and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/metadata/.23f9dbea-1e7f-4694-a82c-dc3c9a94953e-m0.avro.crc b/data/iceberg/lineitem_iceberg_gz/metadata/.23f9dbea-1e7f-4694-a82c-dc3c9a94953e-m0.avro.crc
deleted file mode 100644
index c2fab3e..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/metadata/.23f9dbea-1e7f-4694-a82c-dc3c9a94953e-m0.avro.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/metadata/.snap-4468019210336628573-1-23f9dbea-1e7f-4694-a82c-dc3c9a94953e.avro.crc b/data/iceberg/lineitem_iceberg_gz/metadata/.snap-4468019210336628573-1-23f9dbea-1e7f-4694-a82c-dc3c9a94953e.avro.crc
deleted file mode 100644
index a27fd90..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/metadata/.snap-4468019210336628573-1-23f9dbea-1e7f-4694-a82c-dc3c9a94953e.avro.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/metadata/.v1.gz.metadata.json.crc b/data/iceberg/lineitem_iceberg_gz/metadata/.v1.gz.metadata.json.crc
deleted file mode 100644
index 48de20c..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/metadata/.v1.gz.metadata.json.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/metadata/.v2.gz.metadata.json.crc b/data/iceberg/lineitem_iceberg_gz/metadata/.v2.gz.metadata.json.crc
deleted file mode 100644
index 915a750..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/metadata/.v2.gz.metadata.json.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/metadata/.version-hint.text.crc b/data/iceberg/lineitem_iceberg_gz/metadata/.version-hint.text.crc
deleted file mode 100644
index 2003120..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/metadata/.version-hint.text.crc and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/metadata/23f9dbea-1e7f-4694-a82c-dc3c9a94953e-m0.avro b/data/iceberg/lineitem_iceberg_gz/metadata/23f9dbea-1e7f-4694-a82c-dc3c9a94953e-m0.avro
deleted file mode 100644
index 8425c93..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/metadata/23f9dbea-1e7f-4694-a82c-dc3c9a94953e-m0.avro and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/metadata/snap-4468019210336628573-1-23f9dbea-1e7f-4694-a82c-dc3c9a94953e.avro b/data/iceberg/lineitem_iceberg_gz/metadata/snap-4468019210336628573-1-23f9dbea-1e7f-4694-a82c-dc3c9a94953e.avro
deleted file mode 100644
index 9e26d01..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/metadata/snap-4468019210336628573-1-23f9dbea-1e7f-4694-a82c-dc3c9a94953e.avro and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/metadata/v1.gz.metadata.json b/data/iceberg/lineitem_iceberg_gz/metadata/v1.gz.metadata.json
deleted file mode 100644
index efc8150..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/metadata/v1.gz.metadata.json and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/metadata/v2.gz.metadata.json b/data/iceberg/lineitem_iceberg_gz/metadata/v2.gz.metadata.json
deleted file mode 100644
index a48a7c9..0000000
Binary files a/data/iceberg/lineitem_iceberg_gz/metadata/v2.gz.metadata.json and /dev/null differ
diff --git a/data/iceberg/lineitem_iceberg_gz/metadata/version-hint.text b/data/iceberg/lineitem_iceberg_gz/metadata/version-hint.text
deleted file mode 100644
index d8263ee..0000000
--- a/data/iceberg/lineitem_iceberg_gz/metadata/version-hint.text
+++ /dev/null
@@ -1 +0,0 @@
-2
\ No newline at end of file
diff --git a/scripts/test_data_generator/generate_base_parquet.py b/scripts/test_data_generator/generate_base_parquet.py
index 55aad73..87f2837 100755
--- a/scripts/test_data_generator/generate_base_parquet.py
+++ b/scripts/test_data_generator/generate_base_parquet.py
@@ -46,7 +46,8 @@
                     l_commitdate::TIMESTAMPTZ as l_commitdate_timestamp_tz,
                     l_comment as l_comment_string,
                     gen_random_uuid()::VARCHAR as uuid,
-                    l_comment::BLOB as l_comment_blob
+                    l_comment::BLOB as l_comment_blob,
+                    l_shipmode as l_shipmode_string
                     FROM
                     lineitem;""");
 elif (MODE.lower() == "default"):
@@ -67,7 +68,8 @@
                     l_commitdate::TIMESTAMPTZ as l_commitdate_timestamp_tz,
                     l_comment as l_comment_string,
                     gen_random_uuid()::UUID as uuid,
-                    l_comment::BLOB as l_comment_blob
+                    l_comment::BLOB as l_comment_blob,
+                    l_shipmode as l_shipmode_string
                     FROM
                     lineitem;""");
 else:
diff --git a/scripts/test_data_generator/generate_iceberg.py b/scripts/test_data_generator/generate_iceberg.py
index 5f6afe7..5c09ae6 100755
--- a/scripts/test_data_generator/generate_iceberg.py
+++ b/scripts/test_data_generator/generate_iceberg.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/Users/mritchie712/opt/anaconda3/bin/python
 import pyspark
 import pyspark.sql
 import sys
@@ -8,7 +8,7 @@
 from pathlib import Path
 
 if (len(sys.argv) != 4 ):
-    print("Usage: generate_iceberg.py <SCALE_FACTOR> <DEST_PATH> <ICBERG_SPEC_VERSION>")
+    print("Usage: generate_iceberg.py <SCALE_FACTOR> <DEST_PATH> <ICEBERG_SPEC_VERSION>")
     exit(1)
 
 SCALE = sys.argv[1]
@@ -16,7 +16,7 @@
 ICEBERG_SPEC_VERSION = sys.argv[3]
 
 PARQUET_SRC_FILE = f'{DEST_PATH}/base_file/file.parquet'
-TABLE_NAME = "iceberg_catalog.pyspark_iceberg_table";
+TABLE_NAME = "iceberg_catalog.pyspark_iceberg_table"
 CWD = os.getcwd()
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 
@@ -26,7 +26,7 @@
 os.system(f"python3 {SCRIPT_DIR}/generate_base_parquet.py {SCALE} {CWD}/{DEST_PATH} spark")
 
 ###
-### Configure everyone's favorite apache product
+### Configure Spark with Iceberg
 ###
 conf = pyspark.SparkConf()
 conf.setMaster('local[*]')
@@ -42,23 +42,46 @@
 sc.setLogLevel("ERROR")
 
 ###
-### Create Iceberg table from dataset
+### Create Iceberg table from dataset with partitioning
 ###
-spark.read.parquet(PARQUET_SRC_FILE).createOrReplaceTempView('parquet_file_view');
+spark.read.parquet(PARQUET_SRC_FILE).createOrReplaceTempView('parquet_file_view')
+
+# Define your partition columns and transforms
+partition_spec = "year(l_shipdate_date), l_shipmode_string"  # Adjust 'l_shipdate_date' as needed
 
 if ICEBERG_SPEC_VERSION == '1':
-    spark.sql(f"CREATE or REPLACE TABLE {TABLE_NAME} TBLPROPERTIES ('format-version'='{ICEBERG_SPEC_VERSION}') AS SELECT * FROM parquet_file_view");
+    create_table_sql = f"""
+        CREATE OR REPLACE TABLE {TABLE_NAME}
+        USING iceberg
+        PARTITIONED BY ({partition_spec})
+        TBLPROPERTIES (
+            'format-version' = '{ICEBERG_SPEC_VERSION}'
+        )
+        AS SELECT * FROM parquet_file_view
+    """
 elif ICEBERG_SPEC_VERSION == '2':
-    spark.sql(f"CREATE or REPLACE TABLE {TABLE_NAME} TBLPROPERTIES ('format-version'='{ICEBERG_SPEC_VERSION}', 'write.update.mode'='merge-on-read') AS SELECT * FROM parquet_file_view");
+    create_table_sql = f"""
+        CREATE OR REPLACE TABLE {TABLE_NAME}
+        USING iceberg
+        PARTITIONED BY ({partition_spec})
+        TBLPROPERTIES (
+            'format-version' = '{ICEBERG_SPEC_VERSION}',
+            'write.update.mode' = 'merge-on-read'
+        )
+        AS SELECT * FROM parquet_file_view
+    """
 else:
     print(f"Are you from the future? Iceberg spec version '{ICEBERG_SPEC_VERSION}' is unbeknownst to me")
     exit(1)
 
+# Execute the CREATE TABLE statement
+spark.sql(create_table_sql)
+
 ###
 ### Apply modifications to base table generating verification results between each step
 ###
 update_files = [str(path) for path in Path(f'{SCRIPT_DIR}/updates_v{ICEBERG_SPEC_VERSION}').rglob('*.sql')]
-update_files.sort() # Order matters obviously
+update_files.sort()  # Order matters obviously
 last_file = ""
 
 for path in update_files:
@@ -82,7 +105,7 @@
 
         # Create copy of table
         df = spark.read.table(TABLE_NAME)
-        df.write.parquet(f"{DEST_PATH}/expected_results/{file_trimmed}/data");
+        df.write.parquet(f"{DEST_PATH}/expected_results/{file_trimmed}/data")
 
         # For documentation, also write the query we executed to the data
         query_path = f'{DEST_PATH}/expected_results/{file_trimmed}/query.sql'
@@ -90,9 +113,8 @@
             f.write("-- The query executed at this step:\n")
             f.write(query)
 
-
 ###
 ### Finally, we copy the latest results to a "final" dir for easy test writing
 ###
 import shutil
-shutil.copytree(f"{DEST_PATH}/expected_results/{last_file}", f"{DEST_PATH}/expected_results/last")
+shutil.copytree(f"{DEST_PATH}/expected_results/{last_file}", f"{DEST_PATH}/expected_results/last")
\ No newline at end of file
diff --git a/src/common/iceberg.cpp b/src/common/iceberg.cpp
index 6e63969..784583b 100644
--- a/src/common/iceberg.cpp
+++ b/src/common/iceberg.cpp
@@ -80,8 +80,8 @@ vector<IcebergManifestEntry> IcebergTable::ReadManifestEntries(const string &pat
 		}
 	} else {
 		auto schema = avro::compileJsonSchemaFromString(MANIFEST_ENTRY_SCHEMA);
-		avro::DataFileReader<c::manifest_entry> dfr(std::move(stream), schema);
-		c::manifest_entry manifest_entry;
+		avro::DataFileReader<manifest_entry> dfr(std::move(stream), schema);
+		manifest_entry manifest_entry;
 		while (dfr.read(manifest_entry)) {
 			ret.emplace_back(IcebergManifestEntry(manifest_entry));
 		}
diff --git a/src/iceberg_functions/iceberg_scan.cpp b/src/iceberg_functions/iceberg_scan.cpp
index 2b3ec86..c20d027 100644
--- a/src/iceberg_functions/iceberg_scan.cpp
+++ b/src/iceberg_functions/iceberg_scan.cpp
@@ -1,31 +1,33 @@
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
 #include "duckdb/common/enums/join_type.hpp"
-#include "duckdb/parser/query_node/select_node.hpp"
-#include "duckdb/parser/tableref/joinref.hpp"
 #include "duckdb/common/enums/joinref_type.hpp"
 #include "duckdb/common/enums/tableref_type.hpp"
-#include "duckdb/parser/tableref/table_function_ref.hpp"
-#include "duckdb/parser/query_node/recursive_cte_node.hpp"
+#include "duckdb/common/file_opener.hpp"
+#include "duckdb/common/file_system.hpp"
+#include "duckdb/common/printer.hpp"
+#include "duckdb/parser/expression/comparison_expression.hpp"
+#include "duckdb/parser/expression/conjunction_expression.hpp"
 #include "duckdb/parser/expression/constant_expression.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
-#include "duckdb/parser/expression/conjunction_expression.hpp"
-#include "duckdb/planner/expression/bound_reference_expression.hpp"
-#include "duckdb/parser/expression/comparison_expression.hpp"
 #include "duckdb/parser/expression/star_expression.hpp"
-#include "duckdb/parser/tableref/subqueryref.hpp"
+#include "duckdb/parser/query_node/recursive_cte_node.hpp"
+#include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/tableref/emptytableref.hpp"
-#include "duckdb/planner/operator/logical_get.hpp"
+#include "duckdb/parser/tableref/joinref.hpp"
+#include "duckdb/parser/tableref/subqueryref.hpp"
+#include "duckdb/parser/tableref/table_function_ref.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
-#include "duckdb/common/file_opener.hpp"
-#include "duckdb/common/file_system.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
+#include "iceberg_functions.hpp"
 #include "iceberg_metadata.hpp"
 #include "iceberg_utils.hpp"
-#include "iceberg_functions.hpp"
 #include "yyjson.hpp"
 
-#include <string>
+#include <iostream>
 #include <numeric>
+#include <string>
 
 namespace duckdb {
 
@@ -36,6 +38,34 @@ struct IcebergScanGlobalTableFunctionState : public GlobalTableFunctionState {
 	}
 };
 
+// === Derived TableFunctionInfo to hold constraints ===
+struct IcebergTableFunctionInfo : public TableFunctionInfo {
+	vector<unique_ptr<ParsedExpression>> constraints;
+
+	IcebergTableFunctionInfo(vector<unique_ptr<ParsedExpression>> &&constraints_p)
+	    : constraints(std::move(constraints_p)) {
+	}
+};
+
+// === Helper function to recursively extract comparison predicates from expressions ===
+static void ExtractPredicates(ParsedExpression &expr, vector<unique_ptr<ParsedExpression>> &predicates) {
+	if (expr.type == ExpressionType::CONJUNCTION_AND) {
+		auto &conj = (ConjunctionExpression &)expr;
+		// Access children instead of left and right
+		if (conj.children.size() >= 2) {
+			ExtractPredicates(*conj.children[0], predicates);
+			ExtractPredicates(*conj.children[1], predicates);
+		}
+	} else if (expr.type == ExpressionType::COMPARE_EQUAL || expr.type == ExpressionType::COMPARE_GREATERTHAN ||
+	           expr.type == ExpressionType::COMPARE_GREATERTHANOREQUALTO ||
+	           expr.type == ExpressionType::COMPARE_LESSTHAN ||
+	           expr.type == ExpressionType::COMPARE_LESSTHANOREQUALTO) {
+		// Clone the expression and add to predicates
+		predicates.emplace_back(expr.Copy());
+	}
+	// Add more conditions here if you want to handle OR or other expressions
+}
+
 static unique_ptr<ParsedExpression> GetFilenameExpr(unique_ptr<ColumnRefExpression> colref_expr) {
 	vector<unique_ptr<ParsedExpression>> split_children;
 	split_children.push_back(std::move(colref_expr));
@@ -105,50 +135,302 @@ static Value GetParquetSchemaParam(vector<IcebergColumnDefinition> &schema) {
 
 	for (auto &schema_entry : schema) {
 		child_list_t<Value> map_value_children;
-		map_value_children.push_back(make_pair("name", Value(schema_entry.name)));
-		map_value_children.push_back(make_pair("type", Value(schema_entry.type.ToString())));
-		map_value_children.push_back(make_pair("default_value", schema_entry.default_value));
+		map_value_children.emplace_back(make_pair("name", Value(schema_entry.name)));
+		map_value_children.emplace_back(make_pair("type", Value(schema_entry.type.ToString())));
+		map_value_children.emplace_back(make_pair("default_value", schema_entry.default_value));
 		auto map_value = Value::STRUCT(map_value_children);
 
 		child_list_t<Value> map_entry_children;
-		map_entry_children.push_back(make_pair("key", schema_entry.id));
-		map_entry_children.push_back(make_pair("values", map_value));
+		map_entry_children.emplace_back(make_pair("key", schema_entry.id));
+		map_entry_children.emplace_back(make_pair("values", map_value)); // Updated key to "values"
 		auto map_entry = Value::STRUCT(map_entry_children);
 
 		map_entries.push_back(map_entry);
 	}
 
-	auto param_type =
-	    LogicalType::STRUCT({{"key", LogicalType::INTEGER},
-	                         {"value", LogicalType::STRUCT({{{"name", LogicalType::VARCHAR},
-	                                                         {"type", LogicalType::VARCHAR},
-	                                                         {"default_value", LogicalType::VARCHAR}}})}});
+	auto param_type = LogicalType::STRUCT({{"key", LogicalType::INTEGER},
+	                                       {"value", LogicalType::STRUCT({{"name", LogicalType::VARCHAR},
+	                                                                      {"type", LogicalType::VARCHAR},
+	                                                                      {"default_value", LogicalType::VARCHAR}})}});
 	auto ret = Value::MAP(param_type, map_entries);
 	return ret;
 }
 
+// Utility function to convert byte vector to hex string for logging
+static std::string ByteArrayToHexString(const std::vector<uint8_t> &bytes) {
+	std::ostringstream oss;
+	for (auto byte : bytes) {
+		oss << std::hex << std::setw(2) << std::setfill('0') << (int)byte;
+	}
+	return oss.str();
+}
+
+static Value DeserializeBound(const std::vector<uint8_t> &bound_value, const LogicalType &type) {
+	Value deserialized_value;
+	try {
+		switch (type.id()) {
+		case LogicalTypeId::INTEGER: {
+			if (bound_value.size() < sizeof(int32_t)) {
+				throw std::runtime_error("Invalid bound size for INTEGER type");
+			}
+			int32_t val;
+			std::memcpy(&val, bound_value.data(), sizeof(int32_t));
+			deserialized_value = Value::INTEGER(val);
+			break;
+		}
+		case LogicalTypeId::BIGINT: {
+			if (bound_value.size() < sizeof(int64_t)) {
+				throw std::runtime_error("Invalid bound size for BIGINT type");
+			}
+			int64_t val;
+			std::memcpy(&val, bound_value.data(), sizeof(int64_t));
+			deserialized_value = Value::BIGINT(val);
+			break;
+		}
+		case LogicalTypeId::DATE: {
+			if (bound_value.size() < sizeof(int32_t)) { // Dates are typically stored as int32 (days since epoch)
+				throw std::runtime_error("Invalid bound size for DATE type");
+			}
+			int32_t days_since_epoch;
+			std::memcpy(&days_since_epoch, bound_value.data(), sizeof(int32_t));
+			// Convert to DuckDB date
+			date_t date = Date::EpochDaysToDate(days_since_epoch);
+			deserialized_value = Value::DATE(date);
+			break;
+		}
+		case LogicalTypeId::TIMESTAMP: {
+			if (bound_value.size() <
+			    sizeof(int64_t)) { // Timestamps are typically stored as int64 (microseconds since epoch)
+				throw std::runtime_error("Invalid bound size for TIMESTAMP type");
+			}
+			int64_t micros_since_epoch;
+			std::memcpy(&micros_since_epoch, bound_value.data(), sizeof(int64_t));
+			// Convert to DuckDB timestamp using microseconds
+			timestamp_t timestamp = Timestamp::FromEpochMicroSeconds(micros_since_epoch);
+			deserialized_value = Value::TIMESTAMP(timestamp);
+			break;
+		}
+		case LogicalTypeId::TIMESTAMP_TZ: {             // Added support for TIMESTAMP WITH TIME ZONE
+			if (bound_value.size() < sizeof(int64_t)) { // Assuming stored as int64 (microseconds since epoch)
+				throw std::runtime_error("Invalid bound size for TIMESTAMP_TZ type");
+			}
+			int64_t micros_since_epoch;
+			std::memcpy(&micros_since_epoch, bound_value.data(), sizeof(int64_t));
+			// Convert to DuckDB timestamp using microseconds
+			timestamp_t timestamp = Timestamp::FromEpochMicroSeconds(micros_since_epoch);
+			// Create a TIMESTAMPTZ Value
+			deserialized_value = Value::TIMESTAMPTZ(timestamp);
+			break;
+		}
+		case LogicalTypeId::DOUBLE: {
+			if (bound_value.size() < sizeof(double)) {
+				throw std::runtime_error("Invalid bound size for DOUBLE type");
+			}
+			double val;
+			std::memcpy(&val, bound_value.data(), sizeof(double));
+			deserialized_value = Value::DOUBLE(val);
+			break;
+		}
+		case LogicalTypeId::VARCHAR: {
+			// Assume the bytes represent a UTF-8 string
+			std::string str(bound_value.begin(), bound_value.end());
+			deserialized_value = Value(str);
+			break;
+		}
+		// Add more types as needed
+		default:
+			throw std::runtime_error("Unsupported type for DeserializeBound");
+		}
+
+		// Log the final deserialized value
+	} catch (const std::exception &e) {
+		std::cout << "    Error during deserialization: " << e.what() << std::endl;
+		// Depending on your error handling strategy, you might want to rethrow or handle it here
+		throw;
+	}
+
+	return deserialized_value;
+}
+
+static bool EvaluatePredicateAgainstStatistics(const IcebergManifestEntry &entry,
+                                               const vector<unique_ptr<ParsedExpression>> &predicates,
+                                               const std::vector<IcebergColumnDefinition> &schema) {
+	// Create a mapping from column names to field IDs and their LogicalTypes
+	std::unordered_map<std::string, std::pair<int, LogicalType>> column_to_field_info;
+	for (const auto &col_def : schema) {
+		column_to_field_info[col_def.name] = {col_def.id, col_def.type}; // Assuming col_def.type is LogicalType
+	}
+
+	for (const auto &predicate : predicates) {
+		if (auto comparison = dynamic_cast<ComparisonExpression *>(predicate.get())) {
+			// Assume predicates are on columns, possibly transformed
+			std::string column_name;
+			if (auto colref = dynamic_cast<ColumnRefExpression *>(comparison->left.get())) {
+				column_name = colref->GetColumnName();
+			} else {
+				// Unsupported predicate structure
+				continue;
+			}
+
+			// Retrieve field ID and type
+			auto it = column_to_field_info.find(column_name);
+			if (it == column_to_field_info.end()) {
+				// Column not found in schema, cannot evaluate predicate
+				continue;
+			}
+			int field_id = it->second.first;
+			LogicalType field_type = it->second.second;
+
+			// Convert field_id to string for lookup
+			std::string field_id_str = std::to_string(field_id);
+
+			// Get lower and upper bounds
+			auto lower_it = entry.lower_bounds.find(field_id_str);
+			auto upper_it = entry.upper_bounds.find(field_id_str);
+
+			if (lower_it == entry.lower_bounds.end() || upper_it == entry.upper_bounds.end()) {
+				continue; // Cannot filter based on missing bounds
+			}
+
+			// Deserialize bounds
+			Value lower_bound, upper_bound;
+			try {
+				lower_bound = DeserializeBound(lower_it->second, field_type);
+				upper_bound = DeserializeBound(upper_it->second, field_type);
+			} catch (const std::exception &e) {
+				continue;
+			}
+
+			// Extract the constant value from the predicate
+			Value constant_value;
+			if (auto const_expr = dynamic_cast<ConstantExpression *>(comparison->right.get())) {
+				constant_value = const_expr->value;
+			} else {
+				// Unsupported predicate structure
+				continue;
+			}
+			// fprintf(stderr, "    Lower bound: %s\n", lower_bound.ToString().c_str());
+			// fprintf(stderr, "    Upper bound: %s\n", upper_bound.ToString().c_str());
+
+			// Evaluate the predicate against the bounds
+			bool result = true;
+			switch (comparison->type) {
+			case ExpressionType::COMPARE_EQUAL:
+				result = (constant_value >= lower_bound && constant_value <= upper_bound);
+				break;
+			case ExpressionType::COMPARE_GREATERTHAN:
+				result = (constant_value <= upper_bound);
+				break;
+			case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
+				result = (constant_value <= upper_bound);
+				break;
+			case ExpressionType::COMPARE_LESSTHAN:
+				result = (constant_value >= lower_bound);
+				break;
+			case ExpressionType::COMPARE_LESSTHANOREQUALTO:
+				result = (constant_value >= lower_bound);
+				break;
+			default:
+				// For other types of comparisons, we can't make a decision based on bounds
+				result = true; // Conservative approach
+				break;
+			}
+			if (!result) {
+				return false; // If any predicate fails, exclude the file
+			}
+		}
+	}
+	return true; // All predicates passed
+}
+
 //! Build the Parquet Scan expression for the files we need to scan
-static unique_ptr<TableRef> MakeScanExpression(vector<Value> &data_file_values, vector<Value> &delete_file_values,
-                                               vector<IcebergColumnDefinition> &schema, bool allow_moved_paths,
-                                               string metadata_compression_codec, bool skip_schema_inference,
-                                               int64_t data_cardinality, int64_t delete_cardinality) {
-    
-	auto cardinality = make_uniq<ComparisonExpression>(ExpressionType::COMPARE_EQUAL, make_uniq<ColumnRefExpression>("explicit_cardinality"),
-	                                                                                  make_uniq<ConstantExpression>(Value(data_cardinality)));
-
-	// No deletes, just return a TableFunctionRef for a parquet scan of the data files
+static unique_ptr<TableRef>
+MakeScanExpression(const string &iceberg_path, FileSystem &fs, vector<IcebergManifestEntry> &data_file_entries,
+                   vector<Value> &delete_file_values, vector<IcebergColumnDefinition> &schema, bool allow_moved_paths,
+                   string metadata_compression_codec, bool skip_schema_inference, const IcebergTable &iceberg_table,
+                   int64_t data_cardinality, int64_t delete_cardinality,
+                   const IcebergTableFunctionInfo *iceberg_info = nullptr) {
+
+	// Filter data files based on predicates
+	vector<Value> filtered_data_file_values;
+	if (iceberg_info && !iceberg_info->constraints.empty()) {
+		for (const auto &entry : data_file_entries) {
+			if (EvaluatePredicateAgainstStatistics(entry, iceberg_info->constraints, schema)) {
+				auto full_path =
+				    allow_moved_paths ? IcebergUtils::GetFullPath(iceberg_path, entry.file_path, fs) : entry.file_path;
+				filtered_data_file_values.emplace_back(full_path);
+			}
+		}
+	} else {
+		for (const auto &entry : data_file_entries) {
+			auto full_path =
+			    allow_moved_paths ? IcebergUtils::GetFullPath(iceberg_path, entry.file_path, fs) : entry.file_path;
+			filtered_data_file_values.emplace_back(full_path);
+		}
+	}
+
+	// fprintf(stderr, "Total number of data files: %zu\n", data_file_entries.size());
+	// fprintf(stderr, "Total number of filtered data files: %zu\n", filtered_data_file_values.size());
+	// fprintf(stderr, "Total number of delete files: %zu\n", delete_file_values.size());
+
+	auto cardinality = make_uniq<ComparisonExpression>(ExpressionType::COMPARE_EQUAL,
+	                                                   make_uniq<ColumnRefExpression>("explicit_cardinality"),
+	                                                   make_uniq<ConstantExpression>(Value(data_cardinality)));
+
+	// Handle the scenario with no data files
+	if (filtered_data_file_values.empty()) {
+		// **BEGIN: Handling Empty Filtered Data Files**
+		auto select_node = make_uniq<SelectNode>();
+		select_node->where_clause = make_uniq<ConstantExpression>(Value::BOOLEAN(false));
+
+		// Add select expressions for each column based on the schema
+		for (const auto &col : schema) {
+			// Create a NULL constant of the appropriate type
+			auto null_expr = make_uniq<ConstantExpression>(Value(col.type));
+			// Alias it to the column name
+			null_expr->alias = col.name;
+			select_node->select_list.emplace_back(std::move(null_expr));
+		}
+
+		// **Add the FROM clause as EmptyTableRef**
+		select_node->from_table = make_uniq<EmptyTableRef>();
+
+		// Create a SelectStatement
+		auto select_statement = make_uniq<SelectStatement>();
+		select_statement->node = std::move(select_node);
+
+		// Create a SubqueryRef with the SelectStatement
+		auto table_ref_empty = make_uniq<SubqueryRef>(std::move(select_statement), "empty_scan");
+
+		return std::move(table_ref_empty);
+		// **END: Handling Empty Filtered Data Files**
+	}
+
+	// Handle the scenario with no delete files
 	if (delete_file_values.empty()) {
 		auto table_function_ref_data = make_uniq<TableFunctionRef>();
 		table_function_ref_data->alias = "iceberg_scan_data";
 		vector<unique_ptr<ParsedExpression>> left_children;
-		left_children.push_back(make_uniq<ConstantExpression>(Value::LIST(data_file_values)));
+		left_children.emplace_back(make_uniq<ConstantExpression>(Value::LIST(filtered_data_file_values)));
 		left_children.push_back(std::move(cardinality));
-		if (!skip_schema_inference) {
-			left_children.push_back(
-					make_uniq<ComparisonExpression>(ExpressionType::COMPARE_EQUAL, make_uniq<ColumnRefExpression>("schema"),
-					make_uniq<ConstantExpression>(GetParquetSchemaParam(schema))));
+
+		// Add cardinality condition if available
+		int64_t data_cardinality = 0;
+		for (const auto &entry : data_file_entries) {
+			if (entry.status != IcebergManifestEntryStatusType::DELETED &&
+			    entry.content == IcebergManifestEntryContentType::DATA) {
+				data_cardinality += entry.record_count;
+			}
 		}
+		left_children.emplace_back(make_uniq<ComparisonExpression>(
+		    ExpressionType::COMPARE_EQUAL, make_uniq<ColumnRefExpression>("explicit_cardinality"),
+		    make_uniq<ConstantExpression>(Value(data_cardinality))));
 
+		if (!skip_schema_inference) {
+			left_children.emplace_back(
+			    make_uniq<ComparisonExpression>(ExpressionType::COMPARE_EQUAL, make_uniq<ColumnRefExpression>("schema"),
+			                                    make_uniq<ConstantExpression>(GetParquetSchemaParam(schema))));
+		}
 		table_function_ref_data->function = make_uniq<FunctionExpression>("parquet_scan", std::move(left_children));
 		return std::move(table_function_ref_data);
 	}
@@ -172,7 +454,7 @@ static unique_ptr<TableRef> MakeScanExpression(vector<Value> &data_file_values,
 	auto table_function_ref_data = make_uniq<TableFunctionRef>();
 	table_function_ref_data->alias = "iceberg_scan_data";
 	vector<unique_ptr<ParsedExpression>> left_children;
-	left_children.push_back(make_uniq<ConstantExpression>(Value::LIST(data_file_values)));
+	left_children.push_back(make_uniq<ConstantExpression>(Value::LIST(filtered_data_file_values)));
 	left_children.push_back(std::move(cardinality));
 	left_children.push_back(make_uniq<ComparisonExpression>(ExpressionType::COMPARE_EQUAL,
 	                                                        make_uniq<ColumnRefExpression>("filename"),
@@ -181,9 +463,9 @@ static unique_ptr<TableRef> MakeScanExpression(vector<Value> &data_file_values,
 	                                                        make_uniq<ColumnRefExpression>("file_row_number"),
 	                                                        make_uniq<ConstantExpression>(Value(1))));
 	if (!skip_schema_inference) {
-		left_children.push_back(
-			make_uniq<ComparisonExpression>(ExpressionType::COMPARE_EQUAL, make_uniq<ColumnRefExpression>("schema"),
-			make_uniq<ConstantExpression>(GetParquetSchemaParam(schema))));
+		left_children.emplace_back(
+		    make_uniq<ComparisonExpression>(ExpressionType::COMPARE_EQUAL, make_uniq<ColumnRefExpression>("schema"),
+		                                    make_uniq<ConstantExpression>(GetParquetSchemaParam(schema))));
 	}
 	table_function_ref_data->function = make_uniq<FunctionExpression>("parquet_scan", std::move(left_children));
 	join_node->left = std::move(table_function_ref_data);
@@ -192,9 +474,10 @@ static unique_ptr<TableRef> MakeScanExpression(vector<Value> &data_file_values,
 	auto table_function_ref_deletes = make_uniq<TableFunctionRef>();
 	table_function_ref_deletes->alias = "iceberg_scan_deletes";
 	vector<unique_ptr<ParsedExpression>> right_children;
-	right_children.push_back(make_uniq<ConstantExpression>(Value::LIST(delete_file_values)));
-	right_children.push_back(make_uniq<ComparisonExpression>(ExpressionType::COMPARE_EQUAL, make_uniq<ColumnRefExpression>("explicit_cardinality"),
-	                                                                                        make_uniq<ConstantExpression>(Value(delete_cardinality))));
+	right_children.emplace_back(make_uniq<ConstantExpression>(Value::LIST(delete_file_values)));
+	right_children.emplace_back(make_uniq<ComparisonExpression>(
+	    ExpressionType::COMPARE_EQUAL, make_uniq<ColumnRefExpression>("explicit_cardinality"),
+	    make_uniq<ConstantExpression>(Value(delete_cardinality))));
 	table_function_ref_deletes->function = make_uniq<FunctionExpression>("parquet_scan", std::move(right_children));
 	join_node->right = std::move(table_function_ref_deletes);
 
@@ -207,10 +490,11 @@ static unique_ptr<TableRef> MakeScanExpression(vector<Value> &data_file_values,
 	auto select_expr = make_uniq<StarExpression>();
 	select_expr->exclude_list = {"filename", "file_row_number"};
 	vector<unique_ptr<ParsedExpression>> select_exprs;
-	select_exprs.push_back(std::move(select_expr));
+	select_exprs.emplace_back(std::move(select_expr));
 	select_node->select_list = std::move(select_exprs);
 	select_statement->node = std::move(select_node);
 
+	// fprintf(stderr, "Final SQL statement:\n%s\n", select_statement->ToString().c_str());
 	return make_uniq<SubqueryRef>(std::move(select_statement), "iceberg_scan");
 }
 
@@ -219,7 +503,7 @@ static unique_ptr<TableRef> IcebergScanBindReplace(ClientContext &context, Table
 	auto iceberg_path = input.inputs[0].ToString();
 
 	// Enabling this will ensure the ANTI Join with the deletes only looks at filenames, instead of full paths
-	// this allows hive tables to be moved and have mismatching paths, usefull for testing, but will have worse
+	// this allows hive tables to be moved and have mismatching paths, useful for testing, but will have worse
 	// performance
 	bool allow_moved_paths = false;
 	bool skip_schema_inference = false;
@@ -248,41 +532,73 @@ static unique_ptr<TableRef> IcebergScanBindReplace(ClientContext &context, Table
 			version_name_format = StringValue::Get(kv.second);
 		}
 	}
-	auto iceberg_meta_path = IcebergSnapshot::GetMetaDataPath(iceberg_path, fs, metadata_compression_codec, table_version, version_name_format);
+	auto iceberg_meta_path = IcebergSnapshot::GetMetaDataPath(iceberg_path, fs, metadata_compression_codec,
+	                                                          table_version, version_name_format);
+
 	IcebergSnapshot snapshot_to_scan;
 	if (input.inputs.size() > 1) {
 		if (input.inputs[1].type() == LogicalType::UBIGINT) {
-			snapshot_to_scan = IcebergSnapshot::GetSnapshotById(iceberg_meta_path, fs, input.inputs[1].GetValue<uint64_t>(), metadata_compression_codec, skip_schema_inference);
+			snapshot_to_scan =
+			    IcebergSnapshot::GetSnapshotById(iceberg_meta_path, fs, input.inputs[1].GetValue<uint64_t>(),
+			                                     metadata_compression_codec, skip_schema_inference);
 		} else if (input.inputs[1].type() == LogicalType::TIMESTAMP) {
 			snapshot_to_scan =
-			    IcebergSnapshot::GetSnapshotByTimestamp(iceberg_meta_path, fs, input.inputs[1].GetValue<timestamp_t>(), metadata_compression_codec, skip_schema_inference);
+			    IcebergSnapshot::GetSnapshotByTimestamp(iceberg_meta_path, fs, input.inputs[1].GetValue<timestamp_t>(),
+			                                            metadata_compression_codec, skip_schema_inference);
 		} else {
 			throw InvalidInputException("Unknown argument type in IcebergScanBindReplace.");
 		}
 	} else {
-		snapshot_to_scan = IcebergSnapshot::GetLatestSnapshot(iceberg_meta_path, fs, metadata_compression_codec, skip_schema_inference);
+		snapshot_to_scan = IcebergSnapshot::GetLatestSnapshot(iceberg_meta_path, fs, metadata_compression_codec,
+		                                                      skip_schema_inference);
 	}
 
-	IcebergTable iceberg_table = IcebergTable::Load(iceberg_path, snapshot_to_scan, fs, allow_moved_paths, metadata_compression_codec);
-	auto data_files = iceberg_table.GetPaths<IcebergManifestContentType::DATA>();
+	IcebergTable iceberg_table =
+	    IcebergTable::Load(iceberg_path, snapshot_to_scan, fs, allow_moved_paths, metadata_compression_codec);
+
+	auto data_entries = iceberg_table.GetEntries<IcebergManifestContentType::DATA>();
 	auto delete_files = iceberg_table.GetPaths<IcebergManifestContentType::DELETE>();
 	vector<Value> data_file_values;
-	for (auto &data_file : data_files) {
-		data_file_values.push_back(
-		    {allow_moved_paths ? IcebergUtils::GetFullPath(iceberg_path, data_file, fs) : data_file});
-	}
+
 	vector<Value> delete_file_values;
 	for (auto &delete_file : delete_files) {
-		delete_file_values.push_back(
-		    {allow_moved_paths ? IcebergUtils::GetFullPath(iceberg_path, delete_file, fs) : delete_file});
+		auto full_path = allow_moved_paths ? IcebergUtils::GetFullPath(iceberg_path, delete_file, fs) : delete_file;
+		delete_file_values.emplace_back(full_path);
 	}
 
+	// === Extract predicates from input.binder ===
+	vector<unique_ptr<ParsedExpression>> extracted_predicates;
+	if (input.binder) {
+
+		// Access the where_clause from the binder
+		auto statement = input.binder->GetRootStatement();
+		if (statement && statement->type == StatementType::SELECT_STATEMENT) {
+			auto &select_statement = (SelectStatement &)*statement;
+			if (select_statement.node->type == QueryNodeType::SELECT_NODE) {
+				auto &select_node = (SelectNode &)*select_statement.node;
+				if (select_node.where_clause) {
+					ExtractPredicates(*select_node.where_clause, extracted_predicates);
+				}
+			}
+		}
+	}
+
+	// Create IcebergTableFunctionInfo with extracted predicates
+	auto iceberg_info = make_uniq<IcebergTableFunctionInfo>(std::move(extracted_predicates));
+	input.info = iceberg_info.release(); // Assign raw pointer
+
+	// Handle 'mode' and integrate predicate pushdown
 	if (mode == "list_files") {
+		for (const auto &entry : data_entries) {
+			auto full_path =
+			    allow_moved_paths ? IcebergUtils::GetFullPath(iceberg_path, entry.file_path, fs) : entry.file_path;
+			data_file_values.emplace_back(full_path);
+		}
 		return MakeListFilesExpression(data_file_values, delete_file_values);
 	} else if (mode == "default") {
 		int64_t data_cardinality = 0, delete_cardinality = 0;
-		for(auto &manifest : iceberg_table.entries) {
-			for(auto &entry : manifest.manifest_entries) {
+		for (auto &manifest : iceberg_table.entries) {
+			for (auto &entry : manifest.manifest_entries) {
 				if (entry.status != IcebergManifestEntryStatusType::DELETED) {
 					if (entry.content == IcebergManifestEntryContentType::DATA) {
 						data_cardinality += entry.record_count;
@@ -292,7 +608,15 @@ static unique_ptr<TableRef> IcebergScanBindReplace(ClientContext &context, Table
 				}
 			}
 		}
-		return MakeScanExpression(data_file_values, delete_file_values, snapshot_to_scan.schema, allow_moved_paths, metadata_compression_codec, skip_schema_inference, data_cardinality, delete_cardinality);
+
+		IcebergTableFunctionInfo *iceberg_info_cast = dynamic_cast<IcebergTableFunctionInfo *>(input.info.get());
+		if (!iceberg_info_cast) {
+			throw std::bad_cast(); // Handle the error appropriately
+		}
+
+		return MakeScanExpression(iceberg_path, fs, data_entries, delete_file_values, snapshot_to_scan.schema,
+		                          allow_moved_paths, metadata_compression_codec, skip_schema_inference, iceberg_table,
+		                          data_cardinality, delete_cardinality, iceberg_info_cast);
 	} else {
 		throw NotImplementedException("Unknown mode type for ICEBERG_SCAN bind : '" + mode + "'");
 	}
@@ -336,4 +660,4 @@ TableFunctionSet IcebergFunctions::GetIcebergScanFunction() {
 	return function_set;
 }
 
-} // namespace duckdb
+} // namespace duckdb
\ No newline at end of file
diff --git a/src/iceberg_functions/iceberg_utils.cpp b/src/iceberg_functions/iceberg_utils.cpp
new file mode 100644
index 0000000..eb41c6f
--- /dev/null
+++ b/src/iceberg_functions/iceberg_utils.cpp
@@ -0,0 +1,6 @@
+string IcebergUtils::GetFullPath(const string &base_path, const string &relative_path, FileSystem &fs) {
+    if (fs.IsAbsolutePath(relative_path)) {
+        return relative_path;
+    }
+    return fs.JoinPath(base_path, relative_path);
+}
\ No newline at end of file
diff --git a/src/include/avro_codegen/iceberg_manifest_entry_partial.hpp b/src/include/avro_codegen/iceberg_manifest_entry_partial.hpp
index e38d4b8..a8a4557 100644
--- a/src/include/avro_codegen/iceberg_manifest_entry_partial.hpp
+++ b/src/include/avro_codegen/iceberg_manifest_entry_partial.hpp
@@ -1,124 +1,380 @@
 /**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     https://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-
-#ifndef CPX2_HH_2561633724__H_
-#define CPX2_HH_2561633724__H_
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
+/* This code was generated by avrogencpp 1.13.0-SNAPSHOT. Do not edit.*/
+
+#ifndef MANIFEST_ENTRY_HH_2043678367_H
+#define MANIFEST_ENTRY_HH_2043678367_H
 
-#include <sstream>
-#include "boost/any.hpp"
-#include "avro/Specific.hh"
-#include "avro/Encoder.hh"
 #include "avro/Decoder.hh"
+#include "avro/Encoder.hh"
+#include "avro/Specific.hh"
+#include "boost/any.hpp"
+
+#include <sstream>
+#include <utility>
 
-namespace c {
-struct data_file {
-   int32_t content;
-   std::string file_path;
-   std::string file_format;
-   int64_t record_count;
-   data_file() :
-		 content(int32_t()),
-		 file_path(std::string()),
-		 file_format(std::string()),
-		 record_count(int64_t())
-   { }
+struct k126_v127 {
+	int32_t key;
+	std::vector<uint8_t> value;
+	k126_v127() : key(int32_t()), value(std::vector<uint8_t>()) {
+	}
+};
+
+struct manifest_entry_json_Union__0__ {
+private:
+	size_t idx_;
+	boost::any value_;
+
+public:
+	/** enum representing union branches as returned by the idx() function */
+	enum class Branch : size_t {
+		null = 0,
+		array = 1,
+	};
+	size_t idx() const {
+		return idx_;
+	}
+	Branch branch() const {
+		return static_cast<Branch>(idx_);
+	}
+	bool is_null() const {
+		return (idx_ == 0);
+	}
+	void set_null() {
+		idx_ = 0;
+		value_ = boost::any();
+	}
+	const std::vector<k126_v127> &get_array() const;
+	std::vector<k126_v127> &get_array();
+	void set_array(const std::vector<k126_v127> &v);
+	void set_array(std::vector<k126_v127> &&v);
+	manifest_entry_json_Union__0__();
+};
+
+struct k129_v130 {
+	int32_t key;
+	std::vector<uint8_t> value;
+	k129_v130() : key(int32_t()), value(std::vector<uint8_t>()) {
+	}
+};
+
+struct manifest_entry_json_Union__1__ {
+private:
+	size_t idx_;
+	boost::any value_;
+
+public:
+	/** enum representing union branches as returned by the idx() function */
+	enum class Branch : size_t {
+		null = 0,
+		array = 1,
+	};
+	size_t idx() const {
+		return idx_;
+	}
+	Branch branch() const {
+		return static_cast<Branch>(idx_);
+	}
+	bool is_null() const {
+		return (idx_ == 0);
+	}
+	void set_null() {
+		idx_ = 0;
+		value_ = boost::any();
+	}
+	const std::vector<k129_v130> &get_array() const;
+	std::vector<k129_v130> &get_array();
+	void set_array(const std::vector<k129_v130> &v);
+	void set_array(std::vector<k129_v130> &&v);
+	manifest_entry_json_Union__1__();
+};
+
+struct r2 {
+	typedef manifest_entry_json_Union__0__ lower_bounds_t;
+	typedef manifest_entry_json_Union__1__ upper_bounds_t;
+	int32_t content;
+	std::string file_path;
+	std::string file_format;
+	int64_t record_count;
+	lower_bounds_t lower_bounds;
+	upper_bounds_t upper_bounds;
+	r2()
+	    : content(int32_t()), file_path(std::string()), file_format(std::string()), record_count(int64_t()),
+	      lower_bounds(lower_bounds_t()), upper_bounds(upper_bounds_t()) {
+	}
 };
 
 struct manifest_entry {
-   int32_t status;
-   data_file data_file_; // NOTE: as generated, this is called data_file, but this causes issues with GCC
-   manifest_entry() :
-		 status(int32_t()),
-		 data_file_()
-   { }
+	int32_t status;
+	r2 data_file;
+	manifest_entry() : status(int32_t()), data_file(r2()) {
+	}
 };
 
+inline const std::vector<k126_v127> &manifest_entry_json_Union__0__::get_array() const {
+	if (idx_ != 1) {
+		throw avro::Exception("Invalid type for union manifest_entry_json_Union__0__");
+	}
+	return *boost::any_cast<std::vector<k126_v127>>(&value_);
+}
+
+inline std::vector<k126_v127> &manifest_entry_json_Union__0__::get_array() {
+	if (idx_ != 1) {
+		throw avro::Exception("Invalid type for union manifest_entry_json_Union__0__");
+	}
+	return *boost::any_cast<std::vector<k126_v127>>(&value_);
+}
+
+inline void manifest_entry_json_Union__0__::set_array(const std::vector<k126_v127> &v) {
+	idx_ = 1;
+	value_ = v;
+}
+
+inline void manifest_entry_json_Union__0__::set_array(std::vector<k126_v127> &&v) {
+	idx_ = 1;
+	value_ = std::move(v);
+}
+
+inline const std::vector<k129_v130> &manifest_entry_json_Union__1__::get_array() const {
+	if (idx_ != 1) {
+		throw avro::Exception("Invalid type for union manifest_entry_json_Union__1__");
+	}
+	return *boost::any_cast<std::vector<k129_v130>>(&value_);
+}
+
+inline std::vector<k129_v130> &manifest_entry_json_Union__1__::get_array() {
+	if (idx_ != 1) {
+		throw avro::Exception("Invalid type for union manifest_entry_json_Union__1__");
+	}
+	return *boost::any_cast<std::vector<k129_v130>>(&value_);
+}
+
+inline void manifest_entry_json_Union__1__::set_array(const std::vector<k129_v130> &v) {
+	idx_ = 1;
+	value_ = v;
+}
+
+inline void manifest_entry_json_Union__1__::set_array(std::vector<k129_v130> &&v) {
+	idx_ = 1;
+	value_ = std::move(v);
+}
+
+inline manifest_entry_json_Union__0__::manifest_entry_json_Union__0__() : idx_(0) {
+}
+inline manifest_entry_json_Union__1__::manifest_entry_json_Union__1__() : idx_(0) {
 }
 namespace avro {
-template<> struct codec_traits<c::data_file> {
-   static void encode(Encoder& e, const c::data_file& v) {
-	   avro::encode(e, v.content);
-	   avro::encode(e, v.file_path);
-	   avro::encode(e, v.file_format);
-	   avro::encode(e, v.record_count);
-   }
-   static void decode(Decoder& d, c::data_file& v) {
-	   if (avro::ResolvingDecoder *rd =
-			   dynamic_cast<avro::ResolvingDecoder *>(&d)) {
-		   const std::vector<size_t> fo = rd->fieldOrder();
-		   for (std::vector<size_t>::const_iterator it = fo.begin();
-				it != fo.end(); ++it) {
-			   switch (*it) {
-			   case 0:
-				   avro::decode(d, v.content);
-				   break;
-			   case 1:
-				   avro::decode(d, v.file_path);
-				   break;
-			   case 2:
-				   avro::decode(d, v.file_format);
-				   break;
-			   case 3:
-				   avro::decode(d, v.record_count);
-				   break;
-			   default:
-				   break;
-			   }
-		   }
-	   } else {
-		   avro::decode(d, v.content);
-		   avro::decode(d, v.file_path);
-		   avro::decode(d, v.file_format);
-		   avro::decode(d, v.record_count);
-	   }
-   }
+template <>
+struct codec_traits<k126_v127> {
+	static void encode(Encoder &e, const k126_v127 &v) {
+		avro::encode(e, v.key);
+		avro::encode(e, v.value);
+	}
+	static void decode(Decoder &d, k126_v127 &v) {
+		if (avro::ResolvingDecoder *rd = dynamic_cast<avro::ResolvingDecoder *>(&d)) {
+			const std::vector<size_t> fo = rd->fieldOrder();
+			for (std::vector<size_t>::const_iterator it = fo.begin(); it != fo.end(); ++it) {
+				switch (*it) {
+				case 0:
+					avro::decode(d, v.key);
+					break;
+				case 1:
+					avro::decode(d, v.value);
+					break;
+				default:
+					break;
+				}
+			}
+		} else {
+			avro::decode(d, v.key);
+			avro::decode(d, v.value);
+		}
+	}
 };
 
-template<> struct codec_traits<c::manifest_entry> {
-   static void encode(Encoder& e, const c::manifest_entry& v) {
-	   avro::encode(e, v.status);
-	   avro::encode(e, v.data_file_);
-   }
-   static void decode(Decoder& d, c::manifest_entry& v) {
-	   if (avro::ResolvingDecoder *rd =
-			   dynamic_cast<avro::ResolvingDecoder *>(&d)) {
-		   const std::vector<size_t> fo = rd->fieldOrder();
-		   for (std::vector<size_t>::const_iterator it = fo.begin();
-				it != fo.end(); ++it) {
-			   switch (*it) {
-			   case 0:
-				   avro::decode(d, v.status);
-				   break;
-			   case 1:
-				   avro::decode(d, v.data_file_);
-				   break;
-			   default:
-				   break;
-			   }
-		   }
-	   } else {
-		   avro::decode(d, v.status);
-		   avro::decode(d, v.data_file_);
-	   }
-   }
+template <>
+struct codec_traits<manifest_entry_json_Union__0__> {
+	static void encode(Encoder &e, manifest_entry_json_Union__0__ v) {
+		e.encodeUnionIndex(v.idx());
+		switch (v.idx()) {
+		case 0:
+			e.encodeNull();
+			break;
+		case 1:
+			avro::encode(e, v.get_array());
+			break;
+		}
+	}
+	static void decode(Decoder &d, manifest_entry_json_Union__0__ &v) {
+		size_t n = d.decodeUnionIndex();
+		if (n >= 2) {
+			throw avro::Exception("Union index too big");
+		}
+		switch (n) {
+		case 0:
+			d.decodeNull();
+			v.set_null();
+			break;
+		case 1: {
+			std::vector<k126_v127> vv;
+			avro::decode(d, vv);
+			v.set_array(std::move(vv));
+		} break;
+		}
+	}
 };
 
-}
-#endif
+template <>
+struct codec_traits<k129_v130> {
+	static void encode(Encoder &e, const k129_v130 &v) {
+		avro::encode(e, v.key);
+		avro::encode(e, v.value);
+	}
+	static void decode(Decoder &d, k129_v130 &v) {
+		if (avro::ResolvingDecoder *rd = dynamic_cast<avro::ResolvingDecoder *>(&d)) {
+			const std::vector<size_t> fo = rd->fieldOrder();
+			for (std::vector<size_t>::const_iterator it = fo.begin(); it != fo.end(); ++it) {
+				switch (*it) {
+				case 0:
+					avro::decode(d, v.key);
+					break;
+				case 1:
+					avro::decode(d, v.value);
+					break;
+				default:
+					break;
+				}
+			}
+		} else {
+			avro::decode(d, v.key);
+			avro::decode(d, v.value);
+		}
+	}
+};
+
+template <>
+struct codec_traits<manifest_entry_json_Union__1__> {
+	static void encode(Encoder &e, manifest_entry_json_Union__1__ v) {
+		e.encodeUnionIndex(v.idx());
+		switch (v.idx()) {
+		case 0:
+			e.encodeNull();
+			break;
+		case 1:
+			avro::encode(e, v.get_array());
+			break;
+		}
+	}
+	static void decode(Decoder &d, manifest_entry_json_Union__1__ &v) {
+		size_t n = d.decodeUnionIndex();
+		if (n >= 2) {
+			throw avro::Exception("Union index too big");
+		}
+		switch (n) {
+		case 0:
+			d.decodeNull();
+			v.set_null();
+			break;
+		case 1: {
+			std::vector<k129_v130> vv;
+			avro::decode(d, vv);
+			v.set_array(std::move(vv));
+		} break;
+		}
+	}
+};
+
+template <>
+struct codec_traits<r2> {
+	static void encode(Encoder &e, const r2 &v) {
+		avro::encode(e, v.content);
+		avro::encode(e, v.file_path);
+		avro::encode(e, v.file_format);
+		avro::encode(e, v.record_count);
+		avro::encode(e, v.lower_bounds);
+		avro::encode(e, v.upper_bounds);
+	}
+	static void decode(Decoder &d, r2 &v) {
+		if (avro::ResolvingDecoder *rd = dynamic_cast<avro::ResolvingDecoder *>(&d)) {
+			const std::vector<size_t> fo = rd->fieldOrder();
+			for (std::vector<size_t>::const_iterator it = fo.begin(); it != fo.end(); ++it) {
+				switch (*it) {
+				case 0:
+					avro::decode(d, v.content);
+					break;
+				case 1:
+					avro::decode(d, v.file_path);
+					break;
+				case 2:
+					avro::decode(d, v.file_format);
+					break;
+				case 3:
+					avro::decode(d, v.record_count);
+					break;
+				case 4:
+					avro::decode(d, v.lower_bounds);
+					break;
+				case 5:
+					avro::decode(d, v.upper_bounds);
+					break;
+				default:
+					break;
+				}
+			}
+		} else {
+			avro::decode(d, v.content);
+			avro::decode(d, v.file_path);
+			avro::decode(d, v.file_format);
+			avro::decode(d, v.record_count);
+			avro::decode(d, v.lower_bounds);
+			avro::decode(d, v.upper_bounds);
+		}
+	}
+};
+
+template <>
+struct codec_traits<manifest_entry> {
+	static void encode(Encoder &e, const manifest_entry &v) {
+		avro::encode(e, v.status);
+		avro::encode(e, v.data_file);
+	}
+	static void decode(Decoder &d, manifest_entry &v) {
+		if (avro::ResolvingDecoder *rd = dynamic_cast<avro::ResolvingDecoder *>(&d)) {
+			const std::vector<size_t> fo = rd->fieldOrder();
+			for (std::vector<size_t>::const_iterator it = fo.begin(); it != fo.end(); ++it) {
+				switch (*it) {
+				case 0:
+					avro::decode(d, v.status);
+					break;
+				case 1:
+					avro::decode(d, v.data_file);
+					break;
+				default:
+					break;
+				}
+			}
+		} else {
+			avro::decode(d, v.status);
+			avro::decode(d, v.data_file);
+		}
+	}
+};
+
+} // namespace avro
+#endif
\ No newline at end of file
diff --git a/src/include/iceberg_metadata.hpp b/src/include/iceberg_metadata.hpp
index d7d4478..cbfae6a 100644
--- a/src/include/iceberg_metadata.hpp
+++ b/src/include/iceberg_metadata.hpp
@@ -113,6 +113,24 @@ struct IcebergTable {
 		return ret;
 	}
 
+	//! Returns all IcebergManifestEntry objects to be scanned for the given IcebergManifestContentType
+	template <IcebergManifestContentType TYPE>
+	vector<IcebergManifestEntry> GetEntries() {
+		vector<IcebergManifestEntry> ret;
+		for (auto &entry : entries) {
+			if (entry.manifest.content != TYPE) {
+				continue;
+			}
+			for (auto &manifest_entry : entry.manifest_entries) {
+				if (manifest_entry.status == IcebergManifestEntryStatusType::DELETED) {
+					continue;
+				}
+				ret.push_back(manifest_entry);
+			}
+		}
+		return ret;
+	}
+
 	void Print() {
 		Printer::Print("Iceberg table (" + path + ")");
 		for (auto &entry : entries) {
diff --git a/src/include/iceberg_types.hpp b/src/include/iceberg_types.hpp
index 16ec1ca..f6942c4 100644
--- a/src/include/iceberg_types.hpp
+++ b/src/include/iceberg_types.hpp
@@ -13,6 +13,8 @@
 #include "avro_codegen/iceberg_manifest_file_partial.hpp"
 #include "avro_codegen/iceberg_manifest_file_partial_v1.hpp"
 
+#include <iomanip> // Add this line for std::setfill and std::setw
+
 namespace duckdb {
 
 enum class IcebergManifestContentType : uint8_t {
@@ -69,12 +71,12 @@ static string MANIFEST_SCHEMA = "{\n"
 
 // Schema for v1, sequence_number and content are not present there
 static string MANIFEST_SCHEMA_V1 = "{\n"
-                                "     \"type\": \"record\",\n"
-                                "     \"name\": \"manifest_file\",\n"
-                                "     \"fields\" : [\n"
-                                "         {\"name\": \"manifest_path\", \"type\": \"string\"}\n"
-                                "     ]\n"
-                                " }";
+                                   "     \"type\": \"record\",\n"
+                                   "     \"name\": \"manifest_file\",\n"
+                                   "     \"fields\" : [\n"
+                                   "         {\"name\": \"manifest_path\", \"type\": \"string\"}\n"
+                                   "     ]\n"
+                                   " }";
 
 //! An entry in the manifest list file (top level AVRO file)
 struct IcebergManifest {
@@ -119,50 +121,141 @@ struct IcebergManifest {
 
 //! The schema containing the fields from the manifest entry.
 //! this schema should match the generated cpp header from src/include/avro_codegen/iceberg_manifest_entry_partial.hpp
-static string MANIFEST_ENTRY_SCHEMA = "{\n"
-                                      "     \"type\": \"record\",\n"
-                                      "     \"name\": \"manifest_entry\",\n"
-                                      "     \"fields\" : [\n"
-                                      "         {\"name\": \"status\", \"type\" : \"int\"},\n"
-                                      "         {\"name\": \"data_file\", \"type\": {\n"
-                                      "             \"type\": \"record\",\n"
-                                      "             \"name\": \"r2\",\n"
-                                      "             \"fields\" : [\n"
-                                      "                 {\"name\": \"content\", \"type\": \"int\"},\n"
-                                      "                 {\"name\": \"file_path\", \"type\": \"string\"},\n"
-                                      "                 {\"name\": \"file_format\", \"type\": \"string\"},\n"
-                                      "                 {\"name\": \"record_count\", \"type\" : \"long\"}\n"
-                                      "           ]}\n"
-                                      "         }\n"
-                                      "     ]\n"
-                                      " }";
+// static string MANIFEST_ENTRY_SCHEMA = "{\n"
+// "     \"type\": \"record\",\n"
+// "     \"name\": \"manifest_entry\",\n"
+// "     \"fields\" : [\n"
+// "         {\"name\": \"status\", \"type\" : \"int\"},\n"
+// "         {\"name\": \"data_file\", \"type\": {\n"
+// "             \"type\": \"record\",\n"
+// "             \"name\": \"r2\",\n"
+// "             \"fields\" : [\n"
+// "                 {\"name\": \"content\", \"type\": \"int\"},\n"
+// "                 {\"name\": \"file_path\", \"type\": \"string\"},\n"
+// "                 {\"name\": \"file_format\", \"type\": \"string\"},\n"
+// "                 {\"name\": \"record_count\", \"type\" : \"long\"},\n"
+// "                 {\"name\": \"lower_bounds\", \"type\": [\"null\", {\n"
+// "                     \"type\": \"array\",\n"
+// "                     \"items\": {\n"
+// "                         \"type\": \"record\",\n"
+// "                         \"name\": \"k126_v127\",\n"
+// "                         \"fields\": [\n"
+// "                             {\"name\": \"key\", \"type\": \"int\"},\n"
+// "                             {\"name\": \"value\", \"type\": \"bytes\"}\n"
+// "                         ]\n"
+// "                     }\n"
+// "                 }], \"default\": null},\n"
+// "                 {\"name\": \"upper_bounds\", \"type\": [\"null\", {\n"
+// "                     \"type\": \"array\",\n"
+// "                     \"items\": {\n"
+// "                         \"type\": \"record\",\n"
+// "                         \"name\": \"k129_v130\",\n"
+// "                         \"fields\": [\n"
+// "                             {\"name\": \"key\", \"type\": \"int\"},\n"
+// "                             {\"name\": \"value\", \"type\": \"bytes\"}\n"
+// "                         ]\n"
+// "                     }\n"
+// "                 }], \"default\": null}\n"
+// "           ]}\n"
+// "         }\n"
+// "     ]\n"
+// " }";
 
-static string MANIFEST_ENTRY_SCHEMA_V1 = "{\n"
-                                      "     \"type\": \"record\",\n"
-                                      "     \"name\": \"manifest_entry\",\n"
-                                      "     \"fields\" : [\n"
-                                      "         {\"name\": \"status\", \"type\" : \"int\"},\n"
-                                      "         {\"name\": \"data_file\", \"type\": {\n"
-                                      "             \"type\": \"record\",\n"
-                                      "             \"name\": \"r2\",\n"
-                                      "             \"fields\" : [\n"
-                                      "                 {\"name\": \"file_path\", \"type\": \"string\"},\n"
-                                      "                 {\"name\": \"file_format\", \"type\": \"string\"},\n"
-                                      "                 {\"name\": \"record_count\", \"type\" : \"long\"}\n"
-                                      "           ]}\n"
-                                      "         }\n"
-                                      "     ]\n"
-                                      " }";
+static string MANIFEST_ENTRY_SCHEMA = R"(
+{
+  "type": "record",
+  "name": "manifest_entry",
+  "fields": [
+    { "name": "status", "type": "int", "field-id": 0 },
+    { "name": "data_file", "type": {
+      "type": "record",
+      "name": "r2",
+      "fields": [
+        { "name": "content", "type": "int", "field-id": 134 },
+        { "name": "file_path", "type": "string", "field-id": 100 },
+        { "name": "file_format", "type": "string", "field-id": 101 },
+        { "name": "record_count", "type": "long", "field-id": 103 },
+        { "name": "lower_bounds", "type": [
+          "null",
+          {
+            "type": "array",
+            "items": {
+              "type": "record",
+              "name": "k126_v127",
+              "fields": [
+                { "name": "key", "type": "int", "field-id": 126 },
+                { "name": "value", "type": "bytes", "field-id": 127 }
+              ]
+            }
+          }
+        ], 
+        "default": null,
+        "field-id": 125 
+        },
+        { "name": "upper_bounds", "type": [
+          "null",
+          {
+            "type": "array",
+            "items": {
+              "type": "record",
+              "name": "k129_v130",
+              "fields": [
+                { "name": "key", "type": "int", "field-id": 129 },
+                { "name": "value", "type": "bytes", "field-id": 130 }
+              ]
+            }
+          }
+        ], "default": null, "field-id": 128 }
+      ]
+    }, "field-id": 2 }
+  ]
+}
+)";
 
+static string MANIFEST_ENTRY_SCHEMA_V1 = "{\n"
+                                         "     \"type\": \"record\",\n"
+                                         "     \"name\": \"manifest_entry\",\n"
+                                         "     \"fields\" : [\n"
+                                         "         {\"name\": \"status\", \"type\" : \"int\"},\n"
+                                         "         {\"name\": \"data_file\", \"type\": {\n"
+                                         "             \"type\": \"record\",\n"
+                                         "             \"name\": \"r2\",\n"
+                                         "             \"fields\" : [\n"
+                                         "                 {\"name\": \"file_path\", \"type\": \"string\"},\n"
+                                         "                 {\"name\": \"file_format\", \"type\": \"string\"},\n"
+                                         "                 {\"name\": \"record_count\", \"type\" : \"long\"}\n"
+                                         "           ]}\n"
+                                         "         }\n"
+                                         "     ]\n"
+                                         " }";
 
 //! An entry in a manifest file
 struct IcebergManifestEntry {
-	explicit IcebergManifestEntry(const c::manifest_entry &schema) {
+	explicit IcebergManifestEntry(const manifest_entry &schema) {
 		status = (IcebergManifestEntryStatusType)schema.status;
-		content = (IcebergManifestEntryContentType)schema.data_file_.content;
-		file_path = schema.data_file_.file_path;
-		file_format = schema.data_file_.file_format;
-		record_count = schema.data_file_.record_count;
+		content = (IcebergManifestEntryContentType)schema.data_file.content;
+		const auto &data_file = schema.data_file;
+		file_path = data_file.file_path;
+		file_format = data_file.file_format;
+		record_count = data_file.record_count;
+		lower_bounds.clear();
+		upper_bounds.clear();
+
+		// Handle lower_bounds
+		if (data_file.lower_bounds.idx() == static_cast<size_t>(manifest_entry_json_Union__0__::Branch::array)) {
+			const auto &bounds_array = data_file.lower_bounds.get_array();
+			for (const auto &lb : bounds_array) {
+				lower_bounds[std::to_string(lb.key)] = lb.value;
+			}
+		}
+
+		// Handle upper_bounds
+		if (data_file.upper_bounds.idx() == static_cast<size_t>(manifest_entry_json_Union__1__::Branch::array)) {
+			const auto &bounds_array = data_file.upper_bounds.get_array();
+			for (const auto &ub : bounds_array) {
+				upper_bounds[std::to_string(ub.key)] = ub.value;
+			}
+		}
 	}
 
 	explicit IcebergManifestEntry(const c::manifest_entry_v1 &schema) {
@@ -171,6 +264,10 @@ struct IcebergManifestEntry {
 		file_path = schema.data_file_.file_path;
 		file_format = schema.data_file_.file_format;
 		record_count = schema.data_file_.record_count;
+
+		// Initialize bounds as empty maps
+		lower_bounds.clear();
+		upper_bounds.clear();
 	}
 
 	IcebergManifestEntryStatusType status;
@@ -181,6 +278,10 @@ struct IcebergManifestEntry {
 	string file_format;
 	int64_t record_count;
 
+	// Add new members for bounds
+	std::unordered_map<std::string, std::vector<uint8_t>> lower_bounds;
+	std::unordered_map<std::string, std::vector<uint8_t>> upper_bounds;
+
 	void Print() {
 		Printer::Print("    -> ManifestEntry = { type: " + IcebergManifestEntryStatusTypeToString(status) +
 		               ", content: " + IcebergManifestEntryContentTypeToString(content) + ", file: " + file_path +
@@ -209,4 +310,4 @@ struct IcebergTableEntry {
 		}
 	}
 };
-} // namespace duckdb
+} // namespace duckdb
\ No newline at end of file
diff --git a/test/sql/iceberg_scan.test b/test/sql/iceberg_scan.test
index 4f84bb7..56c6168 100644
--- a/test/sql/iceberg_scan.test
+++ b/test/sql/iceberg_scan.test
@@ -79,3 +79,27 @@ query I
 SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg_gz', ALLOW_MOVED_PATHS=TRUE, METADATA_COMPRESSION_CODEC="gzip", version='2', version_name_format='v%s%s.metadata.json');
 ----
 111968
+
+### Pushdown l_shipdate with results
+query I
+SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg', ALLOW_MOVED_PATHS=TRUE) WHERE l_shipdate >= '1996-01-01';
+----
+21674
+
+### Pushdown l_shipdate with no results
+query I
+SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg', ALLOW_MOVED_PATHS=TRUE) WHERE l_shipdate >= '2222-01-01';
+----
+0
+
+### Pushdown l_shipmode with results
+query I
+SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg', ALLOW_MOVED_PATHS=TRUE) WHERE l_shipmode = 'AIR';
+----
+7265
+
+### Pushdown l_shipmode with no results
+query I
+SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg', ALLOW_MOVED_PATHS=TRUE) WHERE l_shipmode = 'FOOBAR';
+----
+0