diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/count.csv b/data/iceberg/generated_spec1_0_001/expected_results/last/count.csv new file mode 100644 index 0000000..3b35899 --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/expected_results/last/count.csv @@ -0,0 +1,2 @@ +count +7690 \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/data/._SUCCESS.crc b/data/iceberg/generated_spec1_0_001/expected_results/last/data/._SUCCESS.crc new file mode 100644 index 0000000..3b7b044 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/expected_results/last/data/._SUCCESS.crc differ diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/data/.part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet.crc b/data/iceberg/generated_spec1_0_001/expected_results/last/data/.part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet.crc new file mode 100644 index 0000000..62fa3b7 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/expected_results/last/data/.part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet.crc differ diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/data/part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet b/data/iceberg/generated_spec1_0_001/expected_results/last/data/part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet new file mode 100644 index 0000000..54fb77d Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/expected_results/last/data/part-00000-90484277-8f62-41f7-b073-a4c8999e393e-c000.snappy.parquet differ diff --git a/data/iceberg/generated_spec1_0_001/expected_results/last/query.sql b/data/iceberg/generated_spec1_0_001/expected_results/last/query.sql new file mode 100644 index 0000000..6c7f91c --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/expected_results/last/query.sql @@ -0,0 +1,3 @@ +-- The query executed at this step: +ALTER TABLE iceberg_catalog.pyspark_iceberg_table +ALTER COLUMN schema_evol_added_col_1 TYPE BIGINT; \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-1-bcb5bbb9-a993-41f7-95e1-09e0c2475f4a-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-1-bcb5bbb9-a993-41f7-95e1-09e0c2475f4a-00001.parquet new file mode 100644 index 0000000..97ac7e8 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-1-bcb5bbb9-a993-41f7-95e1-09e0c2475f4a-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-14-db28d572-b7de-4568-9b00-8c55c69cb179-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-14-db28d572-b7de-4568-9b00-8c55c69cb179-00001.parquet new file mode 100644 index 0000000..9b8b181 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-14-db28d572-b7de-4568-9b00-8c55c69cb179-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-26-f6481588-9ba4-4a7b-b3dd-f188d41fa5b8-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-26-f6481588-9ba4-4a7b-b3dd-f188d41fa5b8-00001.parquet new file mode 100644 index 0000000..3755277 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-26-f6481588-9ba4-4a7b-b3dd-f188d41fa5b8-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-36-cf35a788-d8c2-4ded-a9f7-5239797e80b8-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-36-cf35a788-d8c2-4ded-a9f7-5239797e80b8-00001.parquet new file mode 100644 index 0000000..facc504 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-36-cf35a788-d8c2-4ded-a9f7-5239797e80b8-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-5-bd5417f5-f28c-46b1-b1ab-39ee9c191368-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-5-bd5417f5-f28c-46b1-b1ab-39ee9c191368-00001.parquet new file mode 100644 index 0000000..5998bc2 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-5-bd5417f5-f28c-46b1-b1ab-39ee9c191368-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-9-6cdc0135-4256-4772-8c3e-3f4803ded842-00001.parquet b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-9-6cdc0135-4256-4772-8c3e-3f4803ded842-00001.parquet new file mode 100644 index 0000000..8f8a093 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/data/00000-9-6cdc0135-4256-4772-8c3e-3f4803ded842-00001.parquet differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m0.avro new file mode 100644 index 0000000..90e2897 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m1.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m1.avro new file mode 100644 index 0000000..584671d Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/0f120788-1926-4605-a1ab-450f4cf3ccee-m1.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m0.avro new file mode 100644 index 0000000..7dd57c8 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m1.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m1.avro new file mode 100644 index 0000000..80355e7 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/4aa40041-ccc6-4e64-a9ab-366875aafd63-m1.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/7723fb1b-ae48-49de-9e77-cd7945667cb9-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/7723fb1b-ae48-49de-9e77-cd7945667cb9-m0.avro new file mode 100644 index 0000000..54b6edf Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/7723fb1b-ae48-49de-9e77-cd7945667cb9-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m0.avro new file mode 100644 index 0000000..2974743 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m1.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m1.avro new file mode 100644 index 0000000..2ecadc1 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/c091e891-ac3a-4429-be9a-e63f1ed63b99-m1.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m0.avro new file mode 100644 index 0000000..925a068 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m1.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m1.avro new file mode 100644 index 0000000..3666460 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m1.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m2.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m2.avro new file mode 100644 index 0000000..feb60ea Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/d76c5203-4f0a-46ef-a293-268e0afec64b-m2.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/e3febcc2-7f11-44b9-80af-571fb1c0463a-m0.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/e3febcc2-7f11-44b9-80af-571fb1c0463a-m0.avro new file mode 100644 index 0000000..b812dca Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/e3febcc2-7f11-44b9-80af-571fb1c0463a-m0.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro new file mode 100644 index 0000000..9a7b32f Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro new file mode 100644 index 0000000..45cc87a Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro new file mode 100644 index 0000000..c0715d7 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro new file mode 100644 index 0000000..e6edc42 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro new file mode 100644 index 0000000..be74220 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro new file mode 100644 index 0000000..fde9231 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro new file mode 100644 index 0000000..63e2d76 Binary files /dev/null and b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro differ diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json new file mode 100644 index 0000000..ec7f17f --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json @@ -0,0 +1,217 @@ +{ + "format-version" : 1, + "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1", + "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table", + "last-updated-ms" : 1719580919873, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 9145725745960929259, + "refs" : { + "main" : { + "snapshot-id" : 9145725745960929259, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580919873, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440845", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580919873, + "snapshot-id" : 9145725745960929259 + } ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json new file mode 100644 index 0000000..8eb6b17 --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json @@ -0,0 +1,246 @@ +{ + "format-version" : 1, + "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1", + "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table", + "last-updated-ms" : 1719580920785, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 8671490307245765264, + "refs" : { + "main" : { + "snapshot-id" : 8671490307245765264, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580919873, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440845", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8671490307245765264, + "parent-snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580920785, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340114", + "removed-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340114", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580919873, + "snapshot-id" : 9145725745960929259 + }, { + "timestamp-ms" : 1719580920785, + "snapshot-id" : 8671490307245765264 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580919873, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json new file mode 100644 index 0000000..ae141f5 --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json @@ -0,0 +1,272 @@ +{ + "format-version" : 1, + "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1", + "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table", + "last-updated-ms" : 1719580921348, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 4543110679664799316, + "refs" : { + "main" : { + "snapshot-id" : 4543110679664799316, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580919873, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440845", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8671490307245765264, + "parent-snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580920785, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340114", + "removed-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340114", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4543110679664799316, + "parent-snapshot-id" : 8671490307245765264, + "timestamp-ms" : 1719580921348, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133331", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473445", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580919873, + "snapshot-id" : 9145725745960929259 + }, { + "timestamp-ms" : 1719580920785, + "snapshot-id" : 8671490307245765264 + }, { + "timestamp-ms" : 1719580921348, + "snapshot-id" : 4543110679664799316 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580919873, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580920785, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json new file mode 100644 index 0000000..b44942a --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json @@ -0,0 +1,301 @@ +{ + "format-version" : 1, + "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1", + "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table", + "last-updated-ms" : 1719580921764, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 6238750566879819059, + "refs" : { + "main" : { + "snapshot-id" : 6238750566879819059, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580919873, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440845", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8671490307245765264, + "parent-snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580920785, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340114", + "removed-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340114", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4543110679664799316, + "parent-snapshot-id" : 8671490307245765264, + "timestamp-ms" : 1719580921348, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133331", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473445", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6238750566879819059, + "parent-snapshot-id" : 4543110679664799316, + "timestamp-ms" : 1719580921764, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399010", + "removed-files-size" : "473445", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580919873, + "snapshot-id" : 9145725745960929259 + }, { + "timestamp-ms" : 1719580920785, + "snapshot-id" : 8671490307245765264 + }, { + "timestamp-ms" : 1719580921348, + "snapshot-id" : 4543110679664799316 + }, { + "timestamp-ms" : 1719580921764, + "snapshot-id" : 6238750566879819059 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580919873, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580920785, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580921348, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json new file mode 100644 index 0000000..1aabc15 --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json @@ -0,0 +1,324 @@ +{ + "format-version" : 1, + "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1", + "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table", + "last-updated-ms" : 1719580922113, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 2276968461870063565, + "refs" : { + "main" : { + "snapshot-id" : 2276968461870063565, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580919873, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440845", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8671490307245765264, + "parent-snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580920785, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340114", + "removed-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340114", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4543110679664799316, + "parent-snapshot-id" : 8671490307245765264, + "timestamp-ms" : 1719580921348, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133331", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473445", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6238750566879819059, + "parent-snapshot-id" : 4543110679664799316, + "timestamp-ms" : 1719580921764, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399010", + "removed-files-size" : "473445", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 2276968461870063565, + "parent-snapshot-id" : 6238750566879819059, + "timestamp-ms" : 1719580922113, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "changed-partition-count" : "0", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580919873, + "snapshot-id" : 9145725745960929259 + }, { + "timestamp-ms" : 1719580920785, + "snapshot-id" : 8671490307245765264 + }, { + "timestamp-ms" : 1719580921348, + "snapshot-id" : 4543110679664799316 + }, { + "timestamp-ms" : 1719580921764, + "snapshot-id" : 6238750566879819059 + }, { + "timestamp-ms" : 1719580922113, + "snapshot-id" : 2276968461870063565 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580919873, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580920785, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580921348, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719580921764, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json new file mode 100644 index 0000000..9162f1b --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json @@ -0,0 +1,353 @@ +{ + "format-version" : 1, + "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1", + "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table", + "last-updated-ms" : 1719580922559, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 1692767036460164714, + "refs" : { + "main" : { + "snapshot-id" : 1692767036460164714, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580919873, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440845", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8671490307245765264, + "parent-snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580920785, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340114", + "removed-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340114", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4543110679664799316, + "parent-snapshot-id" : 8671490307245765264, + "timestamp-ms" : 1719580921348, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133331", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473445", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6238750566879819059, + "parent-snapshot-id" : 4543110679664799316, + "timestamp-ms" : 1719580921764, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399010", + "removed-files-size" : "473445", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 2276968461870063565, + "parent-snapshot-id" : 6238750566879819059, + "timestamp-ms" : 1719580922113, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "changed-partition-count" : "0", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 1692767036460164714, + "parent-snapshot-id" : 2276968461870063565, + "timestamp-ms" : 1719580922559, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399010", + "removed-files-size" : "399010", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580919873, + "snapshot-id" : 9145725745960929259 + }, { + "timestamp-ms" : 1719580920785, + "snapshot-id" : 8671490307245765264 + }, { + "timestamp-ms" : 1719580921348, + "snapshot-id" : 4543110679664799316 + }, { + "timestamp-ms" : 1719580921764, + "snapshot-id" : 6238750566879819059 + }, { + "timestamp-ms" : 1719580922113, + "snapshot-id" : 2276968461870063565 + }, { + "timestamp-ms" : 1719580922559, + "snapshot-id" : 1692767036460164714 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580919873, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580920785, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580921348, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719580921764, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719580922113, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v7.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v7.metadata.json new file mode 100644 index 0000000..744af3d --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v7.metadata.json @@ -0,0 +1,445 @@ +{ + "format-version" : 1, + "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1", + "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table", + "last-updated-ms" : 1719580922734, + "last-column-id" : 16, + "schema" : { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 1692767036460164714, + "refs" : { + "main" : { + "snapshot-id" : 1692767036460164714, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580919873, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440845", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8671490307245765264, + "parent-snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580920785, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340114", + "removed-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340114", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4543110679664799316, + "parent-snapshot-id" : 8671490307245765264, + "timestamp-ms" : 1719580921348, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133331", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473445", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6238750566879819059, + "parent-snapshot-id" : 4543110679664799316, + "timestamp-ms" : 1719580921764, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399010", + "removed-files-size" : "473445", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 2276968461870063565, + "parent-snapshot-id" : 6238750566879819059, + "timestamp-ms" : 1719580922113, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "changed-partition-count" : "0", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 1692767036460164714, + "parent-snapshot-id" : 2276968461870063565, + "timestamp-ms" : 1719580922559, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399010", + "removed-files-size" : "399010", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580919873, + "snapshot-id" : 9145725745960929259 + }, { + "timestamp-ms" : 1719580920785, + "snapshot-id" : 8671490307245765264 + }, { + "timestamp-ms" : 1719580921348, + "snapshot-id" : 4543110679664799316 + }, { + "timestamp-ms" : 1719580921764, + "snapshot-id" : 6238750566879819059 + }, { + "timestamp-ms" : 1719580922113, + "snapshot-id" : 2276968461870063565 + }, { + "timestamp-ms" : 1719580922559, + "snapshot-id" : 1692767036460164714 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580919873, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580920785, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580921348, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719580921764, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719580922113, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719580922559, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v8.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v8.metadata.json new file mode 100644 index 0000000..ede9cca --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v8.metadata.json @@ -0,0 +1,474 @@ +{ + "format-version" : 1, + "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1", + "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table", + "last-updated-ms" : 1719580923120, + "last-column-id" : 16, + "schema" : { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 4407328776463037310, + "refs" : { + "main" : { + "snapshot-id" : 4407328776463037310, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580919873, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440845", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8671490307245765264, + "parent-snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580920785, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340114", + "removed-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340114", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4543110679664799316, + "parent-snapshot-id" : 8671490307245765264, + "timestamp-ms" : 1719580921348, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133331", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473445", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6238750566879819059, + "parent-snapshot-id" : 4543110679664799316, + "timestamp-ms" : 1719580921764, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399010", + "removed-files-size" : "473445", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 2276968461870063565, + "parent-snapshot-id" : 6238750566879819059, + "timestamp-ms" : 1719580922113, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "changed-partition-count" : "0", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 1692767036460164714, + "parent-snapshot-id" : 2276968461870063565, + "timestamp-ms" : 1719580922559, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399010", + "removed-files-size" : "399010", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4407328776463037310, + "parent-snapshot-id" : 1692767036460164714, + "timestamp-ms" : 1719580923120, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "400831", + "removed-files-size" : "399010", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "400831", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580919873, + "snapshot-id" : 9145725745960929259 + }, { + "timestamp-ms" : 1719580920785, + "snapshot-id" : 8671490307245765264 + }, { + "timestamp-ms" : 1719580921348, + "snapshot-id" : 4543110679664799316 + }, { + "timestamp-ms" : 1719580921764, + "snapshot-id" : 6238750566879819059 + }, { + "timestamp-ms" : 1719580922113, + "snapshot-id" : 2276968461870063565 + }, { + "timestamp-ms" : 1719580922559, + "snapshot-id" : 1692767036460164714 + }, { + "timestamp-ms" : 1719580923120, + "snapshot-id" : 4407328776463037310 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580919873, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580920785, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580921348, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719580921764, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719580922113, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719580922559, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1719580922734, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v7.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v9.metadata.json b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v9.metadata.json new file mode 100644 index 0000000..6d60244 --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v9.metadata.json @@ -0,0 +1,561 @@ +{ + "format-version" : 1, + "table-uuid" : "2e23a4d3-2f64-47ac-aad6-f37df92836a1", + "location" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table", + "last-updated-ms" : 1719580923295, + "last-column-id" : 16, + "schema" : { + "type" : "struct", + "schema-id" : 2, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "long" + } ] + }, + "current-schema-id" : 2, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, { + "type" : "struct", + "schema-id" : 2, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "long" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 4407328776463037310, + "refs" : { + "main" : { + "snapshot-id" : 4407328776463037310, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580919873, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440845", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-9145725745960929259-1-7723fb1b-ae48-49de-9e77-cd7945667cb9.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8671490307245765264, + "parent-snapshot-id" : 9145725745960929259, + "timestamp-ms" : 1719580920785, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340114", + "removed-files-size" : "440845", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340114", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-8671490307245765264-1-4aa40041-ccc6-4e64-a9ab-366875aafd63.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4543110679664799316, + "parent-snapshot-id" : 8671490307245765264, + "timestamp-ms" : 1719580921348, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133331", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473445", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4543110679664799316-1-e3febcc2-7f11-44b9-80af-571fb1c0463a.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 6238750566879819059, + "parent-snapshot-id" : 4543110679664799316, + "timestamp-ms" : 1719580921764, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399010", + "removed-files-size" : "473445", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-6238750566879819059-1-d76c5203-4f0a-46ef-a293-268e0afec64b.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 2276968461870063565, + "parent-snapshot-id" : 6238750566879819059, + "timestamp-ms" : 1719580922113, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "changed-partition-count" : "0", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-2276968461870063565-1-5ee46b42-10e4-401d-8f61-2bd3b5ebb548.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 1692767036460164714, + "parent-snapshot-id" : 2276968461870063565, + "timestamp-ms" : 1719580922559, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399010", + "removed-files-size" : "399010", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399010", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-1692767036460164714-1-0f120788-1926-4605-a1ab-450f4cf3ccee.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4407328776463037310, + "parent-snapshot-id" : 1692767036460164714, + "timestamp-ms" : 1719580923120, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580917302", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "400831", + "removed-files-size" : "399010", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "400831", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/snap-4407328776463037310-1-c091e891-ac3a-4429-be9a-e63f1ed63b99.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580919873, + "snapshot-id" : 9145725745960929259 + }, { + "timestamp-ms" : 1719580920785, + "snapshot-id" : 8671490307245765264 + }, { + "timestamp-ms" : 1719580921348, + "snapshot-id" : 4543110679664799316 + }, { + "timestamp-ms" : 1719580921764, + "snapshot-id" : 6238750566879819059 + }, { + "timestamp-ms" : 1719580922113, + "snapshot-id" : 2276968461870063565 + }, { + "timestamp-ms" : 1719580922559, + "snapshot-id" : 1692767036460164714 + }, { + "timestamp-ms" : 1719580923120, + "snapshot-id" : 4407328776463037310 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580919873, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580920785, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580921348, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719580921764, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719580922113, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719580922559, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1719580922734, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v7.metadata.json" + }, { + "timestamp-ms" : 1719580923120, + "metadata-file" : "data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/v8.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/version-hint.text b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/version-hint.text new file mode 100644 index 0000000..f11c82a --- /dev/null +++ b/data/iceberg/generated_spec1_0_001/pyspark_iceberg_table/metadata/version-hint.text @@ -0,0 +1 @@ +9 \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/count.csv b/data/iceberg/generated_spec2_0_001/expected_results/last/count.csv new file mode 100644 index 0000000..1221195 --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/expected_results/last/count.csv @@ -0,0 +1,2 @@ +count +6592 \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/data/._SUCCESS.crc b/data/iceberg/generated_spec2_0_001/expected_results/last/data/._SUCCESS.crc new file mode 100644 index 0000000..3b7b044 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/expected_results/last/data/._SUCCESS.crc differ diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc b/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc new file mode 100644 index 0000000..a0d706d Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc b/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc new file mode 100644 index 0000000..2f43db6 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/expected_results/last/data/.part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet.crc differ diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet b/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet new file mode 100644 index 0000000..53283bf Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00000-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet b/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet new file mode 100644 index 0000000..6d6fe6d Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/expected_results/last/data/part-00001-5aa13b42-2ef5-483d-a6a0-f5bf9eac87c4-c000.snappy.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/expected_results/last/query.sql b/data/iceberg/generated_spec2_0_001/expected_results/last/query.sql new file mode 100644 index 0000000..6c7f91c --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/expected_results/last/query.sql @@ -0,0 +1,3 @@ +-- The query executed at this step: +ALTER TABLE iceberg_catalog.pyspark_iceberg_table +ALTER COLUMN schema_evol_added_col_1 TYPE BIGINT; \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-1-3e88ec3a-0596-440f-9ce6-3debf172be49-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-1-3e88ec3a-0596-440f-9ce6-3debf172be49-00001.parquet new file mode 100644 index 0000000..6837339 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-1-3e88ec3a-0596-440f-9ce6-3debf172be49-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001-deletes.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001-deletes.parquet new file mode 100644 index 0000000..4d44760 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001-deletes.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001.parquet new file mode 100644 index 0000000..b3c12c7 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-12-ac52ac46-8deb-43f9-b745-e7c078928b7a-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-24-3a7a66b3-bd3a-4417-b6a9-45cb309eddc2-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-24-3a7a66b3-bd3a-4417-b6a9-45cb309eddc2-00001.parquet new file mode 100644 index 0000000..98c2d49 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-24-3a7a66b3-bd3a-4417-b6a9-45cb309eddc2-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001-deletes.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001-deletes.parquet new file mode 100644 index 0000000..47329b2 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001-deletes.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001.parquet new file mode 100644 index 0000000..979ad92 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-3-1c142ffe-c3f5-4089-9820-f2a530d50754-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001-deletes.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001-deletes.parquet new file mode 100644 index 0000000..99aafa1 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001-deletes.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001.parquet new file mode 100644 index 0000000..c242949 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-46-08e25db5-5199-4416-8916-bfb07212b1fb-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-7-3be35a72-224f-475b-a0eb-34cea92784b4-00001.parquet b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-7-3be35a72-224f-475b-a0eb-34cea92784b4-00001.parquet new file mode 100644 index 0000000..285069d Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/data/00000-7-3be35a72-224f-475b-a0eb-34cea92784b4-00001.parquet differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/26871791-3133-4757-9cbc-b356c613c83a-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/26871791-3133-4757-9cbc-b356c613c83a-m0.avro new file mode 100644 index 0000000..a922251 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/26871791-3133-4757-9cbc-b356c613c83a-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m0.avro new file mode 100644 index 0000000..f0b73f1 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m1.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m1.avro new file mode 100644 index 0000000..d7b9a64 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/355a32d2-0d4f-4da3-8019-f0b782863350-m1.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m0.avro new file mode 100644 index 0000000..19d600f Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m1.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m1.avro new file mode 100644 index 0000000..c8d664d Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/7c6f85be-3a33-4e3a-817d-7839fa44ff07-m1.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/9ae37730-f1aa-4609-8b39-3f0ded6f78cf-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/9ae37730-f1aa-4609-8b39-3f0ded6f78cf-m0.avro new file mode 100644 index 0000000..92c4470 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/9ae37730-f1aa-4609-8b39-3f0ded6f78cf-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m0.avro new file mode 100644 index 0000000..e28081d Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m1.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m1.avro new file mode 100644 index 0000000..9101316 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/b467c132-3bea-404a-ae0f-54ef5a4fbd1f-m1.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m0.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m0.avro new file mode 100644 index 0000000..9a6b6d5 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m0.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m1.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m1.avro new file mode 100644 index 0000000..31716e1 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/c958489b-0a9b-4c1a-b254-f7162a3fbd6b-m1.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro new file mode 100644 index 0000000..fafc511 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro new file mode 100644 index 0000000..3df6c65 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro new file mode 100644 index 0000000..f4da2fc Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4786266686210019019-1-7c6f85be-3a33-4e3a-817d-7839fa44ff07.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4786266686210019019-1-7c6f85be-3a33-4e3a-817d-7839fa44ff07.avro new file mode 100644 index 0000000..0da1497 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4786266686210019019-1-7c6f85be-3a33-4e3a-817d-7839fa44ff07.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro new file mode 100644 index 0000000..8f34f30 Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro new file mode 100644 index 0000000..3e9ee6b Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro new file mode 100644 index 0000000..aab9ece Binary files /dev/null and b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro differ diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json new file mode 100644 index 0000000..3c9acfc --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json @@ -0,0 +1,139 @@ +{ + "format-version" : 2, + "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7", + "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table", + "last-sequence-number" : 1, + "last-updated-ms" : 1719580927570, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 764624380497366583, + "refs" : { + "main" : { + "snapshot-id" : 764624380497366583, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580927570, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440835", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440835", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580927570, + "snapshot-id" : 764624380497366583 + } ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json new file mode 100644 index 0000000..bae2e24 --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json @@ -0,0 +1,169 @@ +{ + "format-version" : 2, + "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7", + "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table", + "last-sequence-number" : 2, + "last-updated-ms" : 1719580928275, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 4037069315291880534, + "refs" : { + "main" : { + "snapshot-id" : 4037069315291880534, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580927570, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440835", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440835", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4037069315291880534, + "parent-snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580928275, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114786", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555621", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580927570, + "snapshot-id" : 764624380497366583 + }, { + "timestamp-ms" : 1719580928275, + "snapshot-id" : 4037069315291880534 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580927570, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json new file mode 100644 index 0000000..7db94f1 --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json @@ -0,0 +1,196 @@ +{ + "format-version" : 2, + "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7", + "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table", + "last-sequence-number" : 3, + "last-updated-ms" : 1719580929047, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 6287117141668015642, + "refs" : { + "main" : { + "snapshot-id" : 6287117141668015642, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580927570, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440835", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440835", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4037069315291880534, + "parent-snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580928275, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114786", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555621", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 6287117141668015642, + "parent-snapshot-id" : 4037069315291880534, + "timestamp-ms" : 1719580929047, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133314", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688935", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580927570, + "snapshot-id" : 764624380497366583 + }, { + "timestamp-ms" : 1719580928275, + "snapshot-id" : 4037069315291880534 + }, { + "timestamp-ms" : 1719580929047, + "snapshot-id" : 6287117141668015642 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580927570, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580928275, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json new file mode 100644 index 0000000..edcbb4b --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json @@ -0,0 +1,226 @@ +{ + "format-version" : 2, + "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7", + "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table", + "last-sequence-number" : 4, + "last-updated-ms" : 1719580929661, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 6585012225877417653, + "refs" : { + "main" : { + "snapshot-id" : 6585012225877417653, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580927570, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440835", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440835", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4037069315291880534, + "parent-snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580928275, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114786", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555621", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 6287117141668015642, + "parent-snapshot-id" : 4037069315291880534, + "timestamp-ms" : 1719580929047, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133314", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688935", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 6585012225877417653, + "parent-snapshot-id" : 6287117141668015642, + "timestamp-ms" : 1719580929661, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410506", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099441", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580927570, + "snapshot-id" : 764624380497366583 + }, { + "timestamp-ms" : 1719580928275, + "snapshot-id" : 4037069315291880534 + }, { + "timestamp-ms" : 1719580929047, + "snapshot-id" : 6287117141668015642 + }, { + "timestamp-ms" : 1719580929661, + "snapshot-id" : 6585012225877417653 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580927570, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580928275, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580929047, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json new file mode 100644 index 0000000..61b94fc --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json @@ -0,0 +1,256 @@ +{ + "format-version" : 2, + "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7", + "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table", + "last-sequence-number" : 5, + "last-updated-ms" : 1719580930402, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 4440319347650982524, + "refs" : { + "main" : { + "snapshot-id" : 4440319347650982524, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580927570, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440835", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440835", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4037069315291880534, + "parent-snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580928275, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114786", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555621", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 6287117141668015642, + "parent-snapshot-id" : 4037069315291880534, + "timestamp-ms" : 1719580929047, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133314", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688935", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 6585012225877417653, + "parent-snapshot-id" : 6287117141668015642, + "timestamp-ms" : 1719580929661, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410506", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099441", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 4440319347650982524, + "parent-snapshot-id" : 6585012225877417653, + "timestamp-ms" : 1719580930402, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6592", + "deleted-records" : "7690", + "added-files-size" : "333848", + "removed-files-size" : "388851", + "changed-partition-count" : "1", + "total-records" : "17359", + "total-files-size" : "1044438", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580927570, + "snapshot-id" : 764624380497366583 + }, { + "timestamp-ms" : 1719580928275, + "snapshot-id" : 4037069315291880534 + }, { + "timestamp-ms" : 1719580929047, + "snapshot-id" : 6287117141668015642 + }, { + "timestamp-ms" : 1719580929661, + "snapshot-id" : 6585012225877417653 + }, { + "timestamp-ms" : 1719580930402, + "snapshot-id" : 4440319347650982524 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580927570, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580928275, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580929047, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719580929661, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json new file mode 100644 index 0000000..2f8a8fb --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json @@ -0,0 +1,280 @@ +{ + "format-version" : 2, + "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7", + "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table", + "last-sequence-number" : 6, + "last-updated-ms" : 1719580930749, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 3119545726281138740, + "refs" : { + "main" : { + "snapshot-id" : 3119545726281138740, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580927570, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440835", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440835", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4037069315291880534, + "parent-snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580928275, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114786", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555621", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 6287117141668015642, + "parent-snapshot-id" : 4037069315291880534, + "timestamp-ms" : 1719580929047, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133314", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688935", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 6585012225877417653, + "parent-snapshot-id" : 6287117141668015642, + "timestamp-ms" : 1719580929661, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410506", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099441", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 4440319347650982524, + "parent-snapshot-id" : 6585012225877417653, + "timestamp-ms" : 1719580930402, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6592", + "deleted-records" : "7690", + "added-files-size" : "333848", + "removed-files-size" : "388851", + "changed-partition-count" : "1", + "total-records" : "17359", + "total-files-size" : "1044438", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 3119545726281138740, + "parent-snapshot-id" : 4440319347650982524, + "timestamp-ms" : 1719580930749, + "summary" : { + "operation" : "delete", + "spark.app.id" : "local-1719580924876", + "changed-partition-count" : "0", + "total-records" : "17359", + "total-files-size" : "1044438", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580927570, + "snapshot-id" : 764624380497366583 + }, { + "timestamp-ms" : 1719580928275, + "snapshot-id" : 4037069315291880534 + }, { + "timestamp-ms" : 1719580929047, + "snapshot-id" : 6287117141668015642 + }, { + "timestamp-ms" : 1719580929661, + "snapshot-id" : 6585012225877417653 + }, { + "timestamp-ms" : 1719580930402, + "snapshot-id" : 4440319347650982524 + }, { + "timestamp-ms" : 1719580930749, + "snapshot-id" : 3119545726281138740 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580927570, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580928275, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580929047, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719580929661, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719580930402, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v7.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v7.metadata.json new file mode 100644 index 0000000..7dd8062 --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v7.metadata.json @@ -0,0 +1,367 @@ +{ + "format-version" : 2, + "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7", + "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table", + "last-sequence-number" : 6, + "last-updated-ms" : 1719580930997, + "last-column-id" : 16, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 3119545726281138740, + "refs" : { + "main" : { + "snapshot-id" : 3119545726281138740, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580927570, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440835", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440835", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4037069315291880534, + "parent-snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580928275, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114786", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555621", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 6287117141668015642, + "parent-snapshot-id" : 4037069315291880534, + "timestamp-ms" : 1719580929047, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133314", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688935", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 6585012225877417653, + "parent-snapshot-id" : 6287117141668015642, + "timestamp-ms" : 1719580929661, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410506", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099441", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 4440319347650982524, + "parent-snapshot-id" : 6585012225877417653, + "timestamp-ms" : 1719580930402, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6592", + "deleted-records" : "7690", + "added-files-size" : "333848", + "removed-files-size" : "388851", + "changed-partition-count" : "1", + "total-records" : "17359", + "total-files-size" : "1044438", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 3119545726281138740, + "parent-snapshot-id" : 4440319347650982524, + "timestamp-ms" : 1719580930749, + "summary" : { + "operation" : "delete", + "spark.app.id" : "local-1719580924876", + "changed-partition-count" : "0", + "total-records" : "17359", + "total-files-size" : "1044438", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580927570, + "snapshot-id" : 764624380497366583 + }, { + "timestamp-ms" : 1719580928275, + "snapshot-id" : 4037069315291880534 + }, { + "timestamp-ms" : 1719580929047, + "snapshot-id" : 6287117141668015642 + }, { + "timestamp-ms" : 1719580929661, + "snapshot-id" : 6585012225877417653 + }, { + "timestamp-ms" : 1719580930402, + "snapshot-id" : 4440319347650982524 + }, { + "timestamp-ms" : 1719580930749, + "snapshot-id" : 3119545726281138740 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580927570, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580928275, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580929047, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719580929661, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719580930402, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719580930749, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v8.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v8.metadata.json new file mode 100644 index 0000000..56d07a9 --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v8.metadata.json @@ -0,0 +1,397 @@ +{ + "format-version" : 2, + "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7", + "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table", + "last-sequence-number" : 7, + "last-updated-ms" : 1719580931465, + "last-column-id" : 16, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 4786266686210019019, + "refs" : { + "main" : { + "snapshot-id" : 4786266686210019019, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580927570, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440835", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440835", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4037069315291880534, + "parent-snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580928275, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114786", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555621", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 6287117141668015642, + "parent-snapshot-id" : 4037069315291880534, + "timestamp-ms" : 1719580929047, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133314", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688935", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 6585012225877417653, + "parent-snapshot-id" : 6287117141668015642, + "timestamp-ms" : 1719580929661, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410506", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099441", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 4440319347650982524, + "parent-snapshot-id" : 6585012225877417653, + "timestamp-ms" : 1719580930402, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6592", + "deleted-records" : "7690", + "added-files-size" : "333848", + "removed-files-size" : "388851", + "changed-partition-count" : "1", + "total-records" : "17359", + "total-files-size" : "1044438", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 3119545726281138740, + "parent-snapshot-id" : 4440319347650982524, + "timestamp-ms" : 1719580930749, + "summary" : { + "operation" : "delete", + "spark.app.id" : "local-1719580924876", + "changed-partition-count" : "0", + "total-records" : "17359", + "total-files-size" : "1044438", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro", + "schema-id" : 0 + }, { + "sequence-number" : 7, + "snapshot-id" : 4786266686210019019, + "parent-snapshot-id" : 3119545726281138740, + "timestamp-ms" : 1719580931465, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "685", + "added-files-size" : "51653", + "added-position-deletes" : "685", + "changed-partition-count" : "1", + "total-records" : "18044", + "total-files-size" : "1096091", + "total-data-files" : "5", + "total-delete-files" : "3", + "total-position-deletes" : "11452", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4786266686210019019-1-7c6f85be-3a33-4e3a-817d-7839fa44ff07.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580927570, + "snapshot-id" : 764624380497366583 + }, { + "timestamp-ms" : 1719580928275, + "snapshot-id" : 4037069315291880534 + }, { + "timestamp-ms" : 1719580929047, + "snapshot-id" : 6287117141668015642 + }, { + "timestamp-ms" : 1719580929661, + "snapshot-id" : 6585012225877417653 + }, { + "timestamp-ms" : 1719580930402, + "snapshot-id" : 4440319347650982524 + }, { + "timestamp-ms" : 1719580930749, + "snapshot-id" : 3119545726281138740 + }, { + "timestamp-ms" : 1719580931465, + "snapshot-id" : 4786266686210019019 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580927570, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580928275, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580929047, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719580929661, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719580930402, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719580930749, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1719580930997, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v7.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v9.metadata.json b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v9.metadata.json new file mode 100644 index 0000000..85c01bd --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v9.metadata.json @@ -0,0 +1,484 @@ +{ + "format-version" : 2, + "table-uuid" : "7c10a28a-8931-4e12-8142-0befc8b0eed7", + "location" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table", + "last-sequence-number" : 7, + "last-updated-ms" : 1719580931691, + "last-column-id" : 16, + "current-schema-id" : 2, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, { + "type" : "struct", + "schema-id" : 2, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "long" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 4786266686210019019, + "refs" : { + "main" : { + "snapshot-id" : 4786266686210019019, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580927570, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440835", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440835", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-764624380497366583-1-26871791-3133-4757-9cbc-b356c613c83a.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 4037069315291880534, + "parent-snapshot-id" : 764624380497366583, + "timestamp-ms" : 1719580928275, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114786", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555621", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4037069315291880534-1-c958489b-0a9b-4c1a-b254-f7162a3fbd6b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 6287117141668015642, + "parent-snapshot-id" : 4037069315291880534, + "timestamp-ms" : 1719580929047, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133314", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688935", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6287117141668015642-1-9ae37730-f1aa-4609-8b39-3f0ded6f78cf.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 6585012225877417653, + "parent-snapshot-id" : 6287117141668015642, + "timestamp-ms" : 1719580929661, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410506", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099441", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-6585012225877417653-1-355a32d2-0d4f-4da3-8019-f0b782863350.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 4440319347650982524, + "parent-snapshot-id" : 6585012225877417653, + "timestamp-ms" : 1719580930402, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6592", + "deleted-records" : "7690", + "added-files-size" : "333848", + "removed-files-size" : "388851", + "changed-partition-count" : "1", + "total-records" : "17359", + "total-files-size" : "1044438", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4440319347650982524-1-b467c132-3bea-404a-ae0f-54ef5a4fbd1f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 3119545726281138740, + "parent-snapshot-id" : 4440319347650982524, + "timestamp-ms" : 1719580930749, + "summary" : { + "operation" : "delete", + "spark.app.id" : "local-1719580924876", + "changed-partition-count" : "0", + "total-records" : "17359", + "total-files-size" : "1044438", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-3119545726281138740-1-e3f073e1-20d8-4831-b927-86100e4ad98c.avro", + "schema-id" : 0 + }, { + "sequence-number" : 7, + "snapshot-id" : 4786266686210019019, + "parent-snapshot-id" : 3119545726281138740, + "timestamp-ms" : 1719580931465, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719580924876", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "685", + "added-files-size" : "51653", + "added-position-deletes" : "685", + "changed-partition-count" : "1", + "total-records" : "18044", + "total-files-size" : "1096091", + "total-data-files" : "5", + "total-delete-files" : "3", + "total-position-deletes" : "11452", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/snap-4786266686210019019-1-7c6f85be-3a33-4e3a-817d-7839fa44ff07.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719580927570, + "snapshot-id" : 764624380497366583 + }, { + "timestamp-ms" : 1719580928275, + "snapshot-id" : 4037069315291880534 + }, { + "timestamp-ms" : 1719580929047, + "snapshot-id" : 6287117141668015642 + }, { + "timestamp-ms" : 1719580929661, + "snapshot-id" : 6585012225877417653 + }, { + "timestamp-ms" : 1719580930402, + "snapshot-id" : 4440319347650982524 + }, { + "timestamp-ms" : 1719580930749, + "snapshot-id" : 3119545726281138740 + }, { + "timestamp-ms" : 1719580931465, + "snapshot-id" : 4786266686210019019 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719580927570, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719580928275, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719580929047, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719580929661, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719580930402, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719580930749, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1719580930997, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v7.metadata.json" + }, { + "timestamp-ms" : 1719580931465, + "metadata-file" : "data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/v8.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/version-hint.text b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/version-hint.text new file mode 100644 index 0000000..f11c82a --- /dev/null +++ b/data/iceberg/generated_spec2_0_001/pyspark_iceberg_table/metadata/version-hint.text @@ -0,0 +1 @@ +9 \ No newline at end of file diff --git a/test/sql/iceberg_scan_generated_data_0_01.test_slow b/test/sql/iceberg_scan_generated_data_0_001.test similarity index 100% rename from test/sql/iceberg_scan_generated_data_0_01.test_slow rename to test/sql/iceberg_scan_generated_data_0_001.test