Skip to content

Commit

Permalink
made AddedSnapshotID optional for V1
Browse files Browse the repository at this point in the history
  • Loading branch information
zeroshade committed Sep 22, 2023
1 parent f5a8aaf commit 0d64b47
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 9 deletions.
2 changes: 1 addition & 1 deletion internal/avro_schemas.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func init() {
{"name": "partition_spec_id", "type": "int", "doc": "Spec ID used to write", "field-id": 502},
{
"name": "added_snapshot_id",
"type": "long",
"type": ["null", "long"],
"doc": "Snapshot ID that added the manifest",
"field-id": 503
},
Expand Down
21 changes: 14 additions & 7 deletions manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,17 +52,21 @@ type ManifestV1Builder struct {
// NewManifestV1Builder is passed all of the required fields and then allows
// all of the optional fields to be set by calling the corresponding methods
// before calling [ManifestV1Builder.Build] to construct the object.
func NewManifestV1Builder(path string, length int64, partitionSpecID int32, addedSnapshotID int64) *ManifestV1Builder {
func NewManifestV1Builder(path string, length int64, partitionSpecID int32) *ManifestV1Builder {
return &ManifestV1Builder{
m: &manifestFileV1{
Path: path,
Len: length,
SpecID: partitionSpecID,
AddedSnapshotID: addedSnapshotID,
Path: path,
Len: length,
SpecID: partitionSpecID,
},
}
}

func (b *ManifestV1Builder) AddedSnapshotID(id int64) *ManifestV1Builder {
b.m.AddedSnapshotID = &id
return b
}

func (b *ManifestV1Builder) AddedFiles(cnt int32) *ManifestV1Builder {
b.m.AddedFilesCount = &cnt
return b
Expand Down Expand Up @@ -115,7 +119,7 @@ type manifestFileV1 struct {
Path string `avro:"manifest_path"`
Len int64 `avro:"manifest_length"`
SpecID int32 `avro:"partition_spec_id"`
AddedSnapshotID int64 `avro:"added_snapshot_id"`
AddedSnapshotID *int64 `avro:"added_snapshot_id"`
AddedFilesCount *int32 `avro:"added_data_files_count"`
ExistingFilesCount *int32 `avro:"existing_data_files_count"`
DeletedFilesCount *int32 `avro:"deleted_data_files_count"`
Expand All @@ -134,7 +138,10 @@ func (m *manifestFileV1) ManifestContent() ManifestContent {
return ManifestContentData
}
func (m *manifestFileV1) SnapshotID() int64 {
return m.AddedSnapshotID
if m.AddedSnapshotID == nil {
return 0
}
return *m.AddedSnapshotID
}

func (m *manifestFileV1) AddedDataFiles() int32 {
Expand Down
8 changes: 7 additions & 1 deletion manifest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ var (
addedRows int64 = 237993
manifestFileRecordsV1 = []ManifestFile{
NewManifestV1Builder("/home/iceberg/warehouse/nyc/taxis_partitioned/metadata/0125c686-8aa6-4502-bdcc-b6d17ca41a3b-m0.avro",
7989, 0, snapshotID).
7989, 0).
AddedSnapshotID(snapshotID).
AddedFiles(3).
ExistingFiles(0).
DeletedFiles(0).
Expand Down Expand Up @@ -70,6 +71,11 @@ var (
EntryStatus: EntryStatusADDED,
Snapshot: entrySnapshotID,
Data: dataFile{
// bad value for Content but this field doesn't exist in V1
// so it shouldn't get written and shouldn't be read back out
// so the roundtrip test asserts that we get the default value
// back out.
Content: EntryContentEqDeletes,
Path: "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet",
Format: ParquetFile,
PartitionData: map[string]any{"VendorID": int(1), "tpep_pickup_datetime": time.Unix(1925, 0)},
Expand Down

0 comments on commit 0d64b47

Please sign in to comment.