Skip to content

Commit

Permalink
allow discarding unknown enum values
Browse files Browse the repository at this point in the history
  • Loading branch information
ubunatic committed Jan 15, 2024
1 parent 8a16b41 commit bfc1fc2
Show file tree
Hide file tree
Showing 4 changed files with 53 additions and 0 deletions.
6 changes: 6 additions & 0 deletions encoding/protobq/marshal.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ func Marshal(msg proto.Message) (map[string]bigquery.Value, error) {
type MarshalOptions struct {
// Schema contains the schema options.
Schema SchemaOptions

DiscardUnknownEnumValues bool
}

// Marshal marshals the given proto.Message in the BigQuery format using options in
Expand Down Expand Up @@ -266,6 +268,10 @@ func (o MarshalOptions) marshalEnumValue(
if enumValue := field.Enum().Values().ByNumber(enumNumber); enumValue != nil {
return string(enumValue.Name()), nil
}
if o.DiscardUnknownEnumValues {
// Use 'null' for BQ rows to indicate that no value could be determined.
return nil, nil
}
return nil, fmt.Errorf("unknown enum number: %v", value.Enum())
}

Expand Down
11 changes: 11 additions & 0 deletions encoding/protobq/marshal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,17 @@ func TestMarshalOptions_Marshal(t *testing.T) {
opt: MarshalOptions{Schema: SchemaOptions{UseOneofFields: true}},
expected: map[string]bigquery.Value{},
},

{
name: "allow unknown enum values",
msg: &examplev1.ExampleEnum{
EnumValue: examplev1.ExampleEnum_Enum(100),
},
opt: MarshalOptions{Schema: SchemaOptions{UseOneofFields: true}, DiscardUnknownEnumValues: true},
expected: map[string]bigquery.Value{
"enum_value": nil,
},
},
} {
tt := tt
t.Run(tt.name, func(t *testing.T) {
Expand Down
15 changes: 15 additions & 0 deletions encoding/protobq/unmarshal.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ type UnmarshalOptions struct {

// If DiscardUnknown is set, unknown fields are ignored.
DiscardUnknown bool

// If DiscardUnknownEnumValues is set, unknown enum values are set to the empty value.
// For compatibility, this is independent of the complementary DiscardUnknown option.
DiscardUnknownEnumValues bool
}

// Unmarshal reads the given BigQuery row and populates the given proto.Message using
Expand Down Expand Up @@ -635,20 +639,31 @@ func (o UnmarshalOptions) unmarshalEnumScalar(
if o.Schema.UseEnumNumbers {
v, ok := bqValue.(int64)
if !ok {
if o.DiscardUnknownEnumValues {
return protoreflect.Value{}, nil
}
return protoreflect.Value{}, fmt.Errorf(
"invalid BigQuery value %#v for enum %s number", bqValue, field.Enum().FullName(),
)
}
return protoreflect.ValueOfEnum(protoreflect.EnumNumber(int32(v))), nil
}

v, ok := bqValue.(string)
if !ok {
if o.DiscardUnknownEnumValues {
return protoreflect.Value{}, nil
}
return protoreflect.Value{}, fmt.Errorf(
"invalid BigQuery value %#v for enum %s", bqValue, field.Enum().FullName(),
)
}

enumVal := field.Enum().Values().ByName(protoreflect.Name(v))
if enumVal == nil {
if o.DiscardUnknownEnumValues {
return protoreflect.Value{}, nil
}
return protoreflect.Value{}, fmt.Errorf(
"unknown enum value %#v for enum %s", bqValue, field.Enum().FullName(),
)
Expand Down
21 changes: 21 additions & 0 deletions encoding/protobq/unmarshal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,27 @@ func TestUnmarshalOptions_Unmarshal(t *testing.T) {
opt: UnmarshalOptions{Schema: SchemaOptions{UseOneofFields: true}},
expected: &examplev1.ExampleOneof{},
},

{
name: "discard unknown enum string",
row: map[string]bigquery.Value{"enum_value": "ENUM_VALUE_100"},
opt: UnmarshalOptions{DiscardUnknownEnumValues: true},
expected: &examplev1.ExampleEnum{EnumValue: examplev1.ExampleEnum_ENUM_UNSPECIFIED},
},

{
name: "discard unknown enum number",
row: map[string]bigquery.Value{"enum_value": 100},
opt: UnmarshalOptions{DiscardUnknownEnumValues: true},
expected: &examplev1.ExampleEnum{EnumValue: examplev1.ExampleEnum_ENUM_UNSPECIFIED},
},

{
name: "discard enum null",
row: map[string]bigquery.Value{"enum_value": nil},
opt: UnmarshalOptions{DiscardUnknownEnumValues: true},
expected: &examplev1.ExampleEnum{EnumValue: examplev1.ExampleEnum_ENUM_UNSPECIFIED},
},
} {
tt := tt
t.Run(tt.name, func(t *testing.T) {
Expand Down

0 comments on commit bfc1fc2

Please sign in to comment.