Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Preserve unrecognized logical types and properties #469

Merged
merged 6 commits into from
Oct 11, 2024
Merged

Preserve unrecognized logical types and properties #469

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
57091b0
preserve all extra properties/metadata, including unrecognized logica…
jhump Oct 10, 2024
cc31452
allow null types to have properties
jhump Oct 11, 2024
7576339
make linter happy; code review feedback: rename otherProps->props, re…
jhump Oct 11, 2024
509e3e3
Merge branch 'main' into jh/preserve-unrecognized
jhump Oct 11, 2024
767e78a
update ocf_test.go to verify that ocf.FullSchemaMarshaler can retain …
jhump Oct 11, 2024
0a95895
allow non-string logicalType property; if not a string, will always g…
jhump Oct 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions gen/testdata/golden.avsc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,15 +106,13 @@
{
"name": "mapOfStrings",
"type": {
"name": "aMapOfStrings",
"type": "map",
"values": "string"
}
},
{
"name": "mapOfRecords",
"type": {
"name": "aMapOfRecords",
"type": "map",
"values": {
"name": "RecordInMap",
Expand Down Expand Up @@ -175,7 +173,6 @@
{
"name": "aRecordArray",
"type": {
"name": "someRecordArray",
"type": "array",
"items": {
"name": "recordInArray",
Expand Down
71 changes: 58 additions & 13 deletions schema.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,19 @@ func (nullDefaultType) MarshalJSON() ([]byte, error) {
var nullDefault nullDefaultType = struct{}{}

var (
schemaReserved = []string{
"doc", "fields", "items", "name", "namespace", "size", "symbols",
"values", "type", "aliases", "logicalType", "precision", "scale",
}
fieldReserved = []string{"default", "doc", "name", "order", "type", "aliases"}
// Note: order matches the order of properties as they are named in the spec.
// https://avro.apache.org/docs/1.12.0/specification
recordReserved = []string{"type", "name", "namespace", "doc", "aliases", "fields"}
fieldReserved = []string{"name", "doc", "type", "order", "aliases", "default"}
enumReserved = []string{"type", "name", "namespace", "aliases", "doc", "symbols", "default"}
arrayReserved = []string{"type", "items"}
mapReserved = []string{"type", "values"}
fixedReserved = []string{"type", "name", "namespace", "aliases", "size"}
fixedWithLogicalTypeReserved = []string{"type", "name", "namespace", "aliases", "size", "logicalType"}
fixedWithDecimalTypeReserved = []string{"type", "name", "namespace", "aliases", "size", "logicalType", "precision", "scale"}
primitiveReserved = []string{"type"}
primitiveWithLogicalTypeReserved = []string{"type", "logicalType"}
primitiveWithDecimalTypeReserved = []string{"type", "logicalType", "precision", "scale"}
)

// Type is a schema type.
Expand Down Expand Up @@ -482,9 +490,16 @@ func NewPrimitiveSchema(t Type, l LogicalSchema, opts ...SchemaOption) *Primitiv
for _, opt := range opts {
opt(&cfg)
}

reservedProps := primitiveReserved
if l != nil {
if l.Type() == Decimal {
reservedProps = primitiveWithDecimalTypeReserved
} else {
reservedProps = primitiveWithLogicalTypeReserved
}
}
return &PrimitiveSchema{
properties: newProperties(cfg.props, schemaReserved),
properties: newProperties(cfg.props, reservedProps),
cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp},
typ: t,
logical: l,
Expand Down Expand Up @@ -574,7 +589,7 @@ func NewRecordSchema(name, namespace string, fields []*Field, opts ...SchemaOpti

return &RecordSchema{
name: n,
properties: newProperties(cfg.props, schemaReserved),
properties: newProperties(cfg.props, recordReserved),
cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp},
fields: fields,
doc: cfg.doc,
Expand Down Expand Up @@ -919,7 +934,7 @@ func NewEnumSchema(name, namespace string, symbols []string, opts ...SchemaOptio

return &EnumSchema{
name: n,
properties: newProperties(cfg.props, schemaReserved),
properties: newProperties(cfg.props, enumReserved),
cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp},
symbols: symbols,
def: def,
Expand Down Expand Up @@ -1072,7 +1087,7 @@ func NewArraySchema(items Schema, opts ...SchemaOption) *ArraySchema {
}

return &ArraySchema{
properties: newProperties(cfg.props, schemaReserved),
properties: newProperties(cfg.props, arrayReserved),
cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp},
items: items,
}
Expand Down Expand Up @@ -1142,7 +1157,7 @@ func NewMapSchema(values Schema, opts ...SchemaOption) *MapSchema {
}

return &MapSchema{
properties: newProperties(cfg.props, schemaReserved),
properties: newProperties(cfg.props, mapReserved),
cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp},
values: values,
}
Expand Down Expand Up @@ -1323,9 +1338,17 @@ func NewFixedSchema(
return nil, err
}

reservedProps := fixedReserved
if logical != nil {
if logical.Type() == Decimal {
reservedProps = fixedWithDecimalTypeReserved
} else {
reservedProps = fixedWithLogicalTypeReserved
}
}
return &FixedSchema{
name: n,
properties: newProperties(cfg.props, schemaReserved),
properties: newProperties(cfg.props, reservedProps),
cacheFingerprinter: cacheFingerprinter{writerFingerprint: cfg.wfp},
size: size,
logical: logical,
Expand Down Expand Up @@ -1406,9 +1429,22 @@ func (s *FixedSchema) CacheFingerprint() [32]byte {

// NullSchema is an Avro null type schema.
type NullSchema struct {
properties
fingerprinter
}

// NewNullSchema creates a new NullSchema.
func NewNullSchema(opts ...SchemaOption) *NullSchema {
var cfg schemaConfig
for _, opt := range opts {
opt(&cfg)
}

return &NullSchema{
properties: newProperties(cfg.props, primitiveReserved),
}
}

// Type returns the type of the schema.
func (s *NullSchema) Type() Type {
return Null
Expand All @@ -1421,7 +1457,16 @@ func (s *NullSchema) String() string {

// MarshalJSON marshals the schema to json.
func (s *NullSchema) MarshalJSON() ([]byte, error) {
return []byte(`"null"`), nil
if len(s.props) == 0 {
return []byte(`"null"`), nil
}
buf := new(bytes.Buffer)
buf.WriteString(`{"type":"null"`)
if err := s.marshalPropertiesToJSON(buf); err != nil {
return nil, err
}
buf.WriteString("}")
return buf.Bytes(), nil
}

// Fingerprint returns the SHA256 fingerprint of the schema.
Expand Down
4 changes: 4 additions & 0 deletions schema_json_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ func TestSchema_JSON(t *testing.T) {
input: `{"type":"null"}`,
json: `"null"`,
},
{
input: `{"type":"null","other":"foo"}`,
json: `{"type":"null","other":"foo"}`,
},
{
input: `"boolean"`,
json: `"boolean"`,
Expand Down
Loading
Loading