From 0a95895de14b92fb8e263b342f61ea35d9c2ef57 Mon Sep 17 00:00:00 2001 From: Josh Humphries <2035234+jhump@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:55:44 -0400 Subject: [PATCH] allow non-string logicalType property; if not a string, will always go into other props, even for fixed and primitive --- schema_parse.go | 68 +++++++++++++++---------------------------------- schema_test.go | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 48 deletions(-) diff --git a/schema_parse.go b/schema_parse.go index 7aab400..6903c02 100644 --- a/schema_parse.go +++ b/schema_parse.go @@ -161,9 +161,8 @@ func parseComplexType(namespace string, m map[string]any, seen seenCache, cache } type primitiveSchema struct { - Type string `mapstructure:"type"` - LogicalType string `mapstructure:"logicalType"` - Props map[string]any `mapstructure:",remain"` + Type string `mapstructure:"type"` + Props map[string]any `mapstructure:",remain"` } func parsePrimitive(typ Type, m map[string]any) (Schema, error) { @@ -183,10 +182,10 @@ func parsePrimitive(typ Type, m map[string]any) (Schema, error) { } var logical LogicalSchema - if p.LogicalType != "" { - logical = parsePrimitiveLogicalType(typ, p.LogicalType, p.Props) - if logical == nil { - preserveLogicalType(p.LogicalType, &p.Props) + if logicalType := logicalTypeProperty(p.Props); logicalType != "" { + logical = parsePrimitiveLogicalType(typ, logicalType, p.Props) + if logical != nil { + delete(p.Props, "logicalType") } } @@ -241,9 +240,6 @@ func parseRecord(typ Type, namespace string, m map[string]any, seen seenCache, c if r.Namespace == "" { r.Namespace = namespace } - if err := checkLogicalType(r.Props); err != nil { - return nil, err - } if !hasKey(meta.Keys, "fields") { return nil, errors.New("avro: record must have an array of fields") @@ -360,9 +356,6 @@ func parseEnum(namespace string, m map[string]any, seen seenCache, cache *Schema if e.Namespace == "" { e.Namespace = namespace } - if err := checkLogicalType(e.Props); err != nil { - return nil, err - } enum, err := NewEnumSchema(e.Name, e.Namespace, e.Symbols, WithDefault(e.Default), WithAliases(e.Aliases), WithDoc(e.Doc), WithProps(e.Props), @@ -406,9 +399,6 @@ func parseArray(namespace string, m map[string]any, seen seenCache, cache *Schem if err != nil { return nil, err } - if err := checkLogicalType(a.Props); err != nil { - return nil, err - } return NewArraySchema(schema, WithProps(a.Props)), nil } @@ -435,9 +425,6 @@ func parseMap(namespace string, m map[string]any, seen seenCache, cache *SchemaC if err != nil { return nil, err } - if err := checkLogicalType(ms.Props); err != nil { - return nil, err - } return NewMapSchema(schema, WithProps(ms.Props)), nil } @@ -456,13 +443,12 @@ func parseUnion(namespace string, v []any, seen seenCache, cache *SchemaCache) ( } type fixedSchema struct { - Name string `mapstructure:"name"` - Namespace string `mapstructure:"namespace"` - Aliases []string `mapstructure:"aliases"` - Type string `mapstructure:"type"` - Size int `mapstructure:"size"` - LogicalType string `mapstructure:"logicalType"` - Props map[string]any `mapstructure:",remain"` + Name string `mapstructure:"name"` + Namespace string `mapstructure:"namespace"` + Aliases []string `mapstructure:"aliases"` + Type string `mapstructure:"type"` + Size int `mapstructure:"size"` + Props map[string]any `mapstructure:",remain"` } func parseFixed(namespace string, m map[string]any, seen seenCache, cache *SchemaCache) (Schema, error) { @@ -486,10 +472,10 @@ func parseFixed(namespace string, m map[string]any, seen seenCache, cache *Schem } var logical LogicalSchema - if f.LogicalType != "" { - logical = parseFixedLogicalType(f.Size, f.LogicalType, f.Props) - if logical == nil { - preserveLogicalType(f.LogicalType, &f.Props) + if logicalType := logicalTypeProperty(f.Props); logicalType != "" { + logical = parseFixedLogicalType(f.Size, logicalType, f.Props) + if logical != nil { + delete(f.Props, "logicalType") } } @@ -643,23 +629,9 @@ func (c seenCache) Add(name string) error { return nil } -func preserveLogicalType(logicalType string, props *map[string]any) { - if logicalType == "" { - return // nothing to preserve +func logicalTypeProperty(props map[string]any) string { + if lt, ok := props["logicalType"].(string); ok { + return lt } - if *props == nil { - *props = make(map[string]any, 1) - } - (*props)["logicalType"] = logicalType -} - -func checkLogicalType(props map[string]any) error { - val, ok := props["logicalType"] - if !ok { - return nil - } - if _, isString := val.(string); !isString { - return fmt.Errorf(`"logicalType" attribute must be a string, got %T`, val) - } - return nil + return "" } diff --git a/schema_test.go b/schema_test.go index 4d0c064..cc8ccea 100644 --- a/schema_test.go +++ b/schema_test.go @@ -1695,6 +1695,30 @@ func TestParse_PreservesAllProperties(t *testing.T) { }, rec.Props()) }, }, + { + name: "fixed-invalid-logical-type", + schema: `{ + "type": "fixed", + "name": "SomeFixed", + "size": 16, + "logicalType": {"foo": "bar", "baz": ["x","y","z"]}, + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.FixedSchema) + assert.Equal(t, map[string]any{ + "logicalType": map[string]any{ + "foo": "bar", + "baz": []any{"x", "y", "z"}, + }, + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, { name: "fixed-decimal-logical-type", schema: `{ @@ -1819,6 +1843,30 @@ func TestParse_PreservesAllProperties(t *testing.T) { }, rec.Props()) }, }, + { + name: "primitive-invalid-logical-type", + schema: `{ + "type": "string", + "name": "SomeString", + "logicalType": {"foo": "bar", "baz": ["x","y","z"]}, + "precision": "abc", + "scale": "def", + "other": [1,2,3] + }`, + check: func(t *testing.T, schema avro.Schema) { + rec := schema.(*avro.PrimitiveSchema) + assert.Equal(t, map[string]any{ + "name": "SomeString", + "logicalType": map[string]any{ + "foo": "bar", + "baz": []any{"x", "y", "z"}, + }, + "precision": "abc", + "scale": "def", + "other": []any{1.0, 2.0, 3.0}, + }, rec.Props()) + }, + }, { name: "primitive-date-logical-type", schema: `{