From 3f78dc41a17ac4b1abc96ad1d1d8b67f90df4d66 Mon Sep 17 00:00:00 2001 From: worstell Date: Tue, 14 May 2024 12:47:37 -0400 Subject: [PATCH] feat: support sum types on ingress (#1440) --- backend/controller/ingress/alias.go | 60 +++++++++++--- backend/controller/ingress/alias_test.go | 30 +++++++ backend/controller/ingress/ingress.go | 31 ++++++- backend/controller/ingress/ingress_test.go | 27 +++++++ backend/controller/ingress/request.go | 4 +- .../xyz/block/ftl/v1/schema/schema.proto | 9 --- backend/schema/jsonschema.go | 26 +++++- backend/schema/jsonschema_test.go | 50 +++++++++++- backend/schema/protobuf.go | 1 - backend/schema/type_registry.go | 72 ----------------- backend/schema/validate.go | 16 +++- buildengine/build_go_test.go | 15 ++++ .../testdata/projects/another/another.go | 16 ++++ buildengine/testdata/projects/other/other.go | 5 +- buildengine/testdata/type_registry_main.go | 38 +++------ .../xyz/block/ftl/v1/schema/schema_pb.ts | 80 ------------------- .../build-template/_ftl.tmpl/go/main/main.go | 16 +--- go-runtime/compile/build.go | 63 ++++++++++----- go-runtime/compile/schema.go | 27 +------ integration/integration_test.go | 6 ++ .../testdata/go/httpingress/httpingress.go | 18 +++++ 21 files changed, 337 insertions(+), 273 deletions(-) delete mode 100644 backend/schema/type_registry.go diff --git a/backend/controller/ingress/alias.go b/backend/controller/ingress/alias.go index c316998a43..7f3048135a 100644 --- a/backend/controller/ingress/alias.go +++ b/backend/controller/ingress/alias.go @@ -12,19 +12,55 @@ func transformAliasedFields(sch *schema.Schema, t schema.Type, obj any, aliaser } switch t := t.(type) { case *schema.Ref: - data, err := sch.ResolveRefMonomorphised(t) - if err != nil { - return fmt.Errorf("%s: failed to resolve data type: %w", t.Pos, err) - } - m, ok := obj.(map[string]any) - if !ok { - return fmt.Errorf("%s: expected map, got %T", t.Pos, obj) - } - for _, field := range data.Fields { - name := aliaser(m, field) - if err := transformAliasedFields(sch, field.Type, m[name], aliaser); err != nil { - return err + switch decl := sch.ResolveRef(t).(type) { + case *schema.Data: + data, err := sch.ResolveRefMonomorphised(t) + if err != nil { + return fmt.Errorf("%s: failed to resolve data type: %w", t.Pos, err) + } + m, ok := obj.(map[string]any) + if !ok { + return fmt.Errorf("%s: expected map, got %T", t.Pos, obj) + } + for _, field := range data.Fields { + name := aliaser(m, field) + if err := transformAliasedFields(sch, field.Type, m[name], aliaser); err != nil { + return err + } + } + case *schema.Enum: + if decl.IsValueEnum() { + return nil + } + + // type enum + m, ok := obj.(map[string]any) + if !ok { + return fmt.Errorf("%s: expected map, got %T", t.Pos, obj) + } + name, ok := m["name"] + if !ok { + return fmt.Errorf("%s: expected type enum request to have 'name' field", t.Pos) + } + nameStr, ok := name.(string) + if !ok { + return fmt.Errorf("%s: expected 'name' field to be a string, got %T", t.Pos, name) + } + + value, ok := m["value"] + if !ok { + return fmt.Errorf("%s: expected type enum request to have 'value' field", t.Pos) + } + + for _, v := range decl.Variants { + if v.Name == nameStr { + if err := transformAliasedFields(sch, v.Value.(*schema.TypeValue).Value, value, aliaser); err != nil { //nolint:forcetypeassert + return err + } + } } + case *schema.Config, *schema.Database, *schema.FSM, *schema.Secret, *schema.Verb: + return fmt.Errorf("%s: unsupported ref type %T", t.Pos, decl) } case *schema.Array: diff --git a/backend/controller/ingress/alias_test.go b/backend/controller/ingress/alias_test.go index d435b67628..4129e3cbd3 100644 --- a/backend/controller/ingress/alias_test.go +++ b/backend/controller/ingress/alias_test.go @@ -11,6 +11,11 @@ import ( func TestTransformFromAliasedFields(t *testing.T) { schemaText := ` module test { + enum TypeEnum { + A test.Inner + B String + } + data Inner { waz String +alias json "foo" } @@ -21,9 +26,11 @@ func TestTransformFromAliasedFields(t *testing.T) { array [test.Inner] map {String: test.Inner} optional test.Inner + typeEnum test.TypeEnum } } ` + sch, err := schema.ParseString("test", schemaText) assert.NoError(t, err) actual, err := transformFromAliasedFields(&schema.Ref{Module: "test", Name: "Test"}, sch, map[string]any{ @@ -44,6 +51,10 @@ func TestTransformFromAliasedFields(t *testing.T) { "optional": map[string]any{ "foo": "value", }, + "typeEnum": map[string]any{ + "name": "A", + "value": map[string]any{"foo": "value"}, + }, }) expected := map[string]any{ "scalar": "value", @@ -63,6 +74,10 @@ func TestTransformFromAliasedFields(t *testing.T) { "optional": map[string]any{ "waz": "value", }, + "typeEnum": map[string]any{ + "name": "A", + "value": map[string]any{"waz": "value"}, + }, } assert.NoError(t, err) assert.Equal(t, expected, actual) @@ -71,6 +86,11 @@ func TestTransformFromAliasedFields(t *testing.T) { func TestTransformToAliasedFields(t *testing.T) { schemaText := ` module test { + enum TypeEnum { + A test.Inner + B String + } + data Inner { waz String +alias json "foo" } @@ -81,9 +101,11 @@ func TestTransformToAliasedFields(t *testing.T) { array [test.Inner] map {String: test.Inner} optional test.Inner + typeEnum test.TypeEnum } } ` + sch, err := schema.ParseString("test", schemaText) assert.NoError(t, err) actual, err := transformToAliasedFields(&schema.Ref{Module: "test", Name: "Test"}, sch, map[string]any{ @@ -104,6 +126,10 @@ func TestTransformToAliasedFields(t *testing.T) { "optional": map[string]any{ "waz": "value", }, + "typeEnum": map[string]any{ + "name": "A", + "value": map[string]any{"waz": "value"}, + }, }) expected := map[string]any{ "bar": "value", @@ -123,6 +149,10 @@ func TestTransformToAliasedFields(t *testing.T) { "optional": map[string]any{ "foo": "value", }, + "typeEnum": map[string]any{ + "name": "A", + "value": map[string]any{"foo": "value"}, + }, } assert.NoError(t, err) assert.Equal(t, expected, actual) diff --git a/backend/controller/ingress/ingress.go b/backend/controller/ingress/ingress.go index df013585ce..c371a0e49c 100644 --- a/backend/controller/ingress/ingress.go +++ b/backend/controller/ingress/ingress.go @@ -194,6 +194,8 @@ func validateValue(fieldType schema.Type, path path, value any, sch *schema.Sche typeMatches = true } case *schema.Enum: + var inputName any + inputName = value for _, v := range d.Variants { switch t := v.Value.(type) { case *schema.StringValue: @@ -211,11 +213,36 @@ func validateValue(fieldType schema.Type, path path, value any, sch *schema.Sche } } case *schema.TypeValue: - //TODO: Implement + if reqVariant, ok := value.(map[string]any); ok { + vName, ok := reqVariant["name"] + if !ok { + return fmt.Errorf(`missing name field in enum type %q: expected structure is `+ + "{\"name\": \"\", \"value\": }", value) + } + vNameStr, ok := vName.(string) + if !ok { + return fmt.Errorf(`invalid type for enum %q; name field must be a string, was %T`, + fieldType, vName) + } + inputName = fmt.Sprintf("%q", vNameStr) + + vValue, ok := reqVariant["value"] + if !ok { + return fmt.Errorf(`missing value field in enum type %q: expected structure is `+ + "{\"name\": \"\", \"value\": }", value) + } + + if v.Name == vNameStr { + return validateValue(t.Value, path, vValue, sch) + } + } else { + return fmt.Errorf(`malformed enum type %s: expected structure is `+ + "{\"name\": \"\", \"value\": }", path) + } } } if !typeMatches { - return fmt.Errorf("%s is not a valid variant of enum %s", value, fieldType) + return fmt.Errorf("%s is not a valid variant of enum %s", inputName, fieldType) } case *schema.Config, *schema.Database, *schema.Secret, *schema.Verb, *schema.FSM: diff --git a/backend/controller/ingress/ingress_test.go b/backend/controller/ingress/ingress_test.go index 51e929537e..f9469af477 100644 --- a/backend/controller/ingress/ingress_test.go +++ b/backend/controller/ingress/ingress_test.go @@ -288,6 +288,8 @@ func TestValueForData(t *testing.T) { {&schema.Array{Element: &schema.String{}}, []byte(`["test1", "test2"]`), []any{"test1", "test2"}}, {&schema.Map{Key: &schema.String{}, Value: &schema.String{}}, []byte(`{"key1": "value1", "key2": "value2"}`), obj{"key1": "value1", "key2": "value2"}}, {&schema.Ref{Module: "test", Name: "Test"}, []byte(`{"intValue": 10.0}`), obj{"intValue": 10.0}}, + // type enum + {&schema.Ref{Module: "test", Name: "TypeEnum"}, []byte(`{"name": "A", "value": "hello"}`), obj{"name": "A", "value": "hello"}}, } for _, test := range tests { @@ -319,6 +321,13 @@ func TestEnumValidation(t *testing.T) { {Name: "GreenInt", Value: &schema.IntValue{Value: 2}}, }, }, + &schema.Enum{ + Name: "TypeEnum", + Variants: []*schema.EnumVariant{ + {Name: "String", Value: &schema.TypeValue{Value: &schema.String{}}}, + {Name: "List", Value: &schema.TypeValue{Value: &schema.Array{Element: &schema.String{}}}}, + }, + }, &schema.Data{ Name: "StringEnumRequest", Fields: []*schema.Field{ @@ -339,6 +348,14 @@ func TestEnumValidation(t *testing.T) { }}, }, }, + &schema.Data{ + Name: "TypeEnumRequest", + Fields: []*schema.Field{ + {Name: "message", Type: &schema.Optional{ + Type: &schema.Ref{Name: "TypeEnum", Module: "test"}, + }}, + }, + }, }}, }, } @@ -354,6 +371,16 @@ func TestEnumValidation(t *testing.T) { {&schema.Ref{Name: "OptionalEnumRequest", Module: "test"}, obj{"message": "Red"}, ""}, {&schema.Ref{Name: "StringEnumRequest", Module: "test"}, obj{"message": "akxznc"}, "akxznc is not a valid variant of enum test.Color"}, + {&schema.Ref{Name: "TypeEnumRequest", Module: "test"}, obj{"message": obj{"name": "String", "value": "Hello"}}, ""}, + {&schema.Ref{Name: "TypeEnumRequest", Module: "test"}, obj{"message": obj{"name": "String", "value": `["test1", "test2"]`}}, ""}, + {&schema.Ref{Name: "TypeEnumRequest", Module: "test"}, obj{"message": obj{"name": "String", "value": 0}}, + "test.TypeEnumRequest.message has wrong type, expected String found int"}, + {&schema.Ref{Name: "TypeEnumRequest", Module: "test"}, obj{"message": obj{"name": "Invalid", "value": 0}}, + "\"Invalid\" is not a valid variant of enum test.TypeEnum"}, + {&schema.Ref{Name: "TypeEnumRequest", Module: "test"}, obj{"message": obj{"name": 0, "value": 0}}, + `invalid type for enum "test.TypeEnum"; name field must be a string, was int`}, + {&schema.Ref{Name: "TypeEnumRequest", Module: "test"}, obj{"message": "Hello"}, + `malformed enum type test.TypeEnumRequest.message: expected structure is {"name": "", "value": }`}, } for _, test := range tests { diff --git a/backend/controller/ingress/request.go b/backend/controller/ingress/request.go index 369723fd5b..985dd73635 100644 --- a/backend/controller/ingress/request.go +++ b/backend/controller/ingress/request.go @@ -103,7 +103,9 @@ func extractHTTPRequestBody(route *dal.IngressRoute, r *http.Request, ref *schem } if ref, ok := bodyField.Type.(*schema.Ref); ok { - return buildRequestMap(route, r, ref, sch) + if _, ok := sch.ResolveRef(ref).(*schema.Data); ok { + return buildRequestMap(route, r, ref, sch) + } } bodyData, err := readRequestBody(r) diff --git a/backend/protos/xyz/block/ftl/v1/schema/schema.proto b/backend/protos/xyz/block/ftl/v1/schema/schema.proto index 7875728da8..741ee1e209 100644 --- a/backend/protos/xyz/block/ftl/v1/schema/schema.proto +++ b/backend/protos/xyz/block/ftl/v1/schema/schema.proto @@ -231,10 +231,6 @@ message StringValue { string value = 2; } -message SumTypeVariants { - repeated string value = 1; -} - message Time { optional Position pos = 1; } @@ -261,11 +257,6 @@ message TypeParameter { string name = 2; } -message TypeRegistry { - map schemaTypes = 1; - map sumTypes = 2; -} - message TypeValue { optional Position pos = 1; Type value = 2; diff --git a/backend/schema/jsonschema.go b/backend/schema/jsonschema.go index af3b00f3f2..a2d640ae77 100644 --- a/backend/schema/jsonschema.go +++ b/backend/schema/jsonschema.go @@ -84,11 +84,29 @@ func nodeToJSSchema(node Node, refs map[RefKey]*Ref) *jsonschema.Schema { schema := &jsonschema.Schema{ Description: jsComments(node.Comments), } - values := make([]any, len(node.Variants)) - for i, v := range node.Variants { - values[i] = v.Value.GetValue() + if node.IsValueEnum() { + values := make([]any, len(node.Variants)) + for i, v := range node.Variants { + values[i] = v.Value.GetValue() + } + return schema.WithEnum(values...) + } + + variants := make([]jsonschema.SchemaOrBool, 0, len(node.Variants)) + for _, v := range node.Variants { + obj := jsonschema.Object + str := jsonschema.String + variantSch := &jsonschema.Schema{ + Description: jsComments(v.Comments), + Type: &jsonschema.Type{SimpleTypes: &obj}, + Properties: map[string]jsonschema.SchemaOrBool{}, + AdditionalProperties: jsBool(false), + } + variantSch.Properties["name"] = jsonschema.SchemaOrBool{TypeObject: &jsonschema.Schema{Type: &jsonschema.Type{SimpleTypes: &str}}} + variantSch.Properties["value"] = jsonschema.SchemaOrBool{TypeObject: nodeToJSSchema(v.Value.(*TypeValue).schemaValueType(), refs)} //nolint:forcetypeassert + variants = append(variants, jsonschema.SchemaOrBool{TypeObject: variantSch}) } - return schema.WithEnum(values...) + return schema.WithOneOf(variants...) case *Int: st := jsonschema.Integer diff --git a/backend/schema/jsonschema_test.go b/backend/schema/jsonschema_test.go index ec7fa6a570..ad9e58954f 100644 --- a/backend/schema/jsonschema_test.go +++ b/backend/schema/jsonschema_test.go @@ -32,6 +32,7 @@ var jsonSchemaSample = &Schema{ {Name: "keyValue", Type: &Ref{Module: "foo", Name: "Generic", TypeParameters: []Type{&String{}, &Int{}}}}, {Name: "stringEnumRef", Type: &Ref{Module: "foo", Name: "StringEnum"}}, {Name: "intEnumRef", Type: &Ref{Module: "foo", Name: "IntEnum"}}, + {Name: "typeEnumRef", Type: &Ref{Module: "foo", Name: "TypeEnum"}}, }, }, &Data{ @@ -47,6 +48,7 @@ var jsonSchemaSample = &Schema{ }, &Enum{ Name: "StringEnum", + Type: &String{}, Variants: []*EnumVariant{ {Name: "A", Value: &StringValue{Value: "A"}}, {Name: "B", Value: &StringValue{Value: "B"}}, @@ -54,11 +56,19 @@ var jsonSchemaSample = &Schema{ }, &Enum{ Name: "IntEnum", + Type: &Int{}, Variants: []*EnumVariant{ {Name: "Zero", Value: &IntValue{Value: 0}}, {Name: "One", Value: &IntValue{Value: 1}}, }, }, + &Enum{ + Name: "TypeEnum", + Variants: []*EnumVariant{ + {Name: "StringVariant", Value: &TypeValue{Value: &String{}}}, + {Name: "IntVariant", Value: &TypeValue{Value: &Int{}}}, + }, + }, }}, {Name: "bar", Decls: []Decl{ &Data{Name: "Bar", Fields: []*Field{{Name: "bar", Type: &String{}}}}, @@ -89,7 +99,8 @@ func TestDataToJSONSchema(t *testing.T) { "any", "keyValue", "stringEnumRef", - "intEnumRef" + "intEnumRef", + "typeEnumRef" ], "additionalProperties": false, "definitions": { @@ -144,6 +155,34 @@ func TestDataToJSONSchema(t *testing.T) { "A", "B" ] + }, + "foo.TypeEnum": { + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "type": "object" + } + ] } }, "properties": { @@ -245,6 +284,9 @@ func TestDataToJSONSchema(t *testing.T) { "time": { "type": "string", "format": "date-time" + }, + "typeEnumRef": { + "$ref": "#/definitions/foo.TypeEnum" } }, "type": "object" @@ -270,7 +312,8 @@ func TestJSONSchemaValidation(t *testing.T) { "any": [{"name": "Name"}, "string", 1, 1.23, true, "2018-11-13T20:20:39+00:00", ["one"], {"one": 2}, null], "keyValue": {"key": "string", "value": 1}, "stringEnumRef": "A", - "intEnumRef": 0 + "intEnumRef": 0, + "typeEnumRef": {"name": "IntVariant", "value": 0} } ` @@ -307,7 +350,8 @@ func TestInvalidEnumValidation(t *testing.T) { "any": [{"name": "Name"}, "string", 1, 1.23, true, "2018-11-13T20:20:39+00:00", ["one"], {"one": 2}, null], "keyValue": {"key": "string", "value": 1}, "stringEnumRef": "B", - "intEnumRef": 3 + "intEnumRef": 3, + "typeEnumRef": {"name": "IntVariant", "value": 0} } ` diff --git a/backend/schema/protobuf.go b/backend/schema/protobuf.go index 79a9028301..4b4d59375f 100644 --- a/backend/schema/protobuf.go +++ b/backend/schema/protobuf.go @@ -23,7 +23,6 @@ func ProtobufSchema() string { messages := map[string]string{} generateMessage(reflect.TypeFor[Schema](), messages) generateMessage(reflect.TypeFor[ErrorList](), messages) - generateMessage(reflect.TypeFor[TypeRegistry](), messages) keys := maps.Keys(messages) slices.Sort(keys) w := &strings.Builder{} diff --git a/backend/schema/type_registry.go b/backend/schema/type_registry.go deleted file mode 100644 index ba7bf3f42e..0000000000 --- a/backend/schema/type_registry.go +++ /dev/null @@ -1,72 +0,0 @@ -package schema - -import ( - "context" - - schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema" -) - -type contextKeyTypeRegistry struct{} - -// ContextWithTypeRegistry adds a type registry to the given context. -func ContextWithTypeRegistry(ctx context.Context, r *schemapb.TypeRegistry) context.Context { - return context.WithValue(ctx, contextKeyTypeRegistry{}, r) -} - -// TypeRegistry is a registry of types that can be resolved to a schema type at runtime. -// It also records sum types and their variants, for use in encoding and decoding. -type TypeRegistry struct { - // SchemaTypes associates a type name with a schema type. - SchemaTypes map[string]Type `protobuf:"1"` - // SumTypes associates a sum type discriminator type name with its variant type names. - SumTypes map[string]SumTypeVariants `protobuf:"2"` -} - -// NewTypeRegistry creates a new type registry. -// The type registry is used to instantiate types by their qualified name at runtime. -func NewTypeRegistry() *TypeRegistry { - return &TypeRegistry{ - SchemaTypes: make(map[string]Type), - SumTypes: make(map[string]SumTypeVariants), - } -} - -// RegisterSumType registers a Go sum type with the type registry. Sum types are represented as enums in the -// FTL schema. -func (t *TypeRegistry) RegisterSumType(discriminator string, variants map[string]Type) { - var values []string - for name, vt := range variants { - values = append(values, name) - t.SchemaTypes[name] = vt - } - t.SumTypes[discriminator] = SumTypeVariants{Value: values} -} - -func (t *TypeRegistry) ToProto() *schemapb.TypeRegistry { - typespb := make(map[string]*schemapb.Type, len(t.SchemaTypes)) - for k, v := range t.SchemaTypes { - typespb[k] = typeToProto(v) - } - - return &schemapb.TypeRegistry{ - SumTypes: sumTypeVariantsToProto(t.SumTypes), - SchemaTypes: typespb, - } -} - -type SumTypeVariants struct { - // Value is a list of variant names for the sum type. - Value []string `protobuf:"1"` -} - -func (s *SumTypeVariants) ToProto() *schemapb.SumTypeVariants { - return &schemapb.SumTypeVariants{Value: s.Value} -} - -func sumTypeVariantsToProto(v map[string]SumTypeVariants) map[string]*schemapb.SumTypeVariants { - out := make(map[string]*schemapb.SumTypeVariants, len(v)) - for k, v := range v { - out[k] = v.ToProto() - } - return out -} diff --git a/backend/schema/validate.go b/backend/schema/validate.go index dca2d9d36a..33e021aece 100644 --- a/backend/schema/validate.go +++ b/backend/schema/validate.go @@ -166,13 +166,20 @@ func ValidateModuleInSchema(schema *Schema, m optional.Option[*Module]) (*Schema } case *Enum: - if n.Type != nil { + if n.IsValueEnum() { for _, v := range n.Variants { if reflect.TypeOf(v.Value.schemaValueType()) != reflect.TypeOf(n.Type) { merr = append(merr, errorf(v, "enum variant %q of type %s cannot have a value of "+ "type %q", v.Name, n.Type, v.Value.schemaValueType())) } } + } else { + for _, v := range n.Variants { + if _, ok := v.Value.(*TypeValue); !ok { + merr = append(merr, errorf(v, "type enum variant %q value must be a type, was %T", + v.Name, n)) + } + } } return next() @@ -550,8 +557,13 @@ func validateIngressRequestOrResponse(scopes Scopes, module *Module, n *Verb, re return } body = bodySym.Symbol - switch bodySym.Symbol.(type) { + switch t := bodySym.Symbol.(type) { case *Bytes, *String, *Data, *Unit, *Float, *Int, *Bool, *Map, *Array: // Valid HTTP response payload types. + case *Enum: + // Type enums are valid but value enums are not. + if t.IsValueEnum() { + merr = append(merr, errorf(r, "ingress verb %s: %s type %s must have a body of bytes, string, data structure, unit, float, int, bool, map, or array not enum %s", n.Name, reqOrResp, r, t.Name)) + } default: merr = append(merr, errorf(r, "ingress verb %s: %s type %s must have a body of bytes, string, data structure, unit, float, int, bool, map, or array not %s", n.Name, reqOrResp, r, bodySym.Symbol)) } diff --git a/buildengine/build_go_test.go b/buildengine/build_go_test.go index d59db635bf..11ac59b896 100644 --- a/buildengine/build_go_test.go +++ b/buildengine/build_go_test.go @@ -261,6 +261,21 @@ func TestGeneratedTypeRegistry(t *testing.T) { {Name: "B", Value: &schema.TypeValue{Value: &schema.String{}}}, }, }, + &schema.Enum{ + Name: "SecondTypeEnum", + Export: true, + Variants: []*schema.EnumVariant{ + {Name: "One", Value: &schema.TypeValue{Value: &schema.Int{}}}, + {Name: "Two", Value: &schema.TypeValue{Value: &schema.String{}}}, + }, + }, + &schema.Data{ + Name: "TransitiveTypeEnum", + Export: true, + Fields: []*schema.Field{ + {Name: "TypeEnumRef", Type: &schema.Ref{Name: "SecondTypeEnum", Module: "another"}}, + }, + }, }}, }, } diff --git a/buildengine/testdata/projects/another/another.go b/buildengine/testdata/projects/another/another.go index b31e16d039..29db36055a 100644 --- a/buildengine/testdata/projects/another/another.go +++ b/buildengine/testdata/projects/another/another.go @@ -20,6 +20,22 @@ type B string func (B) tag() {} +//ftl:enum export +type SecondTypeEnum interface{ typeEnum() } + +type One int + +func (One) typeEnum() {} + +type Two string + +func (Two) typeEnum() {} + +//ftl:data export +type TransitiveTypeEnum struct { + TypeEnumRef SecondTypeEnum +} + type EchoRequest struct { Name ftl.Option[string] `json:"name"` } diff --git a/buildengine/testdata/projects/other/other.go b/buildengine/testdata/projects/other/other.go index 2d10057801..4f2c133f1a 100644 --- a/buildengine/testdata/projects/other/other.go +++ b/buildengine/testdata/projects/other/other.go @@ -73,8 +73,9 @@ type B EchoRequest func (B) tag2() {} type EchoRequest struct { - Name ftl.Option[string] `json:"name"` - ExternalSumType another.TypeEnum + Name ftl.Option[string] `json:"name"` + ExternalSumType another.TypeEnum + ExternalNestedSumType another.TransitiveTypeEnum } type EchoResponse struct { diff --git a/buildengine/testdata/type_registry_main.go b/buildengine/testdata/type_registry_main.go index dadfad7ef6..cb60716367 100644 --- a/buildengine/testdata/type_registry_main.go +++ b/buildengine/testdata/type_registry_main.go @@ -6,7 +6,6 @@ import ( "reflect" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" - "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/common/plugin" "github.com/TBD54566975/ftl/go-runtime/ftl/typeregistry" "github.com/TBD54566975/ftl/go-runtime/server" @@ -21,25 +20,20 @@ func main() { ) ctx := context.Background() - goTypeRegistry := typeregistry.NewTypeRegistry() - schemaTypeRegistry := schema.NewTypeRegistry() - goTypeRegistry.RegisterSumType(reflect.TypeFor[another.TypeEnum](), map[string]reflect.Type{ + tr := typeregistry.NewTypeRegistry() + tr.RegisterSumType(reflect.TypeFor[another.SecondTypeEnum](), map[string]reflect.Type{ + "One": reflect.TypeFor[another.One](), + "Two": reflect.TypeFor[another.Two](), + }) + tr.RegisterSumType(reflect.TypeFor[another.TypeEnum](), map[string]reflect.Type{ "A": reflect.TypeFor[another.A](), "B": reflect.TypeFor[another.B](), }) - schemaTypeRegistry.RegisterSumType("another.TypeEnum", map[string]schema.Type{ - "A": &schema.Int{}, - "B": &schema.String{}, - }) - goTypeRegistry.RegisterSumType(reflect.TypeFor[other.SecondTypeEnum](), map[string]reflect.Type{ + tr.RegisterSumType(reflect.TypeFor[other.SecondTypeEnum](), map[string]reflect.Type{ "A": reflect.TypeFor[other.A](), "B": reflect.TypeFor[other.B](), }) - schemaTypeRegistry.RegisterSumType("other.SecondTypeEnum", map[string]schema.Type{ - "A": &schema.String{}, - "B": &schema.Ref{Module: "other", Name: "B"}, - }) - goTypeRegistry.RegisterSumType(reflect.TypeFor[other.TypeEnum](), map[string]reflect.Type{ + tr.RegisterSumType(reflect.TypeFor[other.TypeEnum](), map[string]reflect.Type{ "Bool": reflect.TypeFor[other.Bool](), "Bytes": reflect.TypeFor[other.Bytes](), "Float": reflect.TypeFor[other.Float](), @@ -52,21 +46,7 @@ func main() { "Option": reflect.TypeFor[other.Option](), "Unit": reflect.TypeFor[other.Unit](), }) - schemaTypeRegistry.RegisterSumType("other.TypeEnum", map[string]schema.Type{ - "Bool": &schema.Bool{}, - "Bytes": &schema.Bytes{}, - "Float": &schema.Float{}, - "Int": &schema.Int{}, - "Time": &schema.Time{}, - "List": &schema.Array{Element: &schema.String{}}, - "Map": &schema.Map{Key: &schema.String{}, Value: &schema.String{}}, - "String": &schema.String{}, - "Struct": &schema.Ref{Module: "other", Name: "Struct"}, - "Option": &schema.Optional{Type: &schema.String{}}, - "Unit": &schema.Unit{}, - }) - ctx = typeregistry.ContextWithTypeRegistry(ctx, goTypeRegistry) - ctx = schema.ContextWithTypeRegistry(ctx, schemaTypeRegistry.ToProto()) + ctx = typeregistry.ContextWithTypeRegistry(ctx, tr) plugin.Start(ctx, "other", verbConstructor, ftlv1connect.VerbServiceName, ftlv1connect.NewVerbServiceHandler) } diff --git a/frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts b/frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts index f067a91e4e..cc6fdf3a4d 100644 --- a/frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts +++ b/frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts @@ -1802,43 +1802,6 @@ export class StringValue extends Message { } } -/** - * @generated from message xyz.block.ftl.v1.schema.SumTypeVariants - */ -export class SumTypeVariants extends Message { - /** - * @generated from field: repeated string value = 1; - */ - value: string[] = []; - - constructor(data?: PartialMessage) { - super(); - proto3.util.initPartial(data, this); - } - - static readonly runtime: typeof proto3 = proto3; - static readonly typeName = "xyz.block.ftl.v1.schema.SumTypeVariants"; - static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "value", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, - ]); - - static fromBinary(bytes: Uint8Array, options?: Partial): SumTypeVariants { - return new SumTypeVariants().fromBinary(bytes, options); - } - - static fromJson(jsonValue: JsonValue, options?: Partial): SumTypeVariants { - return new SumTypeVariants().fromJson(jsonValue, options); - } - - static fromJsonString(jsonString: string, options?: Partial): SumTypeVariants { - return new SumTypeVariants().fromJsonString(jsonString, options); - } - - static equals(a: SumTypeVariants | PlainMessage | undefined, b: SumTypeVariants | PlainMessage | undefined): boolean { - return proto3.util.equals(SumTypeVariants, a, b); - } -} - /** * @generated from message xyz.block.ftl.v1.schema.Time */ @@ -2039,49 +2002,6 @@ export class TypeParameter extends Message { } } -/** - * @generated from message xyz.block.ftl.v1.schema.TypeRegistry - */ -export class TypeRegistry extends Message { - /** - * @generated from field: map schemaTypes = 1; - */ - schemaTypes: { [key: string]: Type } = {}; - - /** - * @generated from field: map sumTypes = 2; - */ - sumTypes: { [key: string]: SumTypeVariants } = {}; - - constructor(data?: PartialMessage) { - super(); - proto3.util.initPartial(data, this); - } - - static readonly runtime: typeof proto3 = proto3; - static readonly typeName = "xyz.block.ftl.v1.schema.TypeRegistry"; - static readonly fields: FieldList = proto3.util.newFieldList(() => [ - { no: 1, name: "schemaTypes", kind: "map", K: 9 /* ScalarType.STRING */, V: {kind: "message", T: Type} }, - { no: 2, name: "sumTypes", kind: "map", K: 9 /* ScalarType.STRING */, V: {kind: "message", T: SumTypeVariants} }, - ]); - - static fromBinary(bytes: Uint8Array, options?: Partial): TypeRegistry { - return new TypeRegistry().fromBinary(bytes, options); - } - - static fromJson(jsonValue: JsonValue, options?: Partial): TypeRegistry { - return new TypeRegistry().fromJson(jsonValue, options); - } - - static fromJsonString(jsonString: string, options?: Partial): TypeRegistry { - return new TypeRegistry().fromJsonString(jsonString, options); - } - - static equals(a: TypeRegistry | PlainMessage | undefined, b: TypeRegistry | PlainMessage | undefined): boolean { - return proto3.util.equals(TypeRegistry, a, b); - } -} - /** * @generated from message xyz.block.ftl.v1.schema.TypeValue */ diff --git a/go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go b/go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go index f72dff2f6f..26056f4e86 100644 --- a/go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go +++ b/go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go @@ -7,9 +7,6 @@ import ( "reflect" {{ end }} "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" -{{- if .SumTypes }} - "github.com/TBD54566975/ftl/backend/schema" -{{- end }} "github.com/TBD54566975/ftl/common/plugin" {{- if .SumTypes }} "github.com/TBD54566975/ftl/go-runtime/ftl/typeregistry" @@ -38,24 +35,17 @@ func main() { {{- if .SumTypes}} - goTypeRegistry := typeregistry.NewTypeRegistry() - schemaTypeRegistry := schema.NewTypeRegistry() + tr := typeregistry.NewTypeRegistry() {{- end}} {{- range .SumTypes}} - goTypeRegistry.RegisterSumType(reflect.TypeFor[{{.Discriminator}}](), map[string]reflect.Type{ + tr.RegisterSumType(reflect.TypeFor[{{.Discriminator}}](), map[string]reflect.Type{ {{- range .Variants}} "{{.Name}}": reflect.TypeFor[{{.Type}}](), {{- end}} }) - schemaTypeRegistry.RegisterSumType("{{.Discriminator}}", map[string]schema.Type{ - {{- range .Variants}} - "{{.Name}}": {{schemaType .SchemaType}}, - {{- end}} - }) {{- end}} {{- if .SumTypes}} - ctx = typeregistry.ContextWithTypeRegistry(ctx, goTypeRegistry) - ctx = schema.ContextWithTypeRegistry(ctx, schemaTypeRegistry.ToProto()) + ctx = typeregistry.ContextWithTypeRegistry(ctx, tr) {{- end}} plugin.Start(ctx, "{{.Name}}", verbConstructor, ftlv1connect.VerbServiceName, ftlv1connect.NewVerbServiceHandler) diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index b35f3d8b58..42f685d155 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -188,7 +188,7 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans Name: main.Name, Verbs: goVerbs, Replacements: replacements, - SumTypes: getSumTypes(pr.EnumRefs, main, sch, pr.NativeNames), + SumTypes: getSumTypes(main, sch, pr.NativeNames), }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { return err } @@ -502,7 +502,7 @@ func writeSchemaErrors(config moduleconfig.ModuleConfig, errors []*schema.Error) return os.WriteFile(filepath.Join(config.AbsDeployDir(), config.Errors), elBytes, 0600) } -func getSumTypes(enumRefs []*schema.Ref, module *schema.Module, sch *schema.Schema, nativeNames NativeNames) []goSumType { +func getSumTypes(module *schema.Module, sch *schema.Schema, nativeNames NativeNames) []goSumType { sumTypes := make(map[string]goSumType) for _, d := range module.Decls { if e, ok := d.(*schema.Enum); ok && !e.IsValueEnum() { @@ -523,25 +523,19 @@ func getSumTypes(enumRefs []*schema.Ref, module *schema.Module, sch *schema.Sche } // register sum types from other modules - for _, ref := range enumRefs { - if ref.Module == module.Name { - continue + for _, e := range getExternalTypeEnums(module, sch) { + variants := make([]goSumTypeVariant, 0, len(e.resolved.Variants)) + for _, v := range e.resolved.Variants { + variants = append(variants, goSumTypeVariant{ //nolint:forcetypeassert + Name: v.Name, + Type: e.ref.Module + "." + v.Name, + SchemaType: v.Value.(*schema.TypeValue).Value, + }) } - resolved := sch.ResolveRef(ref) - if e, ok := resolved.(*schema.Enum); ok && !e.IsValueEnum() { - variants := make([]goSumTypeVariant, 0, len(e.Variants)) - for _, v := range e.Variants { - variants = append(variants, goSumTypeVariant{ //nolint:forcetypeassert - Name: v.Name, - Type: ref.Module + "." + v.Name, - SchemaType: v.Value.(*schema.TypeValue).Value, - }) - } - stFqName := ref.Module + "." + e.Name - sumTypes[stFqName] = goSumType{ - Discriminator: stFqName, - Variants: variants, - } + stFqName := e.ref.Module + "." + e.ref.Name + sumTypes[e.ref.ToRefKey().String()] = goSumType{ + Discriminator: stFqName, + Variants: variants, } } out := gomaps.Values(sumTypes) @@ -550,3 +544,32 @@ func getSumTypes(enumRefs []*schema.Ref, module *schema.Module, sch *schema.Sche }) return out } + +type externalEnum struct { + ref *schema.Ref + resolved *schema.Enum +} + +// getExternalTypeEnums resolve all type enum references in the full schema +func getExternalTypeEnums(module *schema.Module, sch *schema.Schema) []externalEnum { + combinedSch := schema.Schema{ + Modules: append(sch.Modules, module), + } + var externalTypeEnums []externalEnum + err := schema.Visit(&combinedSch, func(n schema.Node, next func() error) error { + if ref, ok := n.(*schema.Ref); ok && ref.Module != "" && ref.Module != module.Name { + decl := sch.ResolveRef(ref) + if e, ok := decl.(*schema.Enum); ok && !e.IsValueEnum() { + externalTypeEnums = append(externalTypeEnums, externalEnum{ + ref: ref, + resolved: e, + }) + } + } + return next() + }) + if err != nil { + panic(fmt.Sprintf("failed to resolve external type enums schema: %v", err)) + } + return externalTypeEnums +} diff --git a/go-runtime/compile/schema.go b/go-runtime/compile/schema.go index e620097c19..f42ccacf0a 100644 --- a/go-runtime/compile/schema.go +++ b/go-runtime/compile/schema.go @@ -50,7 +50,6 @@ type NativeNames map[schema.Node]string type enums map[string]*schema.Enum type enumInterfaces map[string]*types.Interface -type enumRefs map[string]*schema.Ref func noEndColumnErrorf(pos token.Pos, format string, args ...interface{}) *schema.Error { return tokenErrorf(pos, "", format, args...) @@ -99,9 +98,6 @@ func (e errorSet) addAll(errs ...*schema.Error) { type ParseResult struct { Module *schema.Module NativeNames NativeNames - // EnumRefs contains any external enums referenced by this module. The refs will be resolved and any type enums - // will be registered to the `ftl.TypeRegistry` and provided in the context for this module. - EnumRefs []*schema.Ref // Errors contains schema validation errors encountered during parsing. Errors []*schema.Error } @@ -120,7 +116,6 @@ func ExtractModuleSchema(dir string) (optional.Option[ParseResult], error) { return optional.None[ParseResult](), fmt.Errorf("no packages found in %q, does \"go mod tidy\" need to be run?", dir) } nativeNames := NativeNames{} - eRefs := enumRefs{} // Find module name module := &schema.Module{} merr := []error{} @@ -137,7 +132,7 @@ func ExtractModuleSchema(dir string) (optional.Option[ParseResult], error) { } } pctx := &parseContext{pkg: pkg, pkgs: pkgs, module: module, nativeNames: NativeNames{}, enums: enums{}, - enumInterfaces: enumInterfaces{}, enumRefs: eRefs, errors: errorSet{}} + enumInterfaces: enumInterfaces{}, errors: errorSet{}} for _, file := range pkg.Syntax { err := goast.Visit(file, func(node ast.Node, next func() error) (err error) { switch node := node.(type) { @@ -188,7 +183,6 @@ func ExtractModuleSchema(dir string) (optional.Option[ParseResult], error) { return optional.Some(ParseResult{ NativeNames: nativeNames, Module: module, - EnumRefs: maps.Values(eRefs), }), schema.ValidateModule(module) } @@ -1055,7 +1049,7 @@ func visitType(pctx *parseContext, pos token.Pos, tnode types.Type, isExported b if _, ok := visitType(pctx, pos, named.Underlying(), isExported).Get(); !ok { return optional.None[schema.Type]() } - enumRef, doneWithVisit := visitEnumRef(pctx, pos, named) + enumRef, doneWithVisit := parseEnumRef(pctx, pos, named) if doneWithVisit { return enumRef } @@ -1122,7 +1116,7 @@ func visitType(pctx *parseContext, pos token.Pos, tnode types.Type, isExported b return optional.Some[schema.Type](&schema.Any{Pos: goPosToSchemaPos(pos)}) } if named, ok := tnode.(*types.Named); ok { - enumRef, doneWithVisit := visitEnumRef(pctx, pos, named) + enumRef, doneWithVisit := parseEnumRef(pctx, pos, named) if doneWithVisit { return enumRef } @@ -1134,19 +1128,7 @@ func visitType(pctx *parseContext, pos token.Pos, tnode types.Type, isExported b } } -func visitEnumRef(pctx *parseContext, pos token.Pos, named *types.Named) (optional.Option[schema.Type], bool) { - enumRef, doneWithVisit := visitEnumType(pctx, pos, named) - if er, ok := enumRef.Get(); ok { - refModuleName, ok := ftlModuleFromGoModule(named.Obj().Pkg().Path()).Get() - if !ok { - refModuleName = named.Obj().Pkg().Path() - } - pctx.enumRefs[refModuleName+"."+named.Obj().Name()] = er.(*schema.Ref) //nolint:forcetypeassert - } - return enumRef, doneWithVisit -} - -func visitEnumType(pctx *parseContext, pos token.Pos, named *types.Named) (optional.Option[schema.Type], bool) { +func parseEnumRef(pctx *parseContext, pos token.Pos, named *types.Named) (optional.Option[schema.Type], bool) { if named.Obj().Pkg() == nil { return optional.None[schema.Type](), false } @@ -1266,7 +1248,6 @@ type parseContext struct { nativeNames NativeNames enums enums enumInterfaces enumInterfaces - enumRefs enumRefs activeVerb *schema.Verb errors errorSet } diff --git a/integration/integration_test.go b/integration/integration_test.go index 6abf4f70d8..84f0c49e1b 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -233,6 +233,12 @@ func TestHttpIngress(t *testing.T) { assert.Equal(t, jsonData(t, []obj{{"item": "a"}, {"item": "b"}}), resp.bodyBytes) return nil }), + httpCall(http.MethodGet, "/typeenum", jsonData(t, obj{"name": "A", "value": "hello"}), func(resp *httpResponse) error { + assert.Equal(t, 200, resp.status) + assert.Equal(t, []string{"application/json; charset=utf-8"}, resp.headers["Content-Type"]) + assert.Equal(t, jsonData(t, obj{"name": "A", "value": "hello"}), resp.bodyBytes) + return nil + }), ) } diff --git a/integration/testdata/go/httpingress/httpingress.go b/integration/testdata/go/httpingress/httpingress.go index 5e2361f649..754abb273c 100644 --- a/integration/testdata/go/httpingress/httpingress.go +++ b/integration/testdata/go/httpingress/httpingress.go @@ -23,6 +23,19 @@ type GetResponse struct { Nested Nested `json:"nested"` } +//ftl:enum export +type SumType interface { + tag() +} + +type A string + +func (A) tag() {} + +type B []string + +func (B) tag() {} + //ftl:ingress http GET /users/{userId}/posts/{postId} func Get(ctx context.Context, req builtin.HttpRequest[GetRequest]) (builtin.HttpResponse[GetResponse, string], error) { return builtin.HttpResponse[GetResponse, string]{ @@ -149,3 +162,8 @@ func ArrayData(ctx context.Context, req builtin.HttpRequest[[]ArrayType]) (built Body: ftl.Some(req.Body), }, nil } + +//ftl:ingress http GET /typeenum +func TypeEnum(ctx context.Context, req builtin.HttpRequest[SumType]) (builtin.HttpResponse[SumType, string], error) { + return builtin.HttpResponse[SumType, string]{Body: ftl.Some(req.Body)}, nil +}