feat: add support for a "Bytes" type (#743)

In Go this is []byte, in Kotlin it's ByteArray. This is a prerequisite for supporting raw HTTP request/response ingress routes. PS. Also did some linter fixes forced on me.
block · Jan 8, 2024 · 76d835c · 76d835c
1 parent da5c653
commit 76d835c
Show file tree

Hide file tree

Showing 22 changed files with 627 additions and 432 deletions.
diff --git a/backend/common/bind/bind_allocator.go b/backend/common/bind/bind_allocator.go
@@ -1,7 +1,6 @@
 package bind
 
 import (
-	"fmt"
 	"net"
 	"net/url"
 	"strconv"
@@ -44,7 +43,7 @@ func (b *BindAllocator) Next() *url.URL {
 		_ = l.Close()
 
 		newURL := *b.baseURL
-		newURL.Host = net.JoinHostPort(b.baseURL.Hostname(), fmt.Sprintf("%d", b.port.Load()))
+		newURL.Host = net.JoinHostPort(b.baseURL.Hostname(), strconv.Itoa(int(b.port.Load())))
 		return &newURL
 	}
 }
diff --git a/backend/common/plugin/spawn.go b/backend/common/plugin/spawn.go
@@ -24,7 +24,7 @@ const pluginRetryDelay = time.Millisecond * 100
 
 // PingableClient is a gRPC client that can be pinged.
 type PingableClient interface {
-	Ping(context.Context, *connect.Request[ftlv1.PingRequest]) (*connect.Response[ftlv1.PingResponse], error)
+	Ping(ctx context.Context, req *connect.Request[ftlv1.PingRequest]) (*connect.Response[ftlv1.PingResponse], error)
 }
 
 type pluginOptions struct {

diff --git a/backend/common/rpc/rpc.go b/backend/common/rpc/rpc.go
@@ -59,7 +59,7 @@ var (
 )
 
 type Pingable interface {
-	Ping(context.Context, *connect.Request[ftlv1.PingRequest]) (*connect.Response[ftlv1.PingResponse], error)
+	Ping(ctx context.Context, req *connect.Request[ftlv1.PingRequest]) (*connect.Response[ftlv1.PingResponse], error)
 }
 
 // GetHTTPClient returns a HTTP client usable for the given URL.

diff --git a/backend/schema/encoding.go b/backend/schema/encoding.go
@@ -33,6 +33,12 @@ func (*String) schemaChildren() []Node { return nil }
 func (*String) schemaType()            {}
 func (*String) String() string         { return "String" }
 
+var _ Type = (*Bytes)(nil)
+
+func (*Bytes) schemaChildren() []Node { return nil }
+func (*Bytes) schemaType()            {}
+func (*Bytes) String() string         { return "Bytes" }
+
 var _ Type = (*Bool)(nil)
 
 func (*Bool) schemaChildren() []Node { return nil }

diff --git a/backend/schema/jsonschema.go b/backend/schema/jsonschema.go
@@ -70,6 +70,16 @@ func nodeToJSSchema(node Node, dataRefs map[DataRef]bool) *jsonschema.Schema {
 		st := jsonschema.String
 		return &jsonschema.Schema{Type: &jsonschema.Type{SimpleTypes: &st}}
 
+	case *Bytes:
+		st := jsonschema.String
+		encoding := "base64"
+		mediaType := "application/octet-stream"
+		return &jsonschema.Schema{
+			Type:             &jsonschema.Type{SimpleTypes: &st},
+			ContentEncoding:  &encoding,
+			ContentMediaType: &mediaType,
+		}
+
 	case *Bool:
 		st := jsonschema.Boolean
 		return &jsonschema.Schema{Type: &jsonschema.Type{SimpleTypes: &st}}

diff --git a/backend/schema/normalise.go b/backend/schema/normalise.go
@@ -54,6 +54,10 @@ func Normalise[T Node](n T) T {
 		c.Str = false
 		c.Pos = zero
 
+	case *Bytes:
+		c.Bytes = false
+		c.Pos = zero
+
 	case *Verb:
 		c.Pos = zero
 		c.Request = Normalise(c.Request)

diff --git a/backend/schema/protobuf_dec.go b/backend/schema/protobuf_dec.go
@@ -132,6 +132,8 @@ func typeToSchema(s *schemapb.Type) Type {
 		return &Float{Pos: PosFromProto(s.Float.Pos)}
 	case *schemapb.Type_String_:
 		return &String{Pos: PosFromProto(s.String_.Pos)}
+	case *schemapb.Type_Bytes:
+		return &Bytes{Pos: PosFromProto(s.Bytes.Pos)}
 	case *schemapb.Type_Time:
 		return &Time{Pos: PosFromProto(s.Time.Pos)}
 	case *schemapb.Type_Bool:

diff --git a/backend/schema/protobuf_enc.go b/backend/schema/protobuf_enc.go
@@ -167,6 +167,10 @@ func (s *String) ToProto() proto.Message {
 	return &schemapb.String{}
 }
 
+func (s *Bytes) ToProto() proto.Message {
+	return &schemapb.Bytes{}
+}
+
 func (b *Bool) ToProto() proto.Message {
 	return &schemapb.Bool{}
 }
@@ -211,6 +215,9 @@ func typeToProto(t Type) *schemapb.Type {
 	case *String:
 		return &schemapb.Type{Value: &schemapb.Type_String_{String_: t.ToProto().(*schemapb.String)}}
 
+	case *Bytes:
+		return &schemapb.Type{Value: &schemapb.Type_Bytes{Bytes: t.ToProto().(*schemapb.Bytes)}}
+
 	case *Time:
 		return &schemapb.Type{Value: &schemapb.Type_Time{Time: t.ToProto().(*schemapb.Time)}}
 

diff --git a/backend/schema/schema.go b/backend/schema/schema.go
@@ -15,7 +15,7 @@ import (
 
 var (
 	declUnion            = []Decl{&Data{}, &Verb{}}
-	nonOptionalTypeUnion = []Type{&Int{}, &Float{}, &String{}, &Bool{}, &Time{}, &Array{}, &Map{} /*&VerbRef{},*/, &DataRef{}}
+	nonOptionalTypeUnion = []Type{&Int{}, &Float{}, &String{}, &Bytes{}, &Bool{}, &Time{}, &Array{}, &Map{} /*&VerbRef{},*/, &DataRef{}}
 	typeUnion            = append(nonOptionalTypeUnion, &Optional{})
 	metadataUnion        = []Metadata{&MetadataCalls{}, &MetadataIngress{}}
 	ingressUnion         = []IngressPathComponent{&IngressPathLiteral{}, &IngressPathParameter{}}
@@ -87,6 +87,12 @@ type String struct {
 	Str bool `parser:"@'String'" protobuf:"-"`
 }
 
+type Bytes struct {
+	Pos Position `parser:"" protobuf:"1,optional"`
+
+	Bytes bool `parser:"@'Bytes'" protobuf:"-"`
+}
+
 type Bool struct {
 	Pos Position `parser:"" protobuf:"1,optional"`
 

diff --git a/backend/schema/validate.go b/backend/schema/validate.go
@@ -9,8 +9,12 @@ import (
 
 var (
 	// Identifiers that can't be used as data or verb names.
+	//
+	// We don't need Array/Map/VerbRef/DataRef here because there are no
+	// keywords associated with these types.
 	reservedIdentNames = map[string]bool{
 		"Int": true, "Float": true, "String": true, "Bool": true, "Time": true,
+		"Bytes": true,
 	}
 )
 
@@ -128,7 +132,7 @@ func ValidateModule(module *Module) error {
 			}
 
 		case *Array, *Bool, *DataRef, *Field, *Float, *Int,
-			*Time, *Map, *Module, *Schema, *String, *VerbRef,
+			*Time, *Map, *Module, *Schema, *String, *Bytes, *VerbRef,
 			*MetadataCalls, *MetadataIngress, IngressPathComponent,
 			*IngressPathLiteral, *IngressPathParameter, *Optional:
 		case Type, Metadata, Decl: // Union sql.

diff --git a/cmd/ftl/cmd_serve.go b/cmd/ftl/cmd_serve.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
@@ -102,7 +103,7 @@ func (s *serveCmd) setupDB(ctx context.Context) (string, error) {
 	}
 
 	recreate := s.Recreate
-	port := fmt.Sprintf("%d", s.DBPort)
+	port := strconv.Itoa(s.DBPort)
 
 	if len(output) == 0 {
 		logger.Infof("Creating docker container '%s' for postgres db", ftlContainerName)

diff --git a/frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts b/frontend/src/protos/xyz/block/ftl/v1/schema/schema_pb.ts
diff --git a/go-runtime/compile/generate/external_module.go b/go-runtime/compile/generate/external_module.go
@@ -91,6 +91,9 @@ func genType(module *schema.Module, t schema.Type) string {
 	case *schema.Time:
 		return "time.Time"
 
+	case *schema.Bytes:
+		return "[]byte"
+
 	case *schema.Int, *schema.Bool, *schema.String:
 		return strings.ToLower(t.String())
 

diff --git a/go-runtime/compile/schema.go b/go-runtime/compile/schema.go
@@ -412,7 +412,11 @@ func parseMap(pctx *parseContext, node ast.Node, tnode *types.Map) (*schema.Map,
 	}, nil
 }
 
-func parseSlice(pctx *parseContext, node ast.Node, tnode *types.Slice) (*schema.Array, error) {
+func parseSlice(pctx *parseContext, node ast.Node, tnode *types.Slice) (schema.Type, error) {
+	// If it's a []byte, treat it as a Bytes type.
+	if basic, ok := tnode.Elem().Underlying().(*types.Basic); ok && basic.Kind() == types.Byte {
+		return &schema.Bytes{Pos: goPosToSchemaPos(node.Pos())}, nil
+	}
 	value, err := parseType(pctx, node, tnode.Elem())
 	if err != nil {
 		return nil, err

diff --git a/go-runtime/compile/schema_test.go b/go-runtime/compile/schema_test.go
@@ -29,6 +29,7 @@ func TestExtractModuleSchema(t *testing.T) {
     optional Nested?
     time Time
     user two.User
+    bytes Bytes
   }
 
   data Resp {

diff --git a/go-runtime/compile/testdata/one/one.go b/go-runtime/compile/testdata/one/one.go
@@ -23,6 +23,7 @@ type Req struct {
 	Optional sdk.Option[Nested]
 	Time     time.Time
 	User     two.User
+	Bytes    []byte
 }
 type Resp struct{}
 

diff --git a/kotlin-runtime/ftl-generator/pom.xml b/kotlin-runtime/ftl-generator/pom.xml
@@ -99,6 +99,10 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>11</source>
+                    <target>11</target>
+                </configuration>
             </plugin>
             <!-- Download the Wire compiler. -->
             <plugin>

diff --git a/kotlin-runtime/ftl-generator/src/main/kotlin/xyz/block/ftl/generator/ModuleGenerator.kt b/kotlin-runtime/ftl-generator/src/main/kotlin/xyz/block/ftl/generator/ModuleGenerator.kt
@@ -143,6 +143,7 @@ class ModuleGenerator() {
       type.int != null -> ClassName("kotlin", "Long")
       type.float != null -> ClassName("kotlin", "Float")
       type.string != null -> ClassName("kotlin", "String")
+      type.bytes != null -> ClassName("kotlin", "ByteArray")
       type.bool != null -> ClassName("kotlin", "Boolean")
       type.time != null -> ClassName("java.time", "OffsetDateTime")
       type.array != null -> {

diff --git a/kotlin-runtime/ftl-generator/src/test/kotlin/xyz/block/ftl/generator/ModuleGeneratorTest.kt b/kotlin-runtime/ftl-generator/src/test/kotlin/xyz/block/ftl/generator/ModuleGeneratorTest.kt
@@ -43,6 +43,7 @@ public class TestModule()
             Field(name = "int", type = Type(int = Int())),
             Field(name = "float", type = Type(float = Float())),
             Field(name = "string", type = Type(string = String())),
+            Field(name = "bytes", type = Type(bytes = Bytes())),
             Field(name = "bool", type = Type(bool = Bool())),
             Field(name = "time", type = Type(time = Time())),
             Field(name = "optional", type = Type(optional = Optional(type = Type(string = String())))),
@@ -84,6 +85,7 @@ package ftl.test
 
 import java.time.OffsetDateTime
 import kotlin.Boolean
+import kotlin.ByteArray
 import kotlin.Float
 import kotlin.Long
 import kotlin.String
@@ -106,6 +108,7 @@ public data class TestResponse(
   public val int: Long,
   public val float: Float,
   public val string: String,
+  public val bytes: ByteArray,
   public val bool: Boolean,
   public val time: OffsetDateTime,
   public val optional: String? = null,
@@ -144,9 +147,11 @@ public class TestModule()
           request = DataRef(name = "TestRequest"),
           response = DataRef(name = "TestResponse"),
           metadata = listOf(
-            Metadata(ingress = MetadataIngress(
-              path = listOf(IngressPathComponent(ingressPathLiteral = IngressPathLiteral(text = "test"))),
-              method = "GET")
+            Metadata(
+              ingress = MetadataIngress(
+                path = listOf(IngressPathComponent(ingressPathLiteral = IngressPathLiteral(text = "test"))),
+                method = "GET"
+              )
             ),
           )
         )