feat: Initial Java runtime implementation (#2318)

This is still very much a work in progress, however it contains a lot of
basic functionality.

So far this includes support for:

- Verb invocations
- HTTP ingress
- Cron
- Topics and Subscriptions
- Basic testing of Verbs

The existing Kotlin example has been migrated over to the new approach.
At the moment the module is called Java even though it supports both, in
future this will provide a base layer of functionality with some small
language dependent features in separate Java/Kotlin modules.

Justfile

@@ -31,6 +31,7 @@ clean:
   rm -rf frontend/node_modules
   find . -name '*.zip' -exec rm {} \;
   mvn -f kotlin-runtime/ftl-runtime clean
+  mvn -f java-runtime/ftl-runtime clean
 
 # Live rebuild the ftl binary whenever source changes.
 live-rebuild:
@@ -41,7 +42,7 @@ dev *args:
   watchexec -r {{WATCHEXEC_ARGS}} -- "just build-sqlc && ftl dev {{args}}"
 
 # Build everything
-build-all: build-protos-unconditionally build-frontend build-generate build-sqlc build-zips lsp-generate
+build-all: build-protos-unconditionally build-frontend build-generate build-sqlc build-zips lsp-generate build-java
   @just build ftl ftl-controller ftl-runner ftl-initdb
 
 # Run "go generate" on all packages
@@ -64,6 +65,9 @@ build +tools: build-protos build-zips build-frontend
 build-backend:
   just build ftl ftl-controller ftl-runner
 
+build-java:
+  mvn -f java-runtime/ftl-runtime install
+
 export DATABASE_URL := "postgres://postgres:secret@localhost:15432/ftl?sslmode=disable"
 
 # Explicitly initialise the database

backend/controller/scaling/localscaling/local_scaling.go

@@ -92,7 +92,7 @@ func (l *LocalScaling) SetReplicas(ctx context.Context, replicas int, idleRunner
 		simpleName := fmt.Sprintf("runner%d", keySuffix)
 		if err := kong.ApplyDefaults(&config, kong.Vars{
 			"deploymentdir": filepath.Join(l.cacheDir, "ftl-runner", simpleName, "deployments"),
-			"language":      "go,kotlin,rust",
+			"language":      "go,kotlin,rust,java",
 		}); err != nil {
 			return err
 		}

backend/runner/runner.go

@@ -49,7 +49,7 @@ type Config struct {
 	TemplateDir           string          `help:"Template directory to copy into each deployment, if any." type:"existingdir"`
 	DeploymentDir         string          `help:"Directory to store deployments in." default:"${deploymentdir}"`
 	DeploymentKeepHistory int             `help:"Number of deployments to keep history for." default:"3"`
-	Language              []string        `short:"l" help:"Languages the runner supports." env:"FTL_LANGUAGE" default:"go,kotlin,rust"`
+	Language              []string        `short:"l" help:"Languages the runner supports." env:"FTL_LANGUAGE" default:"go,kotlin,rust,java"`
 	HeartbeatPeriod       time.Duration   `help:"Minimum period between heartbeats." default:"3s"`
 	HeartbeatJitter       time.Duration   `help:"Jitter to add to heartbeat period." default:"2s"`
 	RunnerStartDelay      time.Duration   `help:"Time in seconds for a runner to wait before contacting the controller. This can be needed in istio environments to work around initialization races." env:"FTL_RUNNER_START_DELAY" default:"0s"`

backend/schema/metadatatypemap.go

@@ -11,7 +11,7 @@ import (
 type MetadataTypeMap struct {
 	Pos Position `parser:"" protobuf:"1,optional"`
 
-	Runtime    string `parser:"'+' 'typemap' @('go' | 'kotlin')" protobuf:"2"`
+	Runtime    string `parser:"'+' 'typemap' @('go' | 'kotlin' | 'java')" protobuf:"2"`
 	NativeName string `parser:"@String" protobuf:"3"`
 }
 

buildengine/build.go

@@ -47,6 +47,8 @@ func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema,
 	switch module.Config.Language {
 	case "go":
 		err = buildGoModule(ctx, projectRootDir, sch, module, filesTransaction)
+	case "java":
+		err = buildJavaModule(ctx, module)
 	case "kotlin":
 		err = buildKotlinModule(ctx, sch, module)
 	case "rust":

buildengine/build_java.go

@@ -0,0 +1,23 @@
+package buildengine
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/TBD54566975/ftl/internal/exec"
+	"github.com/TBD54566975/ftl/internal/log"
+)
+
+func buildJavaModule(ctx context.Context, module Module) error {
+	logger := log.FromContext(ctx)
+	if err := SetPOMProperties(ctx, module.Config.Dir); err != nil {
+		return fmt.Errorf("unable to update ftl.version in %s: %w", module.Config.Dir, err)
+	}
+	logger.Infof("Using build command '%s'", module.Config.Build)
+	err := exec.Command(ctx, log.Debug, module.Config.Dir, "bash", "-c", module.Config.Build).RunBuffered(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to build module %q: %w", module.Config.Module, err)
+	}
+
+	return nil
+}

buildengine/build_kotlin.go

@@ -43,16 +43,14 @@ func buildKotlinModule(ctx context.Context, sch *schema.Schema, module Module) e
 	if err := SetPOMProperties(ctx, module.Config.Dir); err != nil {
 		return fmt.Errorf("unable to update ftl.version in %s: %w", module.Config.Dir, err)
 	}
-
 	if err := generateExternalModules(ctx, module, sch); err != nil {
 		return fmt.Errorf("unable to generate external modules for %s: %w", module.Config.Module, err)
 	}
-
 	if err := prepareFTLRoot(module); err != nil {
 		return fmt.Errorf("unable to prepare FTL root for %s: %w", module.Config.Module, err)
 	}
 
-	logger.Debugf("Using build command '%s'", module.Config.Build)
+	logger.Infof("Using build command '%s'", module.Config.Build)
 	err := exec.Command(ctx, log.Debug, module.Config.Dir, "bash", "-c", module.Config.Build).RunBuffered(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to build module %q: %w", module.Config.Module, err)

buildengine/build_test.go

@@ -93,7 +93,7 @@ func testBuildClearsBuildDir(t *testing.T, bctx buildContext) {
 	projectRoot := t.TempDir()
 
 	// generate stubs to create the shared modules directory
-	err = GenerateStubs(ctx, projectRoot, bctx.sch.Modules, []moduleconfig.ModuleConfig{{Dir: bctx.moduleDir}})
+	err = GenerateStubs(ctx, projectRoot, bctx.sch.Modules, []moduleconfig.ModuleConfig{{Dir: bctx.moduleDir, Language: "go"}})
 	assert.NoError(t, err)
 
 	// build to generate the build directory

buildengine/deps.go

@@ -51,6 +51,9 @@ func extractDependencies(module Module) ([]string, error) {
 	case "kotlin":
 		return extractKotlinFTLImports(module.Config.Module, module.Config.Dir)
 
+	case "java":
+		return extractJavaFTLImports(module.Config.Module, module.Config.Dir)
+
 	case "rust":
 		return extractRustFTLImports(module.Config.Module, module.Config.Dir)
 
@@ -140,6 +143,55 @@ func extractKotlinFTLImports(self, dir string) ([]string, error) {
 	return modules, nil
 }
 
+func extractJavaFTLImports(self, dir string) ([]string, error) {
+	dependencies := map[string]bool{}
+	// We also attempt to look at kotlin files
+	// As the Java module supports both
+	kotin, kotlinErr := extractKotlinFTLImports(self, dir)
+	if kotlinErr == nil {
+		// We don't really care about the error case, its probably a Java project
+		for _, imp := range kotin {
+			dependencies[imp] = true
+		}
+	}
+	javaImportRegex := regexp.MustCompile(`^import ftl\.([A-Za-z0-9_.]+)`)
+
+	err := filepath.WalkDir(filepath.Join(dir, "src/main/java"), func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return fmt.Errorf("failed to walk directory: %w", err)
+		}
+		if d.IsDir() || !(strings.HasSuffix(path, ".java")) {
+			return nil
+		}
+		file, err := os.Open(path)
+		if err != nil {
+			return fmt.Errorf("failed to open file: %w", err)
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+		for scanner.Scan() {
+			matches := javaImportRegex.FindStringSubmatch(scanner.Text())
+			if len(matches) > 1 {
+				module := strings.Split(matches[1], ".")[0]
+				if module == self {
+					continue
+				}
+				dependencies[module] = true
+			}
+		}
+		return scanner.Err()
+	})
+
+	// We only error out if they both failed
+	if err != nil && kotlinErr != nil {
+		return nil, fmt.Errorf("%s: failed to extract dependencies from Java module: %w", self, err)
+	}
+	modules := maps.Keys(dependencies)
+	sort.Strings(modules)
+	return modules, nil
+}
+
 func extractRustFTLImports(self, dir string) ([]string, error) {
 	fmt.Fprintf(os.Stderr, "RUST TODO extractRustFTLImports\n")
 

buildengine/discover_test.go

@@ -73,7 +73,7 @@ func TestDiscoverModules(t *testing.T) {
 				Language: "kotlin",
 				Realm:    "home",
 				Module:   "echo",
-				Build:    "mvn -B compile",
+				Build:    "mvn -B package",
 				Deploy: []string{
 					"main",
 					"classes",
@@ -116,7 +116,7 @@ func TestDiscoverModules(t *testing.T) {
 				Language: "kotlin",
 				Realm:    "home",
 				Module:   "externalkotlin",
-				Build:    "mvn -B compile",
+				Build:    "mvn -B package",
 				Deploy: []string{
 					"main",
 					"classes",

buildengine/stubs.go

@@ -3,6 +3,8 @@ package buildengine
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/common/moduleconfig"
@@ -13,7 +15,11 @@ import (
 //
 // Currently, only Go stubs are supported. Kotlin and other language stubs can be added in the future.
 func GenerateStubs(ctx context.Context, projectRoot string, modules []*schema.Module, moduleConfigs []moduleconfig.ModuleConfig) error {
-	return generateGoStubs(ctx, projectRoot, modules, moduleConfigs)
+	err := generateGoStubs(ctx, projectRoot, modules, moduleConfigs)
+	if err != nil {
+		return err
+	}
+	return writeGenericSchemaFiles(ctx, projectRoot, modules, moduleConfigs)
 }
 
 // CleanStubs removes all generated stubs.
@@ -37,6 +43,39 @@ func generateGoStubs(ctx context.Context, projectRoot string, modules []*schema.
 	return nil
 }
 
+func writeGenericSchemaFiles(ctx context.Context, projectRoot string, modules []*schema.Module, moduleConfigs []moduleconfig.ModuleConfig) error {
+	sch := &schema.Schema{Modules: modules}
+	for _, module := range moduleConfigs {
+		if module.GeneratedSchemaDir == "" {
+			continue
+		}
+
+		modPath := module.Abs().GeneratedSchemaDir
+		err := os.MkdirAll(modPath, 0750)
+		if err != nil {
+			return fmt.Errorf("failed to create directory %s: %w", modPath, err)
+		}
+
+		for _, mod := range sch.Modules {
+			if mod.Name == module.Module {
+				continue
+			}
+			data, err := schema.ModuleToBytes(mod)
+			if err != nil {
+				return fmt.Errorf("failed to export module schema for module %s %w", mod.Name, err)
+			}
+			err = os.WriteFile(filepath.Join(modPath, mod.Name+".pb"), data, 0600)
+			if err != nil {
+				return fmt.Errorf("failed to write schema file for module %s %w", mod.Name, err)
+			}
+		}
+	}
+	err := compile.GenerateStubsForModules(ctx, projectRoot, moduleConfigs, sch)
+	if err != nil {
+		return fmt.Errorf("failed to generate go stubs: %w", err)
+	}
+	return nil
+}
 func cleanGoStubs(ctx context.Context, projectRoot string) error {
 	err := compile.CleanStubs(ctx, projectRoot)
 	if err != nil {

buildengine/stubs_test.go

@@ -6,10 +6,11 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/alecthomas/assert/v2"
+
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/common/moduleconfig"
 	"github.com/TBD54566975/ftl/internal/log"
-	"github.com/alecthomas/assert/v2"
 )
 
 func TestGenerateGoStubs(t *testing.T) {
@@ -180,7 +181,7 @@ func init() {
 
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 	projectRoot := t.TempDir()
-	err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{})
+	err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{{Language: "go"}})
 	assert.NoError(t, err)
 
 	generatedPath := filepath.Join(projectRoot, ".ftl/go/modules/other/external_module.go")
@@ -240,7 +241,7 @@ func Call(context.Context, Req) (Resp, error) {
 `
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 	projectRoot := t.TempDir()
-	err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{})
+	err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{{Language: "go"}})
 	assert.NoError(t, err)
 
 	generatedPath := filepath.Join(projectRoot, ".ftl/go/modules/test/external_module.go")

common/moduleconfig/moduleconfig.go

@@ -21,6 +21,9 @@ type ModuleGoConfig struct{}
 // ModuleKotlinConfig is language-specific configuration for Kotlin modules.
 type ModuleKotlinConfig struct{}
 
+// ModuleJavaConfig is language-specific configuration for Java modules.
+type ModuleJavaConfig struct{}
+
 // ModuleConfig is the configuration for an FTL module.
 //
 // Module config files are currently TOML.
@@ -37,6 +40,8 @@ type ModuleConfig struct {
 	Deploy []string `toml:"deploy"`
 	// DeployDir is the directory to deploy from, relative to the module directory.
 	DeployDir string `toml:"deploy-dir"`
+	// GeneratedSchemaDir is the directory to generate protobuf schema files into. These can be picked up by language specific build tools
+	GeneratedSchemaDir string `toml:"generated-schema-dir"`
 	// Schema is the name of the schema file relative to the DeployDir.
 	Schema string `toml:"schema"`
 	// Errors is the name of the error file relative to the DeployDir.
@@ -46,6 +51,7 @@ type ModuleConfig struct {
 
 	Go     ModuleGoConfig     `toml:"go,optional"`
 	Kotlin ModuleKotlinConfig `toml:"kotlin,optional"`
+	Java   ModuleJavaConfig   `toml:"java,optional"`
 }
 
 // AbsModuleConfig is a ModuleConfig with all paths made absolute.
@@ -84,6 +90,12 @@ func (c ModuleConfig) Abs() AbsModuleConfig {
 	if !strings.HasPrefix(clone.DeployDir, clone.Dir) {
 		panic(fmt.Sprintf("deploy-dir %q is not beneath module directory %q", clone.DeployDir, clone.Dir))
 	}
+	if clone.GeneratedSchemaDir != "" {
+		clone.GeneratedSchemaDir = filepath.Clean(filepath.Join(clone.Dir, clone.GeneratedSchemaDir))
+		if !strings.HasPrefix(clone.GeneratedSchemaDir, clone.Dir) {
+			panic(fmt.Sprintf("generated-schema-dir %q is not beneath module directory %q", clone.GeneratedSchemaDir, clone.Dir))
+		}
+	}
 	clone.Schema = filepath.Clean(filepath.Join(clone.DeployDir, clone.Schema))
 	if !strings.HasPrefix(clone.Schema, clone.DeployDir) {
 		panic(fmt.Sprintf("schema %q is not beneath deploy directory %q", clone.Schema, clone.DeployDir))
@@ -119,7 +131,7 @@ func setConfigDefaults(moduleDir string, config *ModuleConfig) error {
 	switch config.Language {
 	case "kotlin":
 		if config.Build == "" {
-			config.Build = "mvn -B compile"
+			config.Build = "mvn -B package"
 		}
 		if config.DeployDir == "" {
 			config.DeployDir = "target"
@@ -130,7 +142,22 @@ func setConfigDefaults(moduleDir string, config *ModuleConfig) error {
 		if len(config.Watch) == 0 {
 			config.Watch = []string{"pom.xml", "src/**", "target/generated-sources"}
 		}
-
+	case "java":
+		if config.Build == "" {
+			config.Build = "mvn -B package"
+		}
+		if config.DeployDir == "" {
+			config.DeployDir = "target"
+		}
+		if config.GeneratedSchemaDir == "" {
+			config.GeneratedSchemaDir = "src/main/ftl-module-schema"
+		}
+		if len(config.Deploy) == 0 {
+			config.Deploy = []string{"main", "quarkus-app"}
+		}
+		if len(config.Watch) == 0 {
+			config.Watch = []string{"pom.xml", "src/**", "target/generated-sources"}
+		}
 	case "go":
 		if config.DeployDir == "" {
 			config.DeployDir = ".ftl"

deployment/base/ftl-runner/ftl-runner.yml

@@ -31,7 +31,7 @@ spec:
             - name: FTL_RUNNER_ADVERTISE
               value: "http://$(MY_POD_IP):8893"
             - name: FTL_LANGUAGE
-              value: "go,kotlin"
+              value: "go,kotlin,java"
           ports:
             - containerPort: 8893
           readinessProbe:

examples/go/echo/echo.go

@@ -23,7 +23,7 @@ type EchoResponse struct {
 
 // Echo returns a greeting with the current time.
 //
-//ftl:verb
+//ftl:verb export
 func Echo(ctx context.Context, req EchoRequest) (EchoResponse, error) {
 	tresp, err := ftl.Call(ctx, time.Time, time.TimeRequest{})
 	if err != nil {

examples/kotlin/echo/ftl.toml

@@ -1,2 +1,2 @@
 module = "echo"
-language = "kotlin"
+language = "java"