diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c085e9bb51..15e1dcb4f7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ jobs:
         run: docker compose up -d --wait
       - name: Test
         run: |
-          go-test-annotate ${{ (github.event_name == 'pull_request' && github.event.action != 'enqueued') && '-short' || '' }}
+          go-test-annotate ${{ (github.event_name == 'pull_request' && github.event.action != 'enqueued' && !contains( github.event.pull_request.labels.*.name, 'run-integration')) && '-short' || '' }}
   test-readme:
     name: Test README
     runs-on: ubuntu-latest
diff --git a/internal/integration/harness.go b/internal/integration/harness.go
index 140e9dd7b9..0dd23b7793 100644
--- a/internal/integration/harness.go
+++ b/internal/integration/harness.go
@@ -138,38 +138,12 @@ func run(t *testing.T, actionsOrOptions ...ActionOrOption) {
 		t.Setenv(key, value)
 	}
 
-	tmpDir := t.TempDir()
-
 	cwd, err := os.Getwd()
 	assert.NoError(t, err)
 
 	rootDir, ok := internal.GitRoot("").Get()
 	assert.True(t, ok)
 
-	if opts.ftlConfigPath != "" {
-		// TODO: We shouldn't be copying the shared config from the "go" testdata...
-		opts.ftlConfigPath = filepath.Join(cwd, "testdata", "go", opts.ftlConfigPath)
-		projectPath := filepath.Join(tmpDir, "ftl-project.toml")
-
-		// Copy the specified FTL config to the temporary directory.
-		err = copy.Copy(opts.ftlConfigPath, projectPath)
-		if err == nil {
-			t.Setenv("FTL_CONFIG", projectPath)
-		} else {
-			// Use a path into the testdata directory instead of one relative to
-			// tmpDir. Otherwise we have a chicken and egg situation where the config
-			// can't be loaded until the module is copied over, and the config itself
-			// is used by FTL during startup.
-			// Some tests still rely on this behavior, so we can't remove it entirely.
-			t.Logf("Failed to copy %s to %s: %s", opts.ftlConfigPath, projectPath, err)
-			t.Setenv("FTL_CONFIG", opts.ftlConfigPath)
-		}
-
-	} else {
-		err = os.WriteFile(filepath.Join(tmpDir, "ftl-project.toml"), []byte(`name = "integration"`), 0644)
-		assert.NoError(t, err)
-	}
-
 	// Build FTL binary
 	logger := log.Configure(&logWriter{logger: t}, log.Config{Level: log.Debug})
 	ctx := log.ContextWithLogger(context.Background(), logger)
@@ -188,6 +162,8 @@ func run(t *testing.T, actionsOrOptions ...ActionOrOption) {
 	for _, language := range opts.languages {
 		ctx, done := context.WithCancel(ctx)
 		t.Run(language, func(t *testing.T) {
+			tmpDir := initWorkDir(t, cwd, opts)
+
 			verbs := rpc.Dial(ftlv1connect.NewVerbServiceClient, "http://localhost:8892", log.Debug)
 
 			var controller ftlv1connect.ControllerServiceClient
@@ -232,6 +208,35 @@ func run(t *testing.T, actionsOrOptions ...ActionOrOption) {
 	}
 }
 
+func initWorkDir(t testing.TB, cwd string, opts options) string {
+	tmpDir := t.TempDir()
+
+	if opts.ftlConfigPath != "" {
+		// TODO: We shouldn't be copying the shared config from the "go" testdata...
+		opts.ftlConfigPath = filepath.Join(cwd, "testdata", "go", opts.ftlConfigPath)
+		projectPath := filepath.Join(tmpDir, "ftl-project.toml")
+
+		// Copy the specified FTL config to the temporary directory.
+		err := copy.Copy(opts.ftlConfigPath, projectPath)
+		if err == nil {
+			t.Setenv("FTL_CONFIG", projectPath)
+		} else {
+			// Use a path into the testdata directory instead of one relative to
+			// tmpDir. Otherwise we have a chicken and egg situation where the config
+			// can't be loaded until the module is copied over, and the config itself
+			// is used by FTL during startup.
+			// Some tests still rely on this behavior, so we can't remove it entirely.
+			t.Logf("Failed to copy %s to %s: %s", opts.ftlConfigPath, projectPath, err)
+			t.Setenv("FTL_CONFIG", opts.ftlConfigPath)
+		}
+
+	} else {
+		err := os.WriteFile(filepath.Join(tmpDir, "ftl-project.toml"), []byte(`name = "integration"`), 0644)
+		assert.NoError(t, err)
+	}
+	return tmpDir
+}
+
 type TestContext struct {
 	context.Context
 	// Temporary directory the test is executing in.
diff --git a/jvm-runtime/jvm_integration_test.go b/jvm-runtime/jvm_integration_test.go
index 31f2e48e43..4038b8b24e 100644
--- a/jvm-runtime/jvm_integration_test.go
+++ b/jvm-runtime/jvm_integration_test.go
@@ -19,7 +19,6 @@ func TestLifecycle(t *testing.T) {
 		in.WithLanguages("java", "kotlin"),
 		in.GitInit(),
 		in.Exec("rm", "ftl-project.toml"),
-		in.IfLanguage("kotlin", in.Exec("rm", "-r", "echo")), //horrible, but we need to do cleanup, I wonder if we should be running each test in a separate directory
 		in.Exec("ftl", "init", "test", "."),
 		in.IfLanguage("java", in.Exec("ftl", "new", "java", ".", "echo")),
 		in.IfLanguage("kotlin", in.Exec("ftl", "new", "kotlin", ".", "echo")),