Add ability to execute scripts via Kubernetes Jobs

source/Octopus.Manager.Tentacle/Octopus.Manager.Tentacle.csproj

@@ -130,7 +130,7 @@
     <PackageReference Include="NuGet.Packaging.Core" Version="3.6.0-octopus-58692" />
     <PackageReference Include="NuGet.Packaging.Core.Types" Version="3.6.0-octopus-58692" />
     <PackageReference Include="NuGet.Versioning" Version="3.6.0-octopus-58692" />
-    <PackageReference Include="Portable.BouncyCastle" Version="1.8.10" />
+    <PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
     <PackageReference Include="System.Collections" Version="4.3.0" />
     <PackageReference Include="System.Collections.Concurrent" Version="4.3.0" />
     <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />

source/Octopus.Tentacle.Contracts/ScriptServiceV3Alpha/KubernetesJobScriptExecutionContext.cs

@@ -0,0 +1,6 @@
+
+namespace Octopus.Tentacle.Contracts.ScriptServiceV3Alpha
+{
+    public class KubernetesJobScriptExecutionContext : IScriptExecutionContext
+    { }
+}

source/Octopus.Tentacle.Tests.Integration/Util/TestScriptLog.cs

@@ -23,8 +23,11 @@ public void Dispose()
         }
 
         public void WriteOutput(ProcessOutputSource source, string message)
+            => WriteOutput(source, message, DateTimeOffset.UtcNow);
+
+        public void WriteOutput(ProcessOutputSource source, string message, DateTimeOffset occurred)
         {
-            Console.WriteLine($"{DateTime.UtcNow} {source} {message}");
+            Console.WriteLine($"{occurred} {source} {message}");
             switch (source)
             {
                 case ProcessOutputSource.Debug:

source/Octopus.Tentacle.Tests/Integration/ScriptServiceV3AlphaFixture.cs

@@ -11,10 +11,13 @@
 using Octopus.Diagnostics;
 using Octopus.Tentacle.CommonTestUtils.Builders;
 using Octopus.Tentacle.Configuration;
+using Octopus.Tentacle.Configuration.Instances;
 using Octopus.Tentacle.Contracts;
 using Octopus.Tentacle.Contracts.ScriptServiceV3Alpha;
 using Octopus.Tentacle.Diagnostics;
+using Octopus.Tentacle.Kubernetes;
 using Octopus.Tentacle.Scripts;
+using Octopus.Tentacle.Scripts.Kubernetes;
 using Octopus.Tentacle.Services.Scripts;
 using Octopus.Tentacle.Util;
 
@@ -41,6 +44,12 @@ public void SetUp()
                 new Lazy<LocalShellScriptExecutor>(() =>
                     new LocalShellScriptExecutor(
                         PlatformDetection.IsRunningOnWindows ? new PowerShell() : new Bash(),
+                        Substitute.For<ISystemLog>())),
+                new Lazy<KubernetesJobScriptExecutor>(() =>
+                    new KubernetesJobScriptExecutor(
+                        Substitute.For<IKubernetesJobService>(),
+                        Substitute.For<IKubernetesClusterService>(),
+                        Substitute.For<IApplicationInstanceSelector>(),
                         Substitute.For<ISystemLog>())));
 
             service = new ScriptServiceV3Alpha(

source/Octopus.Tentacle/ExternalInit.cs

@@ -0,0 +1,4 @@
+namespace System.Runtime.CompilerServices
+{
+    internal static class IsExternalInit {}
+}

source/Octopus.Tentacle/Kubernetes/IKubernetesClientConfigProvider.cs

@@ -0,0 +1,9 @@
+using k8s;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    public interface IKubernetesClientConfigProvider
+    {
+        KubernetesClientConfiguration Get();
+    }
+}

source/Octopus.Tentacle/Kubernetes/InClusterKubernetesClientConfigProvider.cs

@@ -0,0 +1,13 @@
+using System;
+using k8s;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    class InClusterKubernetesClientConfigProvider : IKubernetesClientConfigProvider
+    {
+        public KubernetesClientConfiguration Get()
+        {
+            return KubernetesClientConfiguration.InClusterConfig();
+        }
+    }
+}

source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Threading.Tasks;
+using k8s;
+using Nito.AsyncEx;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    public interface IKubernetesClusterService
+    {
+        Task<ClusterVersion> GetClusterVersion();
+    }
+
+    public class KubernetesClusterService : KubernetesService, IKubernetesClusterService
+    {
+        readonly AsyncLazy<ClusterVersion> lazyVersion;
+        public KubernetesClusterService(IKubernetesClientConfigProvider configProvider)
+            : base(configProvider)
+        {
+            //As the cluster version isn't going to change without restarting, we just cache the version in an AsyncLazy
+            lazyVersion = new AsyncLazy<ClusterVersion>(async () =>
+            {
+                var versionInfo = await Client.Version.GetCodeAsync();
+
+                return new ClusterVersion(int.Parse(versionInfo.Major), int.Parse(versionInfo.Minor));
+            });
+        }
+
+        public async Task<ClusterVersion> GetClusterVersion()
+            => await lazyVersion;
+    }
+
+    public record ClusterVersion(int Major, int Minor);
+}

source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs

@@ -0,0 +1,18 @@
+using System;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    public static class KubernetesConfig
+    {
+        public static string Namespace => Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__K8SNAMESPACE")
+            ?? throw new InvalidOperationException("Unable to determine Kubernetes namespace. An environment variable 'OCTOPUS__TENTACLE__K8SNAMESPACE' must be defined.");
+
+        public static bool UseJobs => bool.TryParse(Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__K8SUSEJOBS"), out var useJobs) && useJobs;
+
+        public static string ServiceAccountName => Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__K8SSERVICEACCOUNTNAME")
+            ?? throw new InvalidOperationException("Unable to determine Kubernetes Job service account name. An environment variable 'OCTOPUS__TENTACLE__K8SSERVICEACCOUNTNAME' must be defined.");
+
+        public static string JobVolumeYaml => Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__K8SJOBVOLUMEYAML")
+            ?? throw new InvalidOperationException("Unable to determine Kubernetes Job volume yaml. An environment variable 'OCTOPUS__TENTACLE__K8SJOBVOLUMEYAML' must be defined.");
+    }
+}

source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs

@@ -0,0 +1,89 @@
+using System;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using k8s;
+using k8s.Autorest;
+using k8s.Models;
+using Octopus.Tentacle.Contracts;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    public interface IKubernetesJobService
+    {
+        Task<V1Job?> TryGet(ScriptTicket scriptTicket, CancellationToken cancellationToken);
+        string BuildJobName(ScriptTicket scriptTicket);
+        Task CreateJob(V1Job job, CancellationToken cancellationToken);
+        void Delete(ScriptTicket scriptTicket);
+        Task Watch(ScriptTicket scriptTicket, Func<V1Job, bool> onChange, Action<Exception> onError, CancellationToken cancellationToken);
+    }
+
+    public class KubernetesJobService : KubernetesService, IKubernetesJobService
+    {
+        public KubernetesJobService(IKubernetesClientConfigProvider configProvider)
+            : base(configProvider)
+        {
+        }
+
+        public async Task<V1Job?> TryGet(ScriptTicket scriptTicket, CancellationToken cancellationToken)
+        {
+            var jobName = BuildJobName(scriptTicket);
+
+            try
+            {
+                return await Client.ReadNamespacedJobStatusAsync(jobName, KubernetesConfig.Namespace, cancellationToken: cancellationToken);
+            }
+            catch (HttpOperationException opException)
+            {
+                if (opException.Response.StatusCode == HttpStatusCode.NotFound)
+                    return null;
+
+                //if there is some other error, just throw the exception
+                throw;
+            }
+        }
+
+        public async Task Watch(ScriptTicket scriptTicket, Func<V1Job, bool> onChange, Action<Exception> onError, CancellationToken cancellationToken)
+        {
+            var jobName = BuildJobName(scriptTicket);
+
+            using var response = Client.BatchV1.ListNamespacedJobWithHttpMessagesAsync(
+                KubernetesConfig.Namespace,
+                //only list this job
+                fieldSelector: $"metadata.name=={jobName}",
+                watch: true,
+                timeoutSeconds: 1800, //same as the TTL of the job itself
+                cancellationToken: cancellationToken);
+
+            await foreach (var (type, job) in response.WatchAsync<V1Job, V1JobList>(onError, cancellationToken: cancellationToken))
+            {
+                //we are only watching for modifications
+                if (type != WatchEventType.Modified)
+                    continue;
+
+                var stopWatching = onChange(job);
+                if (stopWatching)
+                    break;
+            }
+        }
+
+        public string BuildJobName(ScriptTicket scriptTicket) => $"octopus-{scriptTicket.TaskId}".ToLowerInvariant();
+
+        public async Task CreateJob(V1Job job, CancellationToken cancellationToken)
+        {
+            await Client.CreateNamespacedJobAsync(job, KubernetesConfig.Namespace, cancellationToken: cancellationToken);
+        }
+
+        public void Delete(ScriptTicket scriptTicket)
+        {
+            try
+            {
+                Client.DeleteNamespacedJob(BuildJobName(scriptTicket), KubernetesConfig.Namespace);
+            }
+            catch
+            {
+                //we are comfortable silently consuming this as the jobs have a TTL that will clean it up anyway
+            }
+        }
+    }
+}

source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs

@@ -0,0 +1,19 @@
+using Autofac;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    public class KubernetesModule : Module
+    {
+        protected override void Load(ContainerBuilder builder)
+        {
+            builder.RegisterType<KubernetesJobService>().As<IKubernetesJobService>().SingleInstance();
+            builder.RegisterType<KubernetesClusterService>().As<IKubernetesClusterService>().SingleInstance();
+
+#if DEBUG
+            builder.RegisterType<LocalMachineKubernetesClientConfigProvider>().As<IKubernetesClientConfigProvider>().SingleInstance();
+#else
+            builder.RegisterType<InClusterKubernetesClientConfigProvider>().As<IKubernetesClientConfigProvider>().SingleInstance();
+#endif
+        }
+    }
+}

source/Octopus.Tentacle/Kubernetes/KubernetesService.cs

@@ -0,0 +1,14 @@
+using k8sClient = k8s.Kubernetes;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    public abstract class KubernetesService
+    {
+        protected k8sClient Client { get; }
+
+        protected KubernetesService(IKubernetesClientConfigProvider configProvider)
+        {
+            Client = new k8sClient(configProvider.Get());
+        }
+    }
+}

source/Octopus.Tentacle/Kubernetes/LocalMachineKubernetesClientConfigProvider.cs

@@ -0,0 +1,18 @@
+using System;
+using k8s;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    class LocalMachineKubernetesClientConfigProvider : IKubernetesClientConfigProvider
+    {
+        public KubernetesClientConfiguration Get()
+        {
+#if DEBUG
+            var kubeConfigEnvVar = Environment.GetEnvironmentVariable("KUBECONFIG");
+            return KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeConfigEnvVar);
+#else
+            throw new NotSupportedException("Local machine configuration is only supported when debugging.");
+#endif
+        }
+    }
+}

source/Octopus.Tentacle/Kubernetes/bootstrapRunner.sh

@@ -0,0 +1,46 @@
+#! /usr/bin/bash
+
+WORK_DIR=$1
+STDOUT_LOG="$WORK_DIR/stdout.log"
+STDERR_LOG="$WORK_DIR/stderr.log"
+
+format() {
+	now=$(date -u +"%Y-%m-%dT%H:%M:%S.%N%z")
+	echo "$now|$2" | tee -a "$1"
+}
+
+logStdOut() {
+  while read -r IN
+  do
+    format "$STDOUT_LOG" "$IN"
+  done
+}
+
+logStdErr() {
+  while read -r IN
+  do
+	format "$STDERR_LOG" "$IN"
+  done
+}
+
+#ensure these files exist
+rm -f "$STDOUT_LOG";
+rm -f "$STDERR_LOG";
+touch "$STDOUT_LOG"
+touch "$STDERR_LOG"
+
+#pass the remaining args (skipping the first which is the working directory)
+shift
+
+BOOTSTRAP_SCRIPT=$1
+
+#This is the args for the Bootstrap script
+shift
+
+exec > >(logStdOut)
+exec 2> >(logStdErr >&2)
+
+/bin/bash $BOOTSTRAP_SCRIPT "$@"
+
+# This ungodly hack is to stop the pod from being killed before the last log has been flushed
+sleep 0.250 #250ms

source/Octopus.Tentacle/Octopus.Tentacle.csproj

@@ -40,6 +40,12 @@
 	<PropertyGroup Condition=" '$(TargetFramework)' == 'net6.0' ">
 		<DefineConstants>$(DefineConstants);HTTP_CLIENT_SUPPORTS_SSL_OPTIONS;REQUIRES_EXPLICIT_LOG_CONFIG;REQUIRES_CODE_PAGE_PROVIDER;USER_INTERACTIVE_DOES_NOT_WORK;DEFAULT_PROXY_IS_NOT_AVAILABLE;HAS_NULLABLE_REF_TYPES</DefineConstants>
 	</PropertyGroup>
+	<ItemGroup Condition=" '$(TargetFramework)' == 'net48' ">
+		<PackageReference Include="KubernetesClient.Classic" Version="10.1.19" />
+	</ItemGroup>
+	<ItemGroup Condition="'$(TargetFramework)' == 'net6.0'">
+		<PackageReference Include="KubernetesClient" Version="10.1.19" />
+	</ItemGroup>
 	<ItemGroup>
 		<PackageReference Include="Octopus.Client" Version="14.3.980" />
 	</ItemGroup>
@@ -54,14 +60,14 @@
 		<PackageReference Include="Octopus.Configuration" Version="4.1.0" />
 		<PackageReference Include="Octopus.Diagnostics" Version="2.1.0" />
 		<PackageReference Include="Octopus.Time" Version="1.1.339" />
-		<PackageReference Include="Portable.BouncyCastle" Version="1.8.4" />
+		<PackageReference Include="Polly" Version="7.2.2" />
+		<PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
+		<PackageReference Include="System.Management" Version="4.7.0" />
 		<PackageReference Include="System.ValueTuple" Version="4.5.0" />
 		<PackageReference Include="System.Net.Primitives" Version="4.3.1" />
 		<PackageReference Include="TaskScheduler" Version="2.7.2" />
 		<PackageReference Include="Nito.AsyncEx" Version="5.0.0" />
 		<PackageReference Include="NLog" Version="5.0.4" />
-		<PackageReference Include="Polly" Version="7.2.2" />
-		<PackageReference Include="System.Management" Version="4.7.0" />
 	</ItemGroup>
 	<ItemGroup>
 		<Compile Include="..\Solution Items\SolutionInfo.cs">
@@ -112,5 +118,9 @@
 	  <EmbeddedResource Include="Internals\Options\_Readme.txt" />
 	  <EmbeddedResource Include="Startup\PathsToDeleteOnStartup.core.txt" />
 	  <EmbeddedResource Include="Startup\PathsToDeleteOnStartup.netfx.txt" />
+	  <None Remove="Kubernetes\bootstrapRunner.sh" />
+	  <EmbeddedResource Include="Kubernetes\bootstrapRunner.sh">
+	    <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+	  </EmbeddedResource>
 	</ItemGroup>
 </Project>

source/Octopus.Tentacle/Program.cs

@@ -7,6 +7,7 @@
 using Octopus.Tentacle.Communications;
 using Octopus.Tentacle.Configuration;
 using Octopus.Tentacle.Diagnostics;
+using Octopus.Tentacle.Kubernetes;
 using Octopus.Tentacle.Maintenance;
 using Octopus.Tentacle.Properties;
 using Octopus.Tentacle.Services;
@@ -56,6 +57,7 @@ public override IContainer BuildContainer(StartUpInstanceRequest startUpInstance
             builder.RegisterModule(new ServicesModule());
             builder.RegisterModule(new VersioningModule(GetType().Assembly));
             builder.RegisterModule(new MaintenanceModule());
+            builder.RegisterModule<KubernetesModule>();
 
             builder.RegisterCommand<CreateInstanceCommand>("create-instance", "Registers a new instance of the Tentacle service");
             builder.RegisterCommand<DeleteInstanceCommand>("delete-instance", "Deletes an instance of the Tentacle service");