Merge pull request #1787 from tgstation/v620 [TGSDeploy]

v6.2.0 Release

README.md

@@ -539,7 +539,11 @@ Manual operations on the repository while an instance is running may lead to git
 
 #### Byond
 
-The `Byond` folder contains installations of [BYOND](https://www.byond.com/) versions. The version which is used by your game code can be changed on a whim (Note that only versions >= 511.1385 have been thouroughly tested. Lower versions should work but if one doesn't function, please open an issue report) and the server will take care of installing it.
+The `Byond` folder contains installations of [BYOND](https://www.byond.com/) or [OpenDream](https://github.com/OpenDreamProject/OpenDream) versions. The version which is used by your game code can be changed on a whim (Note that only versions >= 511.1385 have been thouroughly tested. Lower versions should work but if one doesn't function, please open an issue report) and the server will take care of installing it.
+
+##### Environment Variables
+
+You can specify additional environment variables to launch your server/compiler with by adding `server.env`/`compiler.env` to your engine installation directory (i.e. `<instance>/Byond/515.1530/server.env`). These are [.env](https://hexdocs.pm/dotenvy/dotenv-file-format.html) files.
 
 #### Compiler
 

build/SrcCommon.props

@@ -17,7 +17,7 @@
 
   <ItemGroup>
     <!-- Usage: Linting -->
-    <PackageReference Include="StyleCop.Analyzers" Version="1.2.0-beta.507">
+    <PackageReference Include="StyleCop.Analyzers" Version="1.2.0-beta.556">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

build/TestCommon.props

@@ -18,9 +18,9 @@
     <!-- Pinned: Be VERY careful about updating https://github.com/moq/moq/issues/1372 -->
     <PackageReference Include="Moq" Version="4.20.70" />
     <!-- Usage: MSTest execution -->
-    <PackageReference Include="MSTest.TestAdapter" Version="3.1.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.2.0" />
     <!-- Usage: MSTest asserts etc... -->
-    <PackageReference Include="MSTest.TestFramework" Version="3.1.1" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.2.0" />
   </ItemGroup>
 
 </Project>

build/Version.props

@@ -3,13 +3,13 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="WebpanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>6.1.5</TgsCoreVersion>
-    <TgsConfigVersion>5.0.0</TgsConfigVersion>
-    <TgsApiVersion>10.0.0</TgsApiVersion>
+    <TgsCoreVersion>6.2.0</TgsCoreVersion>
+    <TgsConfigVersion>5.1.0</TgsConfigVersion>
+    <TgsApiVersion>10.1.0</TgsApiVersion>
     <TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>
-    <TgsApiLibraryVersion>13.0.1</TgsApiLibraryVersion>
-    <TgsClientVersion>15.0.1</TgsClientVersion>
-    <TgsDmapiVersion>7.0.1</TgsDmapiVersion>
+    <TgsApiLibraryVersion>13.1.0</TgsApiLibraryVersion>
+    <TgsClientVersion>15.1.0</TgsClientVersion>
+    <TgsDmapiVersion>7.0.2</TgsDmapiVersion>
     <TgsInteropVersion>5.8.0</TgsInteropVersion>
     <TgsHostWatchdogVersion>1.4.1</TgsHostWatchdogVersion>
     <TgsContainerScriptVersion>1.2.1</TgsContainerScriptVersion>

build/WebpanelVersion.props

@@ -1,6 +1,6 @@
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <!-- This is in it's own file to help incremental building, changing it causes a complete rebuild of the web panel -->
-    <TgsWebpanelVersion>5.4.2</TgsWebpanelVersion>
+    <TgsWebpanelVersion>5.5.0</TgsWebpanelVersion>
   </PropertyGroup>
 </Project>

src/DMAPI/tgs.dm

@@ -1,6 +1,6 @@
 // tgstation-server DMAPI
 
-#define TGS_DMAPI_VERSION "7.0.1"
+#define TGS_DMAPI_VERSION "7.0.2"
 
 // All functions and datums outside this document are subject to change with any version and should not be relied on.
 
@@ -426,6 +426,7 @@
 
 /**
  * Send a message to connected chats. This function may sleep!
+ * If TGS is offline when called, the message may be placed in a queue to be sent and this function will return immediately. Your message will be sent when TGS reconnects to the game.
  *
  * message - The [/datum/tgs_message_content] to send.
  * admin_only: If [TRUE], message will be sent to admin connected chats. Vice-versa applies.
@@ -435,6 +436,7 @@
 
 /**
  * Send a private message to a specific user. This function may sleep!
+ * If TGS is offline when called, the message may be placed in a queue to be sent and this function will return immediately. Your message will be sent when TGS reconnects to the game.
  *
  * message - The [/datum/tgs_message_content] to send.
  * user: The [/datum/tgs_chat_user] to PM.
@@ -444,6 +446,7 @@
 
 /**
  * Send a message to connected chats that are flagged as game-related in TGS. This function may sleep!
+ * If TGS is offline when called, the message may be placed in a queue to be sent and this function will return immediately. Your message will be sent when TGS reconnects to the game.
  *
  * message - The [/datum/tgs_message_content] to send.
  * channels - Optional list of [/datum/tgs_chat_channel]s to restrict the message to.

src/DMAPI/tgs/v5/api.dm

@@ -8,8 +8,12 @@
 
 	var/reboot_mode = TGS_REBOOT_MODE_NORMAL
 
+	/// List of chat messages list()s that attempted to be sent during a topic call. To be bundled in the result of the call
 	var/list/intercepted_message_queue
 
+	/// List of chat messages list()s that attempted to be sent during a topic call. To be bundled in the result of the call
+	var/list/offline_message_queue
+
 	var/list/custom_commands
 
 	var/list/test_merges
@@ -194,17 +198,7 @@
 		var/datum/tgs_chat_channel/channel = I
 		ids += channel.id
 
-	message2 = UpgradeDeprecatedChatMessage(message2)
-
-	if (!length(channels))
-		return
-
-	var/list/data = message2._interop_serialize()
-	data[DMAPI5_CHAT_MESSAGE_CHANNEL_IDS] = ids
-	if(intercepted_message_queue)
-		intercepted_message_queue += list(data)
-	else
-		Bridge(DMAPI5_BRIDGE_COMMAND_CHAT_SEND, list(DMAPI5_BRIDGE_PARAMETER_CHAT_MESSAGE = data))
+	SendChatMessageRaw(message2, ids)
 
 /datum/tgs_api/v5/ChatTargetedBroadcast(datum/tgs_message_content/message2, admin_only)
 	var/list/channels = list()
@@ -213,26 +207,42 @@
 		if (!channel.is_private_channel && ((channel.is_admin_channel && admin_only) || (!channel.is_admin_channel && !admin_only)))
 			channels += channel.id
 
+	SendChatMessageRaw(message2, channels)
+
+/datum/tgs_api/v5/ChatPrivateMessage(datum/tgs_message_content/message2, datum/tgs_chat_user/user)
+	SendChatMessageRaw(message2, list(user.channel.id))
+
+/datum/tgs_api/v5/proc/SendChatMessageRaw(datum/tgs_message_content/message2, list/channel_ids)
 	message2 = UpgradeDeprecatedChatMessage(message2)
 
-	if (!length(channels))
+	if (!length(channel_ids))
 		return
 
 	var/list/data = message2._interop_serialize()
-	data[DMAPI5_CHAT_MESSAGE_CHANNEL_IDS] = channels
+	data[DMAPI5_CHAT_MESSAGE_CHANNEL_IDS] = channel_ids
 	if(intercepted_message_queue)
 		intercepted_message_queue += list(data)
-	else
-		Bridge(DMAPI5_BRIDGE_COMMAND_CHAT_SEND, list(DMAPI5_BRIDGE_PARAMETER_CHAT_MESSAGE = data))
+		return
 
-/datum/tgs_api/v5/ChatPrivateMessage(datum/tgs_message_content/message2, datum/tgs_chat_user/user)
-	message2 = UpgradeDeprecatedChatMessage(message2)
-	var/list/data = message2._interop_serialize()
-	data[DMAPI5_CHAT_MESSAGE_CHANNEL_IDS] = list(user.channel.id)
-	if(intercepted_message_queue)
-		intercepted_message_queue += list(data)
+	if(offline_message_queue)
+		offline_message_queue += list(data)
+		return
+
+	if(detached)
+		offline_message_queue = list(data)
+
+		WaitForReattach(FALSE)
+
+		data = offline_message_queue
+		offline_message_queue = null
+
+		for(var/queued_message in data)
+			SendChatDataRaw(queued_message)
 	else
-		Bridge(DMAPI5_BRIDGE_COMMAND_CHAT_SEND, list(DMAPI5_BRIDGE_PARAMETER_CHAT_MESSAGE = data))
+		SendChatDataRaw(data)
+
+/datum/tgs_api/v5/proc/SendChatDataRaw(list/data)
+	Bridge(DMAPI5_BRIDGE_COMMAND_CHAT_SEND, list(DMAPI5_BRIDGE_PARAMETER_CHAT_MESSAGE = data))
 
 /datum/tgs_api/v5/ChatChannelInfo()
 	RequireInitialBridgeResponse()

src/Tgstation.Server.Api/Models/Internal/DreamDaemonLaunchParameters.cs

@@ -98,6 +98,13 @@ public class DreamDaemonLaunchParameters
 		[ResponseOptions]
 		public uint? MapThreads { get; set; }
 
+		/// <summary>
+		/// If minidumps should be taken instead of full dumps.
+		/// </summary>
+		[Required]
+		[ResponseOptions]
+		public bool? Minidumps { get; set; }
+
 		/// <summary>
 		/// Check if we match a given set of <paramref name="otherParameters"/>. <see cref="StartupTimeout"/> is excluded.
 		/// </summary>
@@ -116,7 +123,7 @@ public bool CanApplyWithoutReboot(DreamDaemonLaunchParameters otherParameters)
 				&& AdditionalParameters == otherParameters.AdditionalParameters
 				&& StartProfiler == otherParameters.StartProfiler
 				&& LogOutput == otherParameters.LogOutput
-				&& MapThreads == otherParameters.MapThreads; // We intentionally don't check StartupTimeout, health check seconds, or health check dump as they don't matter in terms of the watchdog
+				&& MapThreads == otherParameters.MapThreads; // We intentionally don't check StartupTimeout, Minidumps, health check seconds, or health check dump as they don't matter in terms of the watchdog
 		}
 	}
 }

src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs

@@ -117,5 +117,10 @@ public enum DreamDaemonRights : ulong
 		/// User can use <see cref="Models.Request.DreamDaemonRequest.BroadcastMessage"/>.
 		/// </summary>
 		BroadcastMessage = 1 << 20,
+
+		/// <summary>
+		/// User can use <see cref="Models.Internal.DreamDaemonLaunchParameters.Minidumps"/>.
+		/// </summary>
+		SetMinidumps = 1 << 21,
 	}
 }

src/Tgstation.Server.Api/Tgstation.Server.Api.csproj

@@ -27,7 +27,7 @@
     <!-- Usage: HTTP constants reference -->
     <PackageReference Include="Microsoft.AspNetCore.Http.Extensions" Version="2.2.0" />
     <!-- Usage: Decoding the 'nbf' property of JWTs -->
-    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="7.1.2" />
+    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="7.3.0" />
     <!-- Usage: Primary JSON library -->
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <!-- Usage: Data model annotating -->

src/Tgstation.Server.Host/.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "dotnet-ef": {
-      "version": "8.0.0",
+      "version": "8.0.1",
       "commands": [
         "dotnet-ef"
       ]

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -852,13 +852,15 @@ async ValueTask VerifyApi(
 		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in <see langword="true"/> if compilation succeeded, <see langword="false"/> otherwise.</returns>
 		async ValueTask<bool> RunDreamMaker(IEngineExecutableLock engineLock, Models.CompileJob job, CancellationToken cancellationToken)
 		{
+			var environment = await engineLock.LoadEnv(logger, true, cancellationToken);
 			var arguments = engineLock.FormatCompilerArguments($"{job.DmeName}.{DmeExtension}");
 
 			await using var dm = processExecutor.LaunchProcess(
 				engineLock.CompilerExePath,
 				ioManager.ResolvePath(
 					job.DirectoryName!.Value.ToString()),
 				arguments,
+				environment,
 				readStandardHandles: true,
 				noShellExecute: true);
 
@@ -953,6 +955,12 @@ ValueTask CleanupFailedCompile(Models.CompileJob job, IRemoteDeploymentManager r
 		{
 			async ValueTask CleanDir()
 			{
+				if (sessionConfiguration.DelayCleaningFailedDeployments)
+				{
+					logger.LogDebug("Not cleaning up errored deployment directory {guid} due to config.", job.DirectoryName);
+					return;
+				}
+
 				logger.LogTrace("Cleaning compile directory...");
 				var jobPath = job.DirectoryName!.Value.ToString();
 				try

src/Tgstation.Server.Host/Components/Engine/ByondInstallation.cs

@@ -6,6 +6,7 @@
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Deployment;
+using Tgstation.Server.Host.IO;
 
 namespace Tgstation.Server.Host.Components.Engine
 {
@@ -32,6 +33,9 @@ sealed class ByondInstallation : EngineInstallationBase
 		/// <inheritdoc />
 		public override bool PreferFileLogging => false;
 
+		/// <inheritdoc />
+		public override bool UseDotnetDump => false;
+
 		/// <inheritdoc />
 		public override Task InstallationTask { get; }
 
@@ -75,19 +79,22 @@ static string VisibilityWord(DreamDaemonVisibility visibility)
 		/// <summary>
 		/// Initializes a new instance of the <see cref="ByondInstallation"/> class.
 		/// </summary>
+		/// <param name="installationIOManager">The <see cref="IIOManager"/> for the <see cref="EngineInstallationBase"/>.</param>
 		/// <param name="installationTask">The value of <see cref="InstallationTask"/>.</param>
 		/// <param name="version">The value of <see cref="Version"/>.</param>
 		/// <param name="dreamDaemonPath">The value of <see cref="ServerExePath"/>.</param>
 		/// <param name="dreamMakerPath">The value of <see cref="CompilerExePath"/>.</param>
 		/// <param name="supportsCli">If a CLI application is being used.</param>
 		/// <param name="supportsMapThreads">The value of <see cref="supportsMapThreads"/>.</param>
 		public ByondInstallation(
+			IIOManager installationIOManager,
 			Task installationTask,
 			EngineVersion version,
 			string dreamDaemonPath,
 			string dreamMakerPath,
 			bool supportsCli,
 			bool supportsMapThreads)
+			: base(installationIOManager)
 		{
 			InstallationTask = installationTask ?? throw new ArgumentNullException(nameof(installationTask));
 			ArgumentNullException.ThrowIfNull(version);

src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs

@@ -88,21 +88,22 @@ public override IEngineInstallation CreateInstallation(EngineVersion version, st
 		{
 			CheckVersionValidity(version);
 
-			var binPathForVersion = IOManager.ConcatPath(path, ByondBinPath);
+			var installationIOManager = new ResolvingIOManager(IOManager, path);
 			var supportsMapThreads = version.Version >= MapThreadsVersion;
 
 			return new ByondInstallation(
+				installationIOManager,
 				installationTask,
 				version,
-				IOManager.ResolvePath(
-					IOManager.ConcatPath(
-						binPathForVersion,
+				installationIOManager.ResolvePath(
+					installationIOManager.ConcatPath(
+						ByondBinPath,
 						GetDreamDaemonName(
 							version.Version!,
 							out var supportsCli))),
-				IOManager.ResolvePath(
-					IOManager.ConcatPath(
-						binPathForVersion,
+				installationIOManager.ResolvePath(
+					installationIOManager.ConcatPath(
+						ByondBinPath,
 						DreamMakerName)),
 				supportsCli,
 				supportsMapThreads);

src/Tgstation.Server.Host/Components/Engine/DelegatingEngineInstaller.cs

@@ -41,8 +41,8 @@ public ValueTask<IEngineInstallationData> DownloadVersion(EngineVersion version,
 			=> DelegateCall(version, installer => installer.DownloadVersion(version, jobProgressReporter, cancellationToken));
 
 		/// <inheritdoc />
-		public ValueTask Install(EngineVersion version, string path, CancellationToken cancellationToken)
-			=> DelegateCall(version, installer => installer.Install(version, path, cancellationToken));
+		public ValueTask Install(EngineVersion version, string path, bool deploymentPipelineProcesses, CancellationToken cancellationToken)
+			=> DelegateCall(version, installer => installer.Install(version, path, deploymentPipelineProcesses, cancellationToken));
 
 		/// <inheritdoc />
 		public ValueTask TrustDmbPath(EngineVersion version, string fullDmbPath, CancellationToken cancellationToken)

src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs

@@ -36,6 +36,9 @@ class EngineExecutableLock : ReferenceCounter<IEngineInstallation>, IEngineExecu
 		/// <inheritdoc />
 		public Task InstallationTask => Instance.InstallationTask;
 
+		/// <inheritdoc />
+		public bool UseDotnetDump => Instance.UseDotnetDump;
+
 		/// <inheritdoc />
 		public void DoNotDeleteThisSession() => DangerousDropReference();
 
@@ -62,5 +65,9 @@ public ValueTask StopServerProcess(ILogger logger, IProcess process, string acce
 				accessIdentifier,
 				port,
 				cancellationToken);
+
+		/// <inheritdoc />
+		public ValueTask<Dictionary<string, string>?> LoadEnv(ILogger logger, bool forCompiler, CancellationToken cancellationToken)
+			=> Instance.LoadEnv(logger, forCompiler, cancellationToken);
 	}
 }