From c7b577d03eb92f78b0a0b3734fdb107c9ec52985 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 15 Oct 2023 23:27:22 -0400 Subject: [PATCH 001/138] Add another possible win32 error to dump tests --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 58e101e0a62..d9c418a385c 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -359,7 +359,8 @@ async Task DumpTests(CancellationToken cancellationToken) || job.ExceptionDetails.Contains("BetterWin32Errors.Win32Exception: 3489660936: Unknown error (0xd0000008)") || job.ExceptionDetails.Contains("System.InvalidOperationException: No process is associated with this object.") || job.ExceptionDetails.Contains("BetterWin32Errors.Win32Exception: 2147942424: The program issued a command but the command length is incorrect.") - || job.ExceptionDetails.Contains("BetterWin32Errors.Win32Exception: 2147942699: Only part of a ReadProcessMemory or WriteProcessMemory request was completed.")))) + || job.ExceptionDetails.Contains("BetterWin32Errors.Win32Exception: 2147942699: Only part of a ReadProcessMemory or WriteProcessMemory request was completed.") + || job.ExceptionDetails.Contains("BetterWin32Errors.Win32Exception: 3489660964: Unknown error (0xd0000024)")))) break; var restartJob = await instanceClient.DreamDaemon.Restart(cancellationToken); From 356a2c24f8ac65eef60f280fa8ab9612b3347985 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 17 Oct 2023 18:18:51 -0400 Subject: [PATCH 002/138] Find a good version for linux compat tests Needed https://www.byond.com/forum/post/2402692 fixed --- LinuxTestBed | 1 + .../Live/TestLiveServer.cs | 22 +++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) create mode 160000 LinuxTestBed diff --git a/LinuxTestBed b/LinuxTestBed new file mode 160000 index 00000000000..9865a43b508 --- /dev/null +++ b/LinuxTestBed @@ -0,0 +1 @@ +Subproject commit 9865a43b508a1c04857a807bdf6b20dd35dcf8bc diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 9e13a2ca578..64159824c5f 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1119,17 +1119,17 @@ async Task RunInstanceTests() { // Some earlier linux BYOND versions have a critical bug where replacing the directory in non-basic watchdogs causes the DreamDaemon cwd to change var canRunCompatTests = new PlatformIdentifier().IsWindows; - var compatTests = canRunCompatTests - ? FailFast( - instanceTest - .RunCompatTests( - new Version(510, 1346), - adminClient.Instances.CreateClient(compatInstance), - compatDMPort, - compatDDPort, - server.HighPriorityDreamDaemon, - cancellationToken)) - : Task.CompletedTask; + var compatTests = FailFast( + instanceTest + .RunCompatTests( + new PlatformIdentifier().IsWindows + ? new Version(510, 1346) + : new Version(512, 1451), // http://www.byond.com/forum/?forum=5&command=search&scope=local&text=resolved%3a512.1451 + adminClient.Instances.CreateClient(compatInstance), + compatDMPort, + compatDDPort, + server.HighPriorityDreamDaemon, + cancellationToken)); if (TestingUtils.RunningInGitHubActions) // they only have 2 cores, can't handle intense parallelization await compatTests; From 416fd0a9f0e8cefcd1338bef9ce1b9fed0f69253 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 19 Oct 2023 19:58:52 -0400 Subject: [PATCH 003/138] Delete empty directory wtf? --- LinuxTestBed | 1 - 1 file changed, 1 deletion(-) delete mode 160000 LinuxTestBed diff --git a/LinuxTestBed b/LinuxTestBed deleted file mode 160000 index 9865a43b508..00000000000 --- a/LinuxTestBed +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9865a43b508a1c04857a807bdf6b20dd35dcf8bc From cd61bd317fcfe001a36479c1360bf0f1db74fe49 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 19 Oct 2023 20:07:11 -0400 Subject: [PATCH 004/138] Up timeout for dump tests Addresses the failure in the last nightly --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index d9c418a385c..641085529cd 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -542,7 +542,7 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); - var timeout = 20; + var timeout = 60; DreamDaemonResponse ddStatus; do { From c11afb5ecc8273e21c0d9cf0f7eda92e51ac9479 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 19 Oct 2023 20:27:07 -0400 Subject: [PATCH 005/138] Add /tg/station workstation integration tests --- .../Live/TestLiveServer.cs | 213 +++++++++++++++++- 1 file changed, 211 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 64159824c5f..5a7da4471a4 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -10,6 +10,7 @@ using System.Net.Mime; using System.Net.Sockets; using System.Reflection; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -954,6 +955,216 @@ await Task.WhenAny( new LiveTestingServer(null, false).Dispose(); } + [TestMethod] + public async Task TestTgstationInteractive() => await TestTgstation(true); + + [TestMethod] + public async Task TestTgstationHeadless() => await TestTgstation(false); + + async ValueTask TestTgstation(bool interactive) + { + // i'm only running this on dev machines, actions is too taxed + if (TestingUtils.RunningInGitHubActions) + Assert.Inconclusive("lol. lmao."); + + var discordConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN"); + + var procs = System.Diagnostics.Process.GetProcessesByName("byond"); + if (procs.Any()) + { + foreach (var proc in procs) + proc.Dispose(); + + // Inconclusive and not fail because we don't want to unexpectedly kill a dev's BYOND.exe + Assert.Inconclusive("Cannot run server test because DreamDaemon will not start headless while the BYOND pager is running!"); + } + + + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + builder.SetMinimumLevel(LogLevel.Trace); + }); + using var server = new LiveTestingServer(null, true); + + TerminateAllDDs(); + + using var serverCts = new CancellationTokenSource(); + var cancellationToken = serverCts.Token; + var serverTask = server.Run(cancellationToken); + try + { + using var adminClient = await CreateAdminClient(server.Url, cancellationToken); + + var instanceManagerTest = new InstanceManagerTest(adminClient, server.Directory); + var instance = await instanceManagerTest.CreateTestInstance("TgTestInstance", cancellationToken); + var instanceClient = adminClient.Instances.CreateClient(instance); + + + var ddUpdateTask = instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + LogOutput = true, + }, cancellationToken); + var dmUpdateTask = instanceClient.DreamMaker.Update(new DreamMakerRequest + { + ApiValidationSecurityLevel = DreamDaemonSecurity.Trusted, + }, cancellationToken); + + var ioManager = new Host.IO.DefaultIOManager(); + var repoPath = ioManager.ConcatPath(instance.Path, "Repository"); + var jobsTest = new JobsRequiredTest(instanceClient.Jobs); + var postWriteHandler = (Host.IO.IPostWriteHandler)(new PlatformIdentifier().IsWindows + ? new Host.IO.WindowsPostWriteHandler() + : new Host.IO.PosixPostWriteHandler(loggerFactory.CreateLogger())); + var localRepoPath = Environment.GetEnvironmentVariable("TGS_LOCAL_TG_REPO"); + Task jobWaitTask; + if (!String.IsNullOrWhiteSpace(localRepoPath)) + { + await ioManager.CopyDirectory( + Enumerable.Empty(), + (src, dest) => + { + if (postWriteHandler.NeedsPostWrite(src)) + postWriteHandler.HandleWrite(dest); + + return Task.CompletedTask; + }, + ioManager.ConcatPath( + localRepoPath, + ".git"), + ioManager.ConcatPath( + repoPath, + ".git"), + null, + cancellationToken); + + IProcessExecutor processExecutor = null; + processExecutor = new ProcessExecutor( + new PlatformIdentifier().IsWindows + ? new WindowsProcessFeatures(loggerFactory.CreateLogger()) + : new PosixProcessFeatures(new Lazy(() => processExecutor), ioManager, loggerFactory.CreateLogger()), + ioManager, + loggerFactory.CreateLogger(), + loggerFactory); + + async ValueTask RunGitCommand(string args) + { + await using var gitRemoteOriginFixProc = processExecutor.LaunchProcess( + "git", + repoPath, + args, + null, + true, + true); + + int? exitCode; + using (cancellationToken.Register(gitRemoteOriginFixProc.Terminate)) + exitCode = await gitRemoteOriginFixProc.Lifetime; + + loggerFactory.CreateLogger("TgTest").LogInformation("git {args} output:{newLine}{output}",args, Environment.NewLine, await gitRemoteOriginFixProc.GetCombinedOutput(cancellationToken)); + Assert.AreEqual(0, exitCode); + } + + await RunGitCommand("remote set-url origin https://github.com/tgstation/tgstation"); + await RunGitCommand("checkout -f master"); + await RunGitCommand("reset --hard origin/master"); + + jobWaitTask = Task.CompletedTask; + } + else + { + var repoResponse = await instanceClient.Repository.Clone(new RepositoryCreateRequest + { + Origin = new Uri("https://github.com/tgstation/tgstation"), + }, cancellationToken); + jobWaitTask = jobsTest.WaitForJob(repoResponse.ActiveJob, 300, false, null, cancellationToken); + } + + await Task.WhenAll(jobWaitTask, ddUpdateTask, dmUpdateTask); + + var depsBytesTask = ioManager.ReadAllBytes( + ioManager.ConcatPath(repoPath, "dependencies.sh"), + cancellationToken); + + var scriptsCopyTask = ioManager.CopyDirectory( + Enumerable.Empty(), + (src, dest) => + { + if (postWriteHandler.NeedsPostWrite(src)) + postWriteHandler.HandleWrite(dest); + + return Task.CompletedTask; + }, + ioManager.ConcatPath( + repoPath, + "tools", + "tgs_scripts"), + ioManager.ConcatPath( + instance.Path, + "Configuration", + "EventScripts"), + null, + cancellationToken); + + var dependenciesSh = Encoding.UTF8.GetString(await depsBytesTask); + var lines = dependenciesSh.Split("\n", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + const string MajorPrefix = "export BYOND_MAJOR="; + var major = Int32.Parse(lines.First(x => x.StartsWith(MajorPrefix)).Substring(MajorPrefix.Length)); + const string MinorPrefix = "export BYOND_MINOR="; + var minor = Int32.Parse(lines.First(x => x.StartsWith(MinorPrefix)).Substring(MinorPrefix.Length)); + + var byondJob = await instanceClient.Byond.SetActiveVersion(new ByondVersionRequest + { + Version = new Version(major, minor) + }, null, cancellationToken); + + var byondJobTask = jobsTest.WaitForJob(byondJob.InstallJob, 60, false, null, cancellationToken); + + await Task.WhenAll(scriptsCopyTask, byondJobTask); + + var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); + + await jobsTest.WaitForJob(compileJob, 180, false, null, cancellationToken); + + var startJob = await instanceClient.DreamDaemon.Start(cancellationToken); + await jobsTest.WaitForJob(startJob, 30, false, null, cancellationToken); + + var compileJob2 = await instanceClient.DreamMaker.Compile(cancellationToken); + + await jobsTest.WaitForJob(compileJob2, 360, false, null, cancellationToken); + + if (interactive) + { + bool updated = false; + while (true) + { + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + + var status = await instanceClient.DreamDaemon.Read(cancellationToken); + + if (updated) + { + if (status.Status == WatchdogStatus.Offline) + break; + } + else if (status.StagedCompileJob == null) + { + updated = true; + await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + SoftShutdown = true + }, cancellationToken); + } + } + } + } + finally + { + serverCts.Cancel(); + await serverTask; + } + } + [TestMethod] public async Task TestStandardTgsOperation() { @@ -969,9 +1180,7 @@ public async Task TestStandardTgsOperation() } using (var currentProcess = System.Diagnostics.Process.GetCurrentProcess()) - { Assert.AreEqual(ProcessPriorityClass.Normal, currentProcess.PriorityClass); - } var maximumTestMinutes = TestingUtils.RunningInGitHubActions ? 90 : 20; using var hardCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(maximumTestMinutes)); From 65d9785c240cda3f124fa24725fb56d8f2bff614 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 20 Oct 2023 21:41:14 -0400 Subject: [PATCH 006/138] Fix merge build errors --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 358d20edb72..260a9c457d2 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1027,7 +1027,7 @@ await ioManager.CopyDirectory( if (postWriteHandler.NeedsPostWrite(src)) postWriteHandler.HandleWrite(dest); - return Task.CompletedTask; + return ValueTask.CompletedTask; }, ioManager.ConcatPath( localRepoPath, @@ -1080,7 +1080,7 @@ async ValueTask RunGitCommand(string args) jobWaitTask = jobsTest.WaitForJob(repoResponse.ActiveJob, 300, false, null, cancellationToken); } - await Task.WhenAll(jobWaitTask, ddUpdateTask, dmUpdateTask); + await Task.WhenAll(jobWaitTask, ddUpdateTask.AsTask(), dmUpdateTask.AsTask()); var depsBytesTask = ioManager.ReadAllBytes( ioManager.ConcatPath(repoPath, "dependencies.sh"), @@ -1093,7 +1093,7 @@ async ValueTask RunGitCommand(string args) if (postWriteHandler.NeedsPostWrite(src)) postWriteHandler.HandleWrite(dest); - return Task.CompletedTask; + return ValueTask.CompletedTask; }, ioManager.ConcatPath( repoPath, @@ -1120,7 +1120,7 @@ async ValueTask RunGitCommand(string args) var byondJobTask = jobsTest.WaitForJob(byondJob.InstallJob, 60, false, null, cancellationToken); - await Task.WhenAll(scriptsCopyTask, byondJobTask); + await Task.WhenAll(scriptsCopyTask.AsTask(), byondJobTask); var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); From 232aa679c2f50475c098a3cff25d36a340899e98 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 20 Oct 2023 19:46:39 -0400 Subject: [PATCH 007/138] Version bump client and common library versions --- build/Version.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index ae9b61b8d14..b2f8d5265bd 100644 --- a/build/Version.props +++ b/build/Version.props @@ -6,9 +6,9 @@ 5.16.2 4.7.1 9.12.0 - 6.0.1 + 7.0.0 11.1.2 - 12.1.2 + 13.0.0 6.5.3 5.6.1 1.4.0 From 6d674a71c1d298b19292697bbf524d74f285f65e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 20 Oct 2023 19:56:54 -0400 Subject: [PATCH 008/138] Add explicit dependency on common csproj to client --- src/Tgstation.Server.Client/Tgstation.Server.Client.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index af89800112b..6868098fa69 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -11,6 +11,7 @@ + From 4c846014350bb0d3ecc0be4dc2ea7b455faa446c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 16 Oct 2023 21:31:26 -0400 Subject: [PATCH 009/138] Fix DMAPI not returning a value for world.TgsSecurityLevel() --- build/Version.props | 2 +- src/DMAPI/tgs.dm | 2 +- src/DMAPI/tgs/core/core.dm | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/Version.props b/build/Version.props index 6a908f98fd5..ef2f6523960 100644 --- a/build/Version.props +++ b/build/Version.props @@ -9,7 +9,7 @@ 6.0.1 11.1.2 12.1.2 - 6.5.3 + 6.5.4 5.6.1 1.4.0 1.2.1 diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index 6187a67825a..d0466b806ff 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,6 +1,6 @@ // tgstation-server DMAPI -#define TGS_DMAPI_VERSION "6.5.3" +#define TGS_DMAPI_VERSION "6.5.4" // All functions and datums outside this document are subject to change with any version and should not be relied on. diff --git a/src/DMAPI/tgs/core/core.dm b/src/DMAPI/tgs/core/core.dm index 41a04733945..aa4084904b7 100644 --- a/src/DMAPI/tgs/core/core.dm +++ b/src/DMAPI/tgs/core/core.dm @@ -153,4 +153,4 @@ /world/TgsSecurityLevel() var/datum/tgs_api/api = TGS_READ_GLOBAL(tgs) if(api) - api.SecurityLevel() + return api.SecurityLevel() From c8984cc109134062e7e5546d7ea5eb450d420cd2 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 16 Oct 2023 21:59:41 -0400 Subject: [PATCH 010/138] Fix a log message typo --- src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs index 47a67728aba..7595a70a431 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs @@ -153,7 +153,7 @@ GitHubClient GetOrCreateClient(string accessToken) rateLimitInfo.Reset.ToString("o")); else logger.LogDebug( - "Requested GitHub client has {remainingRequests} requests remaining after the usage {lastUse}. Limit resets at {resetTime}", + "Requested GitHub client has {remainingRequests} requests remaining after the usage at {lastUse}. Limit resets at {resetTime}", rateLimitInfo.Remaining, lastUsed, rateLimitInfo.Reset.ToString("o")); From 8a5d0d40e585a3d19a2d51979cab76fa14b24816 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 16 Oct 2023 21:59:44 -0400 Subject: [PATCH 011/138] Test DMAPI security level and CWD is Live --- tests/DMAPI/LongRunning/Test.dm | 26 +++++++ .../LongRunning/long_running_test_rooted.dme | 17 +++++ .../Live/Instance/ConfigurationTest.cs | 10 ++- .../Live/Instance/InstanceTest.cs | 3 +- .../Live/Instance/WatchdogTest.cs | 68 ++++++++++++++++++- tgstation-server.sln | 1 + 6 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 tests/DMAPI/LongRunning/long_running_test_rooted.dme diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index bed428eceb2..d3515034ee8 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -11,6 +11,12 @@ dab() TgsNew(new /datum/tgs_event_handler/impl, TGS_SECURITY_SAFE) + var/sec = TgsSecurityLevel() + if(isnull(sec)) + FailTest("TGS Security level was null!") + + log << "Running in security level: [sec]" + if(params["expect_chat_channels"]) var/list/channels = TgsChatChannelInfo() if(!length(channels)) @@ -140,6 +146,26 @@ var/run_bridge_test kajigger_test = TRUE return "we love casting spells" + var/expected_path = data["vaporeon"] + if(expected_path) + var/command + if(world.system_type == MS_WINDOWS) + command = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe -ExecutionPolicy Bypass -Command \"if('[expected_path]' -ne $(Get-Location)){ (Get-Location).Path | Out-File \"fuck_up.txt\"; exit 1; }\"" + else + command = "if \[\[ \"[expected_path]\" != \"$(pwd)\" \]\]; then echo $(pwd) > fuck_up.txt; exit 1; fi" + + world.log << "shell: [command]" + var/exitCode = shell(command) + + if(isnull(exitCode) || exitCode != 0) + var/fuck_up_reason = "DIDN'T READ" + if(fexists("fuck_up.txt")) + fuck_up_reason = "COULDN'T READ" + fuck_up_reason = file2text("fuck_up.txt") + return "Dir check shell command failed with code [exitCode || "null"]: [fuck_up_reason]" + + return "is the most pokemon of all time" + TgsChatBroadcast(new /datum/tgs_message_content("Recieved non-tgs topic: `[T]`")) return "feck" diff --git a/tests/DMAPI/LongRunning/long_running_test_rooted.dme b/tests/DMAPI/LongRunning/long_running_test_rooted.dme new file mode 100644 index 00000000000..d8e9d61c967 --- /dev/null +++ b/tests/DMAPI/LongRunning/long_running_test_rooted.dme @@ -0,0 +1,17 @@ +// Hand crafted DME, will not work if saved with DreamMaker + +// BEGIN_INTERNALS +// END_INTERNALS + +// BEGIN_FILE_DIR +#define FILE_DIR . +// END_FILE_DIR + +// BEGIN_PREFERENCES +// END_PREFERENCES + +// BEGIN_INCLUDE +#include "tests/DMAPI/LongRunning/Config.dm" +#include "tests/DMAPI/test_prelude.dm" +#include "tests/DMAPI/LongRunning/Test.dm" +// END_INCLUDE diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index 1008406cd78..2dea3e5b124 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -92,7 +92,7 @@ await configurationClient.Write(new ConfigurationFileRequest await configurationClient.CreateDirectory(staticDir, cancellationToken); } - public Task SetupDMApiTests(CancellationToken cancellationToken) + public Task SetupDMApiTests(bool includingRoot, CancellationToken cancellationToken) { // just use an I/O manager here var ioManager = new DefaultIOManager(); @@ -104,6 +104,12 @@ public Task SetupDMApiTests(CancellationToken cancellationToken) ioManager.ConcatPath(instance.Path, "Repository", "tests", "DMAPI"), null, cancellationToken), + includingRoot + ? ioManager.CopyFile( + "../../../../DMAPI/LongRunning/long_running_test_rooted.dme", + ioManager.ConcatPath(instance.Path, "Repository", "long_running_test_rooted.dme"), + cancellationToken) + : Task.CompletedTask, ioManager.CopyDirectory( Enumerable.Empty(), null, @@ -175,8 +181,8 @@ async Task SequencedApiTests(CancellationToken cancellationToken) } public Task RunPreWatchdog(CancellationToken cancellationToken) => Task.WhenAll( + SetupDMApiTests(false, cancellationToken), SequencedApiTests(cancellationToken), - SetupDMApiTests(cancellationToken), TestPregeneratedFilesExist(cancellationToken)); } } diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 9265e0e7f6e..4457994d792 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -63,6 +63,7 @@ public async Task RunTests( await chatTask; await dmTask; + await configTest.SetupDMApiTests(true, cancellationToken); await byondTask; await new WatchdogTest( @@ -172,7 +173,7 @@ await Task.WhenAll( dmUpdateRequest, cloneRequest); - var configSetupTask = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata).SetupDMApiTests(cancellationToken); + var configSetupTask = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata).SetupDMApiTests(true, cancellationToken); if (TestingUtils.RunningInGitHubActions || String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN")) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 641085529cd..bd1671e05e8 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -131,6 +131,8 @@ await Task.WhenAll( await RunLongRunningTestThenUpdate(cancellationToken); + await TestDDIsntResolvingSymlink(cancellationToken); + await RunLongRunningTestThenUpdateWithNewDme(cancellationToken); await RunLongRunningTestThenUpdateWithByondVersionSwitch(cancellationToken); @@ -150,6 +152,70 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest System.Console.WriteLine($"TEST: END WATCHDOG TESTS {instanceClient.Metadata.Name}"); } + async ValueTask TestDDIsntResolvingSymlink(CancellationToken cancellationToken) + { + System.Console.WriteLine("STARTING SYMLINK RESOLVE TEST"); + var deployTask = DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); + + var previous = await instanceClient.DreamDaemon.Read(cancellationToken); + if (previous.SecurityLevel.Value != DreamDaemonSecurity.Trusted) + { + var updated = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + SecurityLevel = DreamDaemonSecurity.Trusted + }, cancellationToken); + + Assert.AreEqual(DreamDaemonSecurity.Trusted, updated.SecurityLevel); + } + + var startState = await deployTask; + + await WaitForJob(await StartDD(cancellationToken), 30, false, null, cancellationToken); + + var command = topicClient.SanitizeString( + Path.GetFullPath( + Path.Combine( + instanceClient.Metadata.Path, + "Game", + "Live"))); + command = $"vaporeon={command}"; + + var topicRequestResult = await topicClient.SendTopic( + IPAddress.Loopback, + command, + ddPort, + cancellationToken); + + Assert.IsNotNull(topicRequestResult); + Assert.AreEqual("is the most pokemon of all time", topicRequestResult.StringData); + + await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); + var newState = await TellWorldToReboot(cancellationToken); + + Assert.AreNotEqual(startState.ActiveCompileJob.Id, newState.ActiveCompileJob.Id); + topicRequestResult = await topicClient.SendTopic( + IPAddress.Loopback, + command, + ddPort, + cancellationToken); + + Assert.IsNotNull(topicRequestResult); + Assert.AreEqual("is the most pokemon of all time", topicRequestResult.StringData); + + await instanceClient.DreamDaemon.Shutdown(cancellationToken); + if (previous.SecurityLevel.Value != DreamDaemonSecurity.Trusted) + { + var updated = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + SecurityLevel = previous.SecurityLevel + }, cancellationToken); + + Assert.AreEqual(previous.SecurityLevel, updated.SecurityLevel); + } + + System.Console.WriteLine("END SYMLINK RESOLVE TEST"); + } + async Task InteropTestsForLongRunningDme(CancellationToken cancellationToken) { await StartAndLeaveRunning(cancellationToken); @@ -1101,7 +1167,7 @@ async Task DeployTestDme(string dmeName, DreamDaemonSecurit var refreshed = await instanceClient.DreamMaker.Update(new DreamMakerRequest { ApiValidationSecurityLevel = deploymentSecurity, - ProjectName = $"tests/DMAPI/{dmeName}", + ProjectName = dmeName.Contains("rooted") ? dmeName : $"tests/DMAPI/{dmeName}", RequireDMApiValidation = requireApi, Timeout = TimeSpan.FromMilliseconds(1), }, cancellationToken); diff --git a/tgstation-server.sln b/tgstation-server.sln index 6d183435b87..20eef245e10 100644 --- a/tgstation-server.sln +++ b/tgstation-server.sln @@ -157,6 +157,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LongRunning", "LongRunning" tests\DMAPI\LongRunning\Config.dm = tests\DMAPI\LongRunning\Config.dm tests\DMAPI\LongRunning\long_running_test.dme = tests\DMAPI\LongRunning\long_running_test.dme tests\DMAPI\LongRunning\long_running_test_copy.dme = tests\DMAPI\LongRunning\long_running_test_copy.dme + tests\DMAPI\LongRunning\long_running_test_rooted.dme = tests\DMAPI\LongRunning\long_running_test_rooted.dme tests\DMAPI\LongRunning\Test.dm = tests\DMAPI\LongRunning\Test.dm EndProjectSection EndProject From 2f5e62c9a011b591d59860613e93fc50cb9ff4f4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 20 Oct 2023 23:30:39 -0400 Subject: [PATCH 012/138] A note to help future errors on this assert --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 5a7da4471a4..f058d9795fe 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1471,7 +1471,7 @@ await FailFast( await WatchdogTest.TellWorldToReboot2(instanceClient, WatchdogTest.StaticTopicClient, mainDDPort, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); - Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); + Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); // if this assert fails, you likely have to crack open the debugger and read test_fail_reason.txt manually Assert.IsNull(dd.StagedCompileJob); Assert.AreEqual(initialStaged, dd.ActiveCompileJob.Id); From a6cb81902fe075f0f7361bc433db3c1a0be8c2d0 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 20 Oct 2023 23:44:44 -0400 Subject: [PATCH 013/138] Fix RuntimeInformation after reattach having null SecurityLevel and Visibility --- build/Version.props | 2 +- src/DMAPI/tgs/v5/__interop_version.dm | 2 +- .../Components/Interop/Bridge/RuntimeInformation.cs | 12 ++++++++---- .../Components/Session/ReattachInformation.cs | 6 ++---- .../Components/Session/SessionControllerFactory.cs | 13 ++++++++----- .../Models/ReattachInformationBase.cs | 4 ++-- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/build/Version.props b/build/Version.props index ef2f6523960..4a4c556f247 100644 --- a/build/Version.props +++ b/build/Version.props @@ -10,7 +10,7 @@ 11.1.2 12.1.2 6.5.4 - 5.6.1 + 5.6.2 1.4.0 1.2.1 1.0.2 diff --git a/src/DMAPI/tgs/v5/__interop_version.dm b/src/DMAPI/tgs/v5/__interop_version.dm index 5d3d491a736..1b52b31d6a7 100644 --- a/src/DMAPI/tgs/v5/__interop_version.dm +++ b/src/DMAPI/tgs/v5/__interop_version.dm @@ -1 +1 @@ -"5.6.1" +"5.6.2" diff --git a/src/Tgstation.Server.Host/Components/Interop/Bridge/RuntimeInformation.cs b/src/Tgstation.Server.Host/Components/Interop/Bridge/RuntimeInformation.cs index 5d0d13b7fa1..b7d05b67d79 100644 --- a/src/Tgstation.Server.Host/Components/Interop/Bridge/RuntimeInformation.cs +++ b/src/Tgstation.Server.Host/Components/Interop/Bridge/RuntimeInformation.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Linq; +using Newtonsoft.Json; + using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Deployment; @@ -42,12 +44,14 @@ public sealed class RuntimeInformation : ChatUpdate /// /// The level of the launch. /// - public DreamDaemonSecurity? SecurityLevel { get; } + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Include)] + public DreamDaemonSecurity SecurityLevel { get; } /// /// The level of the launch. /// - public DreamDaemonVisibility? Visibility { get; } + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Include)] + public DreamDaemonVisibility Visibility { get; } /// /// The s in the launch. @@ -70,8 +74,8 @@ public RuntimeInformation( IDmbProvider dmbProvider, Version serverVersion, string instanceName, - DreamDaemonSecurity? securityLevel, - DreamDaemonVisibility? visibility, + DreamDaemonSecurity securityLevel, + DreamDaemonVisibility visibility, ushort serverPort, bool apiValidateOnly) : base(chatTrackingContext?.Channels ?? throw new ArgumentNullException(nameof(chatTrackingContext))) diff --git a/src/Tgstation.Server.Host/Components/Session/ReattachInformation.cs b/src/Tgstation.Server.Host/Components/Session/ReattachInformation.cs index 2c9007f4f35..6c429d2d0a6 100644 --- a/src/Tgstation.Server.Host/Components/Session/ReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Session/ReattachInformation.cs @@ -76,13 +76,11 @@ internal ReattachInformation( Dmb = dmb ?? throw new ArgumentNullException(nameof(dmb)); ProcessId = process?.Id ?? throw new ArgumentNullException(nameof(process)); RuntimeInformation = runtimeInformation ?? throw new ArgumentNullException(nameof(runtimeInformation)); - if (!runtimeInformation.SecurityLevel.HasValue) - throw new ArgumentException("runtimeInformation must have a valid SecurityLevel!", nameof(runtimeInformation)); AccessIdentifier = accessIdentifier ?? throw new ArgumentNullException(nameof(accessIdentifier)); - LaunchSecurityLevel = runtimeInformation.SecurityLevel.Value; - LaunchVisibility = runtimeInformation.Visibility.Value; + LaunchSecurityLevel = runtimeInformation.SecurityLevel; + LaunchVisibility = runtimeInformation.Visibility; Port = port; runtimeInformationLock = new object(); diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index 96228c3c327..939632524ab 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -336,7 +336,8 @@ public async Task LaunchNew( var runtimeInformation = CreateRuntimeInformation( dmbProvider, chatTrackingContext, - launchParameters, + launchParameters.SecurityLevel.Value, + launchParameters.Visibility.Value, apiValidate); var reattachInformation = new ReattachInformation( @@ -430,7 +431,8 @@ public async Task Reattach( var runtimeInformation = CreateRuntimeInformation( reattachInformation.Dmb, chatTrackingContext, - null, + reattachInformation.LaunchSecurityLevel, + reattachInformation.LaunchVisibility, false); reattachInformation.SetRuntimeInformation(runtimeInformation); @@ -632,15 +634,16 @@ async Task LogDDOutput(IProcess process, string outputFilePath, bool cliSupporte RuntimeInformation CreateRuntimeInformation( IDmbProvider dmbProvider, IChatTrackingContext chatTrackingContext, - DreamDaemonLaunchParameters launchParameters, + DreamDaemonSecurity securityLevel, + DreamDaemonVisibility visibility, bool apiValidateOnly) => new ( chatTrackingContext, dmbProvider, assemblyInformationProvider.Version, instance.Name, - launchParameters?.SecurityLevel, - launchParameters?.Visibility, + securityLevel, + visibility, serverPortProvider.HttpApiPort, apiValidateOnly); diff --git a/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs b/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs index 76dfbfda36c..5cad9efbce2 100644 --- a/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs +++ b/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs @@ -32,13 +32,13 @@ public abstract class ReattachInformationBase : DMApiParameters /// The level DreamDaemon was launched with. /// [Required] - public DreamDaemonSecurity? LaunchSecurityLevel { get; set; } + public DreamDaemonSecurity LaunchSecurityLevel { get; set; } /// /// The DreamDaemon was launched with. /// [Required] - public DreamDaemonVisibility? LaunchVisibility { get; set; } + public DreamDaemonVisibility LaunchVisibility { get; set; } /// /// Initializes a new instance of the class. From 6ab9fc51fcd68b908eabd945d8695ad4e8c5f813 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 00:16:36 -0400 Subject: [PATCH 014/138] Remove unnecessary `RequiredAttribute`s --- src/Tgstation.Server.Host/Models/ReattachInformationBase.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs b/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs index 5cad9efbce2..241fae5141a 100644 --- a/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs +++ b/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs @@ -1,5 +1,4 @@ using System; -using System.ComponentModel.DataAnnotations; using System.Globalization; using Tgstation.Server.Api.Models; @@ -31,13 +30,11 @@ public abstract class ReattachInformationBase : DMApiParameters /// /// The level DreamDaemon was launched with. /// - [Required] public DreamDaemonSecurity LaunchSecurityLevel { get; set; } /// /// The DreamDaemon was launched with. /// - [Required] public DreamDaemonVisibility LaunchVisibility { get; set; } /// From 42b09376d2d90f783452814333dc6c2bec08ef2a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 00:18:30 -0400 Subject: [PATCH 015/138] Fix documentation comment errors --- .../Components/Session/SessionControllerFactory.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index 939632524ab..cf0dea0468f 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -628,7 +628,8 @@ async Task LogDDOutput(IProcess process, string outputFilePath, bool cliSupporte /// /// The . /// The . - /// The if any. + /// The the server was launched with. + /// The the server was launched with. /// The value of . /// A new class. RuntimeInformation CreateRuntimeInformation( From 279671c7cbd8d2e5ea184e2461c42af14b2b11df Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 03:49:28 -0400 Subject: [PATCH 016/138] Remove failing and badly conceived test --- tests/DMAPI/LongRunning/Test.dm | 20 ------ .../Live/Instance/WatchdogTest.cs | 66 ------------------- 2 files changed, 86 deletions(-) diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index d3515034ee8..a77c3377b5d 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -146,26 +146,6 @@ var/run_bridge_test kajigger_test = TRUE return "we love casting spells" - var/expected_path = data["vaporeon"] - if(expected_path) - var/command - if(world.system_type == MS_WINDOWS) - command = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe -ExecutionPolicy Bypass -Command \"if('[expected_path]' -ne $(Get-Location)){ (Get-Location).Path | Out-File \"fuck_up.txt\"; exit 1; }\"" - else - command = "if \[\[ \"[expected_path]\" != \"$(pwd)\" \]\]; then echo $(pwd) > fuck_up.txt; exit 1; fi" - - world.log << "shell: [command]" - var/exitCode = shell(command) - - if(isnull(exitCode) || exitCode != 0) - var/fuck_up_reason = "DIDN'T READ" - if(fexists("fuck_up.txt")) - fuck_up_reason = "COULDN'T READ" - fuck_up_reason = file2text("fuck_up.txt") - return "Dir check shell command failed with code [exitCode || "null"]: [fuck_up_reason]" - - return "is the most pokemon of all time" - TgsChatBroadcast(new /datum/tgs_message_content("Recieved non-tgs topic: `[T]`")) return "feck" diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index bd1671e05e8..329fab6aa19 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -131,8 +131,6 @@ await Task.WhenAll( await RunLongRunningTestThenUpdate(cancellationToken); - await TestDDIsntResolvingSymlink(cancellationToken); - await RunLongRunningTestThenUpdateWithNewDme(cancellationToken); await RunLongRunningTestThenUpdateWithByondVersionSwitch(cancellationToken); @@ -152,70 +150,6 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest System.Console.WriteLine($"TEST: END WATCHDOG TESTS {instanceClient.Metadata.Name}"); } - async ValueTask TestDDIsntResolvingSymlink(CancellationToken cancellationToken) - { - System.Console.WriteLine("STARTING SYMLINK RESOLVE TEST"); - var deployTask = DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); - - var previous = await instanceClient.DreamDaemon.Read(cancellationToken); - if (previous.SecurityLevel.Value != DreamDaemonSecurity.Trusted) - { - var updated = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest - { - SecurityLevel = DreamDaemonSecurity.Trusted - }, cancellationToken); - - Assert.AreEqual(DreamDaemonSecurity.Trusted, updated.SecurityLevel); - } - - var startState = await deployTask; - - await WaitForJob(await StartDD(cancellationToken), 30, false, null, cancellationToken); - - var command = topicClient.SanitizeString( - Path.GetFullPath( - Path.Combine( - instanceClient.Metadata.Path, - "Game", - "Live"))); - command = $"vaporeon={command}"; - - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - command, - ddPort, - cancellationToken); - - Assert.IsNotNull(topicRequestResult); - Assert.AreEqual("is the most pokemon of all time", topicRequestResult.StringData); - - await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); - var newState = await TellWorldToReboot(cancellationToken); - - Assert.AreNotEqual(startState.ActiveCompileJob.Id, newState.ActiveCompileJob.Id); - topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - command, - ddPort, - cancellationToken); - - Assert.IsNotNull(topicRequestResult); - Assert.AreEqual("is the most pokemon of all time", topicRequestResult.StringData); - - await instanceClient.DreamDaemon.Shutdown(cancellationToken); - if (previous.SecurityLevel.Value != DreamDaemonSecurity.Trusted) - { - var updated = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest - { - SecurityLevel = previous.SecurityLevel - }, cancellationToken); - - Assert.AreEqual(previous.SecurityLevel, updated.SecurityLevel); - } - - System.Console.WriteLine("END SYMLINK RESOLVE TEST"); - } - async Task InteropTestsForLongRunningDme(CancellationToken cancellationToken) { await StartAndLeaveRunning(cancellationToken); From c48ce8deed0103bab9365871ccf6ad56a817f0d6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 12:00:55 -0400 Subject: [PATCH 017/138] Fix deployments always timing out if DMAPI validation fails --- .../Components/Session/SessionController.cs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 2da8c03b58f..c792054b83b 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -192,6 +192,11 @@ async Task Wrap() /// volatile Task rebootGate; + /// + /// for shutting down the server if it is taking too long after validation. + /// + volatile Task postValidationShutdownTask; + /// /// The number of currently active calls to from TgsReboot(). /// @@ -307,6 +312,9 @@ public SessionController( { var exitCode = await process.Lifetime; await postLifetimeCallback(); + if (postValidationShutdownTask != null) + await postValidationShutdownTask; + return exitCode; } @@ -655,12 +663,40 @@ void CheckDisposed() throw new ObjectDisposedException(nameof(SessionController)); } + /// + /// Terminates the server after ten seconds if it does not exit. + /// + /// A that this method s before executing. If the is , this method will return immediately. + /// A representing the running operation. + async Task PostValidationShutdown(Task proceedTask) + { + Logger.LogTrace("Entered post validation terminate task."); + if (!await proceedTask) + { + Logger.LogTrace("Not running post validation terminate task for repeated bridge request."); + return; + } + + Logger.LogDebug("Server will terminated in 10s if it does not exit..."); + var delayTask = asyncDelayer.Delay(TimeSpan.FromSeconds(10), CancellationToken.None); // DCT: None available + var completedTask = await Task.WhenAny(process.Lifetime, delayTask); + if (completedTask == delayTask) + { + Logger.LogWarning("DMAPI took too long to shutdown server after validation request!"); + process.Terminate(); + apiValidationStatus = ApiValidationStatus.BadValidationRequest; + } + else + Logger.LogTrace("Server exited properly post validation."); + } + /// /// Handle a set of bridge . /// /// The to handle. /// The for the operation. /// A resulting in the for the request or if the request could not be dispatched. +#pragma warning disable CA1502 // TODO: Decomplexify async Task ProcessBridgeCommand(BridgeParameters parameters, CancellationToken cancellationToken) { var response = new BridgeResponse(); @@ -734,7 +770,14 @@ async Task ProcessBridgeCommand(BridgeParameters parameters, Can break; case BridgeCommandType.Startup: + var proceedTcs = new TaskCompletionSource(); + var firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, PostValidationShutdown(proceedTcs.Task), null) == null; + proceedTcs.SetResult(firstValidationRequest); apiValidationStatus = ApiValidationStatus.BadValidationRequest; + + if (!firstValidationRequest) + return BridgeError("Startup bridge request was repeated!"); + if (parameters.Version == null) return BridgeError("Missing dmApiVersion field!"); @@ -808,6 +851,7 @@ async Task ProcessBridgeCommand(BridgeParameters parameters, Can return response; } +#pragma warning restore CA1502 /// /// Log and return a for a given . From f124c2aaec13351521b14f2288d8e0c40ef855e8 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 12:01:04 -0400 Subject: [PATCH 018/138] Add a decomplexify TODO --- src/Tgstation.Server.Api/ApiHeaders.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Api/ApiHeaders.cs b/src/Tgstation.Server.Api/ApiHeaders.cs index 2d27908acdb..7fe6edafc33 100644 --- a/src/Tgstation.Server.Api/ApiHeaders.cs +++ b/src/Tgstation.Server.Api/ApiHeaders.cs @@ -164,7 +164,7 @@ public ApiHeaders(ProductHeaderValue userAgent, string username, string password /// The containing the serialized . /// If a missing should be ignored. /// Thrown if the constitue invalid . -#pragma warning disable CA1502 +#pragma warning disable CA1502 // TODO: Decomplexify public ApiHeaders(RequestHeaders requestHeaders, bool ignoreMissingAuth = false) { if (requestHeaders == null) From 571c555360aa3b9fb06d9d0b1592fbd976121408 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 18 Oct 2023 00:19:16 -0400 Subject: [PATCH 019/138] Finally find a way to test if Linux is being shitty to us about links --- tests/DMAPI/LongRunning/Test.dm | 9 ++++ tests/DMAPI/LongRunning/long_running_test.dme | 3 ++ .../LongRunning/long_running_test_copy.dme | 1 + tests/DMAPI/LongRunning/resource.txt | 8 ++++ .../Live/Instance/WatchdogTest.cs | 48 +++++++++++++++++++ tgstation-server.sln | 1 + 6 files changed, 70 insertions(+) create mode 100644 tests/DMAPI/LongRunning/resource.txt diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index a77c3377b5d..657fadbfd07 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -22,6 +22,15 @@ if(!length(channels)) FailTest("Expected some chat channels!") + var/test_str = "aljsdhfjahsfkjnsalkjdfhskljdackmcnvxkljhvkjsdanv,jdshlkufhklasjeFDhfjkalhdkjlfhalksfdjh" + var/res_contents = file2text('resource.txt') // we need a .rsc to be generated + + if(!findtext(res_contents, test_str)) + FailTest("Failed to resource? Did not find magic: [res_contents]") + + if(!fexists("[DME_NAME].rsc")) + FailTest("Failed to create .rsc!") + StartAsync() /proc/dab() diff --git a/tests/DMAPI/LongRunning/long_running_test.dme b/tests/DMAPI/LongRunning/long_running_test.dme index b9270144556..322587647dc 100644 --- a/tests/DMAPI/LongRunning/long_running_test.dme +++ b/tests/DMAPI/LongRunning/long_running_test.dme @@ -13,5 +13,8 @@ // BEGIN_INCLUDE #include "Config.dm" #include "../test_prelude.dm" +#ifndef DME_NAME +#define DME_NAME "long_running_test" +#endif #include "Test.dm" // END_INCLUDE diff --git a/tests/DMAPI/LongRunning/long_running_test_copy.dme b/tests/DMAPI/LongRunning/long_running_test_copy.dme index b4da8e0d964..e443c4b3312 100644 --- a/tests/DMAPI/LongRunning/long_running_test_copy.dme +++ b/tests/DMAPI/LongRunning/long_running_test_copy.dme @@ -11,5 +11,6 @@ // END_PREFERENCES // BEGIN_INCLUDE +#define DME_NAME "long_running_test_copy" #include "long_running_test.dme" // END_INCLUDE diff --git a/tests/DMAPI/LongRunning/resource.txt b/tests/DMAPI/LongRunning/resource.txt new file mode 100644 index 00000000000..94b848fe2ac --- /dev/null +++ b/tests/DMAPI/LongRunning/resource.txt @@ -0,0 +1,8 @@ + + if(!length(channels)) + FailTest("Expected some chat channels!") + + var/test_str = "aljsdhfjahsfkjnsalkjdfhskljdackmcnvxkljhvkjsdanv,jdshlkufhklasjeFDhfjkalhdkjlfhalksfdjh" + var/self_contents = file2text(file('Test.dm')) // we need a .rsc to be generated + + if(!(test_str in self_contents)) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 329fab6aa19..30b2ef36563 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -3,17 +3,22 @@ using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Mono.Unix; +using Mono.Unix.Native; + using Moq; using Newtonsoft.Json; using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Net; using System.Net.Sockets; using System.Runtime.InteropServices; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -471,6 +476,47 @@ async Task RunBasicTest(CancellationToken cancellationToken) Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters); } + void TestLinuxIsntBeingFuckingCheekyAboutFilePaths(DreamDaemonResponse currentStatus, CompileJobResponse previousStatus, CancellationToken cancellationToken) + { + if (new PlatformIdentifier().IsWindows) + return; + + Assert.IsNotNull(currentStatus.ActiveCompileJob); + Assert.IsTrue(currentStatus.ActiveCompileJob.DmeName.Contains("long_running_test")); + Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); + + var procs = TestLiveServer.GetDDProcessesOnPort(currentStatus.Port.Value); + Assert.AreEqual(1, procs.Count); + var failingLinks = new List(); + using var proc = procs[0]; + var pid = proc.Id; + var foundLivePath = false; + var allPaths = new List(); + foreach (var fd in Directory.EnumerateFiles($"/proc/{pid}/fd")) + { + var sb = new StringBuilder(); + if (Syscall.readlink(fd, sb) == -1) + throw new UnixIOException(Stdlib.GetLastError()); + + var path = sb.ToString(); + + allPaths.Add(path); + if (path.Contains($"Game/{previousStatus.DirectoryName}")) + failingLinks.Add($"Found fd {fd} resolving to previous absolute path game dir path: {path}"); + + if (path.Contains($"Game/{currentStatus.ActiveCompileJob.DirectoryName}")) + failingLinks.Add($"Found fd {fd} resolving to current absolute path game dir path: {path}"); + + if (path.Contains($"Game/Live")) + foundLivePath = true; + } + + if (!foundLivePath) + failingLinks.Add($"Failed to find a path containing the 'Live' directory! Found {allPaths.Count}: \"{String.Join("\", \"", allPaths)}\""); + + Assert.IsTrue(failingLinks.Count == 0, String.Join(Environment.NewLine, failingLinks)); + } + async Task RunHealthCheckTest(bool checkDump, CancellationToken cancellationToken) { System.Console.WriteLine("TEST: WATCHDOG HEALTH CHECK TEST"); @@ -901,6 +947,8 @@ async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken) Assert.AreNotEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); Assert.IsNull(daemonStatus.StagedCompileJob); + TestLinuxIsntBeingFuckingCheekyAboutFilePaths(daemonStatus, initialCompileJob, cancellationToken); + await instanceClient.DreamDaemon.Shutdown(cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); diff --git a/tgstation-server.sln b/tgstation-server.sln index 20eef245e10..c1cde46821b 100644 --- a/tgstation-server.sln +++ b/tgstation-server.sln @@ -158,6 +158,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LongRunning", "LongRunning" tests\DMAPI\LongRunning\long_running_test.dme = tests\DMAPI\LongRunning\long_running_test.dme tests\DMAPI\LongRunning\long_running_test_copy.dme = tests\DMAPI\LongRunning\long_running_test_copy.dme tests\DMAPI\LongRunning\long_running_test_rooted.dme = tests\DMAPI\LongRunning\long_running_test_rooted.dme + tests\DMAPI\LongRunning\resource.txt = tests\DMAPI\LongRunning\resource.txt tests\DMAPI\LongRunning\Test.dm = tests\DMAPI\LongRunning\Test.dm EndProjectSection EndProject From 44729820eddd656f55c2090eb31c83e203eaa324 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 12:27:53 -0400 Subject: [PATCH 020/138] Remove unused variable and fix test buffer size --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 4 ++-- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 30b2ef36563..96ef4d3c201 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -494,7 +494,7 @@ void TestLinuxIsntBeingFuckingCheekyAboutFilePaths(DreamDaemonResponse currentSt var allPaths = new List(); foreach (var fd in Directory.EnumerateFiles($"/proc/{pid}/fd")) { - var sb = new StringBuilder(); + var sb = new StringBuilder(UInt16.MaxValue); if (Syscall.readlink(fd, sb) == -1) throw new UnixIOException(Stdlib.GetLastError()); @@ -512,7 +512,7 @@ void TestLinuxIsntBeingFuckingCheekyAboutFilePaths(DreamDaemonResponse currentSt } if (!foundLivePath) - failingLinks.Add($"Failed to find a path containing the 'Live' directory! Found {allPaths.Count}: \"{String.Join("\", \"", allPaths)}\""); + failingLinks.Add($"Failed to find a path containing the 'Live' directory!"); Assert.IsTrue(failingLinks.Count == 0, String.Join(Environment.NewLine, failingLinks)); } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index f058d9795fe..a78b984fc2e 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1326,8 +1326,6 @@ async Task FailFast(Task task) async Task RunInstanceTests() { - // Some earlier linux BYOND versions have a critical bug where replacing the directory in non-basic watchdogs causes the DreamDaemon cwd to change - var canRunCompatTests = new PlatformIdentifier().IsWindows; var compatTests = FailFast( instanceTest .RunCompatTests( From a0d583b5961d353d24024e7e200283cdae799a8a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:01:41 -0400 Subject: [PATCH 021/138] New hard link system for PosixWatchdog - IDmbProvider is now `AsyncDisposable` as opposed to `IDisposable`. - Add hard link support to `ISymlinkFactory`. - `PosixWatchdog` now has to mirror deployment structure as hard links before swapping. Because of this deployments may not immediately be applied if the reboots immediately after they were completed. --- .../Components/Deployment/DmbFactory.cs | 7 +- .../Components/Deployment/DmbProvider.cs | 9 +- .../Components/Deployment/DreamMaker.cs | 2 +- .../Deployment/HardLinkDmbProvider.cs | 222 ++++++++++++++++++ .../Components/Deployment/IDmbProvider.cs | 2 +- .../Deployment/SwappableDmbProvider.cs | 58 +++-- .../Deployment/TemporaryDmbProvider.cs | 5 +- .../Components/Session/ISessionController.cs | 4 +- .../Components/Session/SessionController.cs | 11 +- .../Components/Watchdog/BasicWatchdog.cs | 2 +- .../Components/Watchdog/PosixWatchdog.cs | 21 +- .../Watchdog/PosixWatchdogFactory.cs | 1 + .../Components/Watchdog/WindowsWatchdog.cs | 67 ++++-- .../IO/ISymlinkFactory.cs | 9 + .../IO/PosixSymlinkFactory.cs | 16 ++ .../IO/WindowsSymlinkFactory.cs | 4 + 16 files changed, 378 insertions(+), 62 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 09567ba79cd..df587cab111 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -148,15 +148,18 @@ await remoteDeploymentManager.StageDeployment( cancellationToken); } + ValueTask dmbDisposeTask; lock (jobLockCounts) { - nextDmbProvider?.Dispose(); + dmbDisposeTask = nextDmbProvider?.DisposeAsync() ?? ValueTask.CompletedTask; nextDmbProvider = newProvider; // Oh god dammit var temp = Interlocked.Exchange(ref newerDmbTcs, new TaskCompletionSource()); temp.SetResult(); } + + await dmbDisposeTask; } /// @@ -320,7 +323,7 @@ void CleanupAction() finally { if (!providerSubmitted) - newProvider.Dispose(); + await newProvider.DisposeAsync(); } } #pragma warning restore CA1506 diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs index f3711ede4f8..770409f7902 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs @@ -1,4 +1,5 @@ using System; +using System.Threading.Tasks; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Models; @@ -30,7 +31,7 @@ sealed class DmbProvider : IDmbProvider readonly string directoryAppend; /// - /// The to run when is called. + /// The to run when is called. /// Action onDispose; @@ -50,7 +51,11 @@ public DmbProvider(CompileJob compileJob, IIOManager ioManager, Action onDispose } /// - public void Dispose() => onDispose?.Invoke(); + public ValueTask DisposeAsync() + { + onDispose?.Invoke(); + return ValueTask.CompletedTask; + } /// public void KeepAlive() => onDispose = null; diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index 665efba795e..307a1bb1070 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -802,7 +802,7 @@ async Task VerifyApi( job.MinimumSecurityLevel = securityLevel; // needed for the TempDmbProvider ApiValidationStatus validationStatus; - using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(job.DirectoryName.ToString()), String.Concat(job.DmeName, DmbExtension), job)) + await using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(job.DirectoryName.ToString()), String.Concat(job.DmeName, DmbExtension), job)) await using (var controller = await sessionControllerFactory.LaunchNew(provider, byondLock, launchParameters, true, cancellationToken)) { var launchResult = await controller.LaunchResult.WaitAsync(cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs new file mode 100644 index 00000000000..e5ddede026e --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -0,0 +1,222 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Utils; + +namespace Tgstation.Server.Host.Components.Deployment +{ + /// + /// A that uses hard links. + /// + sealed class HardLinkDmbProvider : SwappableDmbProvider + { + /// + /// The for . + /// + readonly CancellationTokenSource cancellationTokenSource; + + /// + /// The representing the base provider mirroring operation. + /// + readonly Task mirroringTask; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + /// The for the . + /// The for the . + /// The for the . + /// The value of . + /// The for the . + public HardLinkDmbProvider( + IDmbProvider baseProvider, + IIOManager ioManager, + ISymlinkFactory symlinkFactory, + ILogger logger, + GeneralConfiguration generalConfiguration) + : base( + baseProvider, + ioManager, + symlinkFactory) + { + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + cancellationTokenSource = new CancellationTokenSource(); + try + { + mirroringTask = MirrorSourceDirectory(generalConfiguration.GetCopyDirectoryTaskThrottle(), cancellationTokenSource.Token); + } + catch + { + cancellationTokenSource.Dispose(); + throw; + } + } + + /// + public override async ValueTask DisposeAsync() + { + cancellationTokenSource.Cancel(); + cancellationTokenSource.Dispose(); + try + { + await mirroringTask; + } + catch (OperationCanceledException ex) + { + logger.LogDebug(ex, "Mirroring task cancelled!"); + } + + await base.DisposeAsync(); + } + + /// + public override Task FinishActivationPreparation(CancellationToken cancellationToken) + { + if (!mirroringTask.IsCompleted) + logger.LogTrace("Waiting for mirroring to complete..."); + + return mirroringTask.WaitAsync(cancellationToken); + } + + /// + protected override async Task DoSwap(CancellationToken cancellationToken) + { + var mirroredDir = await mirroringTask.WaitAsync(cancellationToken); + var goAheadTcs = new TaskCompletionSource(); + + // I feel dirty... + async void DisposeOfOldDirectory() + { + var directoryMoved = false; + var disposePath = Guid.NewGuid().ToString(); + try + { + await IOManager.MoveDirectory(LiveGameDirectory, disposePath, cancellationToken); + directoryMoved = true; + goAheadTcs.SetResult(); + await IOManager.DeleteDirectory(disposePath, CancellationToken.None); // DCT: We're detached at this point + } + catch (Exception ex) + { + if (directoryMoved) + logger.LogWarning(ex, "Failed to delete hard linked directory: {disposePath}", disposePath); + else + { + logger.LogDebug(ex, "Live directory appears to not exist"); + goAheadTcs.SetResult(); + } + } + } + + DisposeOfOldDirectory(); + await goAheadTcs.Task; + await IOManager.MoveDirectory(mirroredDir, LiveGameDirectory, cancellationToken); + } + + /// + /// Mirror the . + /// + /// The optional maximum number of simultaneous tasks allowed to execute. + /// The for the operation. + /// A resulting in the full path to the mirrored directory. + async Task MirrorSourceDirectory(int? taskThrottle, CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + var mirrorGuid = Guid.NewGuid(); + logger.LogDebug("Starting to mirror {sourceDir} as hard links to {mirrorGuid}...", CompileJob.DirectoryName, mirrorGuid); + if (taskThrottle.HasValue && taskThrottle < 1) + throw new ArgumentOutOfRangeException(nameof(taskThrottle), taskThrottle, "taskThrottle must be at least 1!"); + + var src = IOManager.ResolvePath(CompileJob.DirectoryName.ToString()); + var dest = IOManager.ResolvePath(mirrorGuid.ToString()); + + using var semaphore = taskThrottle.HasValue ? new SemaphoreSlim(taskThrottle.Value) : null; + await Task.WhenAll(MirrorDirectoryImpl(src, dest, semaphore, cancellationToken)); + stopwatch.Stop(); + + logger.LogDebug( + "Finished mirror of {sourceDir} to {mirrorGuid} in {seconds}s...", + CompileJob.DirectoryName, + mirrorGuid, + stopwatch.Elapsed.TotalSeconds.ToString("0.##", CultureInfo.InvariantCulture)); + + return dest; + } + + /// + /// Recursively create tasks to create a hard link directory mirror of to . + /// + /// The source directory path. + /// The destination directory path. + /// Optional used to limit degree of parallelism. + /// The for the operation. + /// A of s representing the running operations. The first returned is always the necessary call to . + /// I genuinely don't know how this will work with symlinked files. Waiting for the issue report I guess. + IEnumerable MirrorDirectoryImpl(string src, string dest, SemaphoreSlim semaphore, CancellationToken cancellationToken) + { + var dir = new DirectoryInfo(src); + Task subdirCreationTask = null; + foreach (var subDirectory in dir.EnumerateDirectories()) + { + // check if we are a symbolic link + if (!subDirectory.Attributes.HasFlag(FileAttributes.Directory) || subDirectory.Attributes.HasFlag(FileAttributes.ReparsePoint)) + { + logger.LogTrace("Skipping symlink to {subdir}", subDirectory.Name); + continue; + } + + var checkingSubdirCreationTask = true; + foreach (var copyTask in MirrorDirectoryImpl(subDirectory.FullName, Path.Combine(dest, subDirectory.Name), semaphore, cancellationToken)) + { + if (subdirCreationTask == null) + { + subdirCreationTask = copyTask; + yield return subdirCreationTask; + } + else if (!checkingSubdirCreationTask) + yield return copyTask; + + checkingSubdirCreationTask = false; + } + } + + foreach (var fileInfo in dir.EnumerateFiles()) + { + if (subdirCreationTask == null) + { + subdirCreationTask = IOManager.CreateDirectory(dest, cancellationToken); + yield return subdirCreationTask; + } + + var sourceFile = fileInfo.FullName; + var destFile = IOManager.ConcatPath(dest, fileInfo.Name); + + async Task LinkThisFile() + { + await subdirCreationTask.WaitAsync(cancellationToken); + using var lockContext = semaphore != null + ? await SemaphoreSlimContext.Lock(semaphore, cancellationToken) + : null; + await SymlinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken); + } + + yield return LinkThisFile(); + } + } + } +} diff --git a/src/Tgstation.Server.Host/Components/Deployment/IDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/IDmbProvider.cs index efc773a9193..c1c477ffff7 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/IDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/IDmbProvider.cs @@ -7,7 +7,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Provides absolute paths to the latest compiled .dmbs. /// - public interface IDmbProvider : IDisposable + public interface IDmbProvider : IAsyncDisposable { /// /// The file name of the .dmb. diff --git a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs index e4964532229..14cdc268e29 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs @@ -10,7 +10,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// A that uses symlinks. /// - sealed class SwappableDmbProvider : IDmbProvider + class SwappableDmbProvider : IDmbProvider { /// /// The directory where the is symlinked to. @@ -21,7 +21,7 @@ sealed class SwappableDmbProvider : IDmbProvider public string DmbName => baseProvider.DmbName; /// - public string Directory => ioManager.ResolvePath(LiveGameDirectory); + public string Directory => IOManager.ResolvePath(LiveGameDirectory); /// public CompileJob CompileJob => baseProvider.CompileJob; @@ -32,19 +32,19 @@ sealed class SwappableDmbProvider : IDmbProvider public bool Swapped => swapped != 0; /// - /// The we are swapping for. + /// The to use. /// - readonly IDmbProvider baseProvider; + protected IIOManager IOManager { get; } /// - /// The to use. + /// The to use. /// - readonly IIOManager ioManager; + protected ISymlinkFactory SymlinkFactory { get; } /// - /// The to use. + /// The we are swapping for. /// - readonly ISymlinkFactory symlinkFactory; + readonly IDmbProvider baseProvider; /// /// Backing field for . @@ -55,17 +55,17 @@ sealed class SwappableDmbProvider : IDmbProvider /// Initializes a new instance of the class. /// /// The value of . - /// The value of . - /// The value of . + /// The value of . + /// The value of . public SwappableDmbProvider(IDmbProvider baseProvider, IIOManager ioManager, ISymlinkFactory symlinkFactory) { this.baseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider)); - this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); - this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + IOManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); } /// - public void Dispose() => baseProvider.Dispose(); + public virtual ValueTask DisposeAsync() => baseProvider.DisposeAsync(); /// public void KeepAlive() => baseProvider.KeepAlive(); @@ -75,19 +75,37 @@ public SwappableDmbProvider(IDmbProvider baseProvider, IIOManager ioManager, ISy /// /// The for the operation. /// A representing the running operation. - public async Task MakeActive(CancellationToken cancellationToken) + public Task MakeActive(CancellationToken cancellationToken) { if (Interlocked.Exchange(ref swapped, 1) != 0) throw new InvalidOperationException("Already swapped!"); - if (symlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) - await ioManager.DeleteFile(LiveGameDirectory, cancellationToken); + return DoSwap(cancellationToken); + } + + /// + /// Should be . before calling to ensure the is ready to instantly swap. Can be called multiple times. + /// + /// The for the operation. + /// A representing the preparation process. + public virtual Task FinishActivationPreparation(CancellationToken cancellationToken) + => Task.CompletedTask; + + /// + /// Perform the swapping action. + /// + /// The for the operation. + /// A representing the running operation. + protected virtual async Task DoSwap(CancellationToken cancellationToken) + { + if (SymlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) + await IOManager.DeleteFile(LiveGameDirectory, cancellationToken); else - await ioManager.DeleteDirectory(LiveGameDirectory, cancellationToken); + await IOManager.DeleteDirectory(LiveGameDirectory, cancellationToken); - await symlinkFactory.CreateSymbolicLink( - ioManager.ResolvePath(baseProvider.Directory), - ioManager.ResolvePath(LiveGameDirectory), + await SymlinkFactory.CreateSymbolicLink( + IOManager.ResolvePath(baseProvider.Directory), + IOManager.ResolvePath(LiveGameDirectory), cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/TemporaryDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/TemporaryDmbProvider.cs index 0bf982cfa38..4476d9d2c3d 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/TemporaryDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/TemporaryDmbProvider.cs @@ -1,4 +1,5 @@ using System; +using System.Threading.Tasks; using Tgstation.Server.Host.Models; @@ -32,9 +33,7 @@ public TemporaryDmbProvider(string directory, string dmb, CompileJob compileJob) } /// - public void Dispose() - { - } + public ValueTask DisposeAsync() => ValueTask.CompletedTask; /// public void KeepAlive() => throw new NotSupportedException(); diff --git a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs index 3ded84429ac..93f10eb17cc 100644 --- a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs @@ -128,7 +128,7 @@ interface ISessionController : IProcessBase, IRenameNotifyee, IAsyncDisposable /// Replace the in use with a given , disposing the old one. /// /// The new . - /// An to be disposed once certain that the original is no longer in use. - IDisposable ReplaceDmbProvider(IDmbProvider newProvider); + /// An to be disposed once certain that the original is no longer in use. + IAsyncDisposable ReplaceDmbProvider(IDmbProvider newProvider); } } diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 2da8c03b58f..eab32bdce18 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -350,8 +350,13 @@ public async ValueTask DisposeAsync() await process.DisposeAsync(); byondLock.Dispose(); bridgeRegistration?.Dispose(); - ReattachInformation.Dmb.Dispose(); - ReattachInformation.InitialDmb?.Dispose(); + var regularDmbDisposeTask = ReattachInformation.Dmb.DisposeAsync(); + var initialDmb = ReattachInformation.InitialDmb; + if (initialDmb != null) + await initialDmb.DisposeAsync(); + + await regularDmbDisposeTask; + chatTrackingContext.Dispose(); reattachTopicCts.Dispose(); @@ -552,7 +557,7 @@ public void ResetRebootState() public void Resume() => process.Resume(); /// - public IDisposable ReplaceDmbProvider(IDmbProvider dmbProvider) + public IAsyncDisposable ReplaceDmbProvider(IDmbProvider dmbProvider) { var oldDmb = ReattachInformation.Dmb; ReattachInformation.Dmb = dmbProvider ?? throw new ArgumentNullException(nameof(dmbProvider)); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index 8396ce7c992..fc5c3461b0c 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -268,7 +268,7 @@ await ReattachFailure( // server didn't get control of this dmb if (dmbToUse != null && !serverWasActive) - dmbToUse.Dispose(); + await dmbToUse.DisposeAsync(); throw; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index 044ef3f1ef8..3bf2927e578 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -1,4 +1,5 @@ -using System.Threading; +using System; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -9,6 +10,7 @@ using Tgstation.Server.Host.Components.Deployment.Remote; using Tgstation.Server.Host.Components.Events; using Tgstation.Server.Host.Components.Session; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; @@ -21,6 +23,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// sealed class PosixWatchdog : WindowsWatchdog { + /// + /// The for the . + /// + readonly GeneralConfiguration generalConfiguration; + /// /// Initializes a new instance of the class. /// @@ -39,6 +46,7 @@ sealed class PosixWatchdog : WindowsWatchdog /// The for the . /// The for the . /// The for the . + /// The value of . /// The autostart value for the . public PosixWatchdog( IChatManager chat, @@ -56,6 +64,7 @@ public PosixWatchdog( ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, + GeneralConfiguration generalConfiguration, bool autoStart) : base( chat, @@ -75,13 +84,15 @@ public PosixWatchdog( instance, autoStart) { + this.generalConfiguration = generalConfiguration ?? throw new ArgumentNullException(nameof(generalConfiguration)); } /// protected override Task ApplyInitialDmb(CancellationToken cancellationToken) - { - // not necessary to hold initial .dmb on Linux because of based inode deletes - return Task.CompletedTask; - } + => Task.CompletedTask; // not necessary to hold initial .dmb on Linux because of based inode deletes + + /// + protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) + => new HardLinkDmbProvider(dmbProvider, GameIOManager, SymlinkFactory, Logger, generalConfiguration); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs index 6774c73cb55..787127a05c9 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs @@ -76,6 +76,7 @@ public override IWatchdog CreateWatchdog( LoggerFactory.CreateLogger(), settings, instance, + GeneralConfiguration, settings.AutoStart ?? throw new ArgumentNullException(nameof(settings))); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index c8d6c7eb442..0e42c3b660e 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -37,7 +37,7 @@ class WindowsWatchdog : BasicWatchdog /// /// The for the . /// - readonly ISymlinkFactory symlinkFactory; + protected ISymlinkFactory SymlinkFactory { get; } /// /// of s that are waiting to clean up old deployments. @@ -68,7 +68,7 @@ class WindowsWatchdog : BasicWatchdog /// The for the . /// The for the . /// The value of . - /// The value of . + /// The value of . /// The for the . /// The for the . /// The for the . @@ -109,7 +109,7 @@ public WindowsWatchdog( try { GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager)); - this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); deploymentCleanupTasks = new List(); } @@ -131,7 +131,7 @@ protected override async Task DisposeAndNullControllersImpl() // If we reach this point, we can guarantee PrepServerForLaunch will be called before starting again. ActiveSwappable = null; - pendingSwappable?.Dispose(); + await (pendingSwappable?.DisposeAsync() ?? ValueTask.CompletedTask); pendingSwappable = null; await DrainDeploymentCleanupTasks(true); @@ -142,9 +142,10 @@ protected override async Task HandleNormalReboot(CancellationToke { if (pendingSwappable != null) { - var updateTask = BeforeApplyDmb(pendingSwappable.CompileJob, cancellationToken); + Task RunPrequel() => BeforeApplyDmb(pendingSwappable.CompileJob, cancellationToken); - if (!pendingSwappable.Swapped) + var needToSwap = !pendingSwappable.Swapped; + if (needToSwap) { // IMPORTANT: THE SESSIONCONTROLLER SHOULD STILL BE PROCESSING THE BRIDGE REQUEST SO WE KNOW DD IS SLEEPING // OTHERWISE, IT COULD RETURN TO /world/Reboot() TOO EARLY AND LOAD THE WRONG .DMB @@ -153,18 +154,29 @@ protected override async Task HandleNormalReboot(CancellationToke // integration test logging will catch this Logger.LogError( "The reboot bridge request completed before the watchdog could suspend the server! This can lead to buggy DreamDaemon behaviour and should be reported! To ensure stability, we will need to hard reboot the server"); - await updateTask; + await RunPrequel(); return MonitorAction.Restart; } - await PerformDmbSwap(pendingSwappable, cancellationToken); + // DCT: Not necessary + if (!pendingSwappable.FinishActivationPreparation(CancellationToken.None).IsCompleted) + { + // rare pokemon + Logger.LogInformation("Deployed .dme is not ready to swap, delaying until next reboot!"); + Chat.QueueWatchdogMessage("The pending deployment was not ready to be activated this reboot. It will be applied at the next one."); + return MonitorAction.Continue; + } } + var updateTask = RunPrequel(); + if (needToSwap) + await PerformDmbSwap(pendingSwappable, cancellationToken); + var currentCompileJobId = Server.ReattachInformation.Dmb.CompileJob.Id; await DrainDeploymentCleanupTasks(false); - IDisposable lingeringDeployment; + IAsyncDisposable lingeringDeployment; var localDeploymentCleanupGate = new TaskCompletionSource(); async Task CleanupLingeringDeployment() { @@ -191,7 +203,7 @@ async Task CleanupLingeringDeployment() ? " due to timeout!" : "..."); - lingeringDeployment.Dispose(); + await lingeringDeployment.DisposeAsync(); } var oldDeploymentCleanupGate = Interlocked.Exchange(ref deploymentCleanupGate, localDeploymentCleanupGate); @@ -247,31 +259,31 @@ protected override async Task HandleNewDmbAvailable(CancellationToken cancellati if (!canSeamlesslySwap) { - compileJobProvider.Dispose(); + await compileJobProvider.DisposeAsync(); await base.HandleNewDmbAvailable(cancellationToken); return; } - SwappableDmbProvider windowsProvider = null; + SwappableDmbProvider swappableProvider = null; try { - windowsProvider = new SwappableDmbProvider(compileJobProvider, GameIOManager, symlinkFactory); + swappableProvider = CreateSwappableDmbProvider(compileJobProvider); if (ActiveCompileJob.DMApiVersion == null) { Logger.LogWarning("Active compile job has no DMAPI! Commencing immediate .dmb swap. Note this behavior is known to be buggy in some DM code contexts. See https://github.com/tgstation/tgstation-server/issues/1550"); - await PerformDmbSwap(windowsProvider, cancellationToken); + await PerformDmbSwap(swappableProvider, cancellationToken); } } catch (Exception ex) { Logger.LogError(ex, "Exception while swapping"); - IDmbProvider providerToDispose = windowsProvider ?? compileJobProvider; - providerToDispose.Dispose(); + IDmbProvider providerToDispose = swappableProvider ?? compileJobProvider; + await providerToDispose.DisposeAsync(); throw; } - pendingSwappable?.Dispose(); - pendingSwappable = windowsProvider; + await (pendingSwappable?.DisposeAsync() ?? ValueTask.CompletedTask); + pendingSwappable = swappableProvider; } /// @@ -284,7 +296,7 @@ protected sealed override async Task PrepServerForLaunch(IDmbProvi Logger.LogTrace("Prep for server launch"); - ActiveSwappable = new SwappableDmbProvider(dmbToUse, GameIOManager, symlinkFactory); + ActiveSwappable = CreateSwappableDmbProvider(dmbToUse); try { await InitialLink(cancellationToken); @@ -310,6 +322,14 @@ protected virtual async Task ApplyInitialDmb(CancellationToken cancellationToken Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken); } + /// + /// Create a for a given . + /// + /// The to create a for. + /// A new . + protected virtual SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) + => new SwappableDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); + /// protected override async Task SessionStartupPersist(CancellationToken cancellationToken) { @@ -332,10 +352,11 @@ protected override async Task HandleMonitorWakeup(MonitorActivati /// /// The for the operation. /// A representing the running operation. - Task InitialLink(CancellationToken cancellationToken) + async ValueTask InitialLink(CancellationToken cancellationToken) { - Logger.LogTrace("Symlinking compile job..."); - return ActiveSwappable.MakeActive(cancellationToken); + await ActiveSwappable.FinishActivationPreparation(cancellationToken); + Logger.LogTrace("Linking compile job..."); + await ActiveSwappable.MakeActive(cancellationToken); } /// @@ -348,6 +369,8 @@ async ValueTask PerformDmbSwap(SwappableDmbProvider newProvider, CancellationTok { Logger.LogDebug("Swapping to compile job {id}...", newProvider.CompileJob.Id); + await newProvider.FinishActivationPreparation(cancellationToken); + var suspended = false; var server = Server; try diff --git a/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs b/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs index 1adbd99d975..08320a3d150 100644 --- a/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs @@ -22,5 +22,14 @@ interface ISymlinkFactory /// The for the operation. /// A representing the running operation. Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken); + + /// + /// Creates a hard link. + /// + /// The path to the hard target. + /// The path to the link. + /// The for the operation. + /// A representing the running operation. + Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs b/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs index ddbcbb86fb7..05d64a3bf07 100644 --- a/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs @@ -15,6 +15,22 @@ sealed class PosixSymlinkFactory : ISymlinkFactory /// public bool SymlinkedDirectoriesAreDeletedAsFiles => true; + /// + public Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew( + () => + { + ArgumentNullException.ThrowIfNull(targetPath); + ArgumentNullException.ThrowIfNull(linkPath); + + cancellationToken.ThrowIfCancellationRequested(); + var fsInfo = new UnixFileInfo(targetPath); + cancellationToken.ThrowIfCancellationRequested(); + fsInfo.CreateLink(linkPath); + }, + cancellationToken, + DefaultIOManager.BlockingTaskCreationOptions, + TaskScheduler.Current); + /// public Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew( () => diff --git a/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs b/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs index 18abde6b8f9..864242d992d 100644 --- a/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs @@ -16,6 +16,10 @@ sealed class WindowsSymlinkFactory : ISymlinkFactory /// public bool SymlinkedDirectoriesAreDeletedAsFiles => false; + /// + public Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken) + => throw new NotSupportedException(); + /// public Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew( () => From f6a843e651942907221f2ed5a6cf54749f9d0825 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:04:14 -0400 Subject: [PATCH 022/138] Rename `WindowsWatchdog` to `AdvancedWatchdog` --- .../{WindowsWatchdog.cs => AdvancedWatchdog.cs} | 14 +++++++------- .../Components/Watchdog/PosixWatchdog.cs | 8 ++++---- .../Components/Watchdog/WindowsWatchdogFactory.cs | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) rename src/Tgstation.Server.Host/Components/Watchdog/{WindowsWatchdog.cs => AdvancedWatchdog.cs} (97%) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs similarity index 97% rename from src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs rename to src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 0e42c3b660e..3552149c3cc 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -20,9 +20,9 @@ namespace Tgstation.Server.Host.Components.Watchdog { /// - /// A that, instead of killing servers for updates, uses the wonders of symlinks to swap out changes without killing DreamDaemon. + /// A that, instead of killing servers for updates, uses the wonders of filesystem links to swap out changes without killing the server process. /// - class WindowsWatchdog : BasicWatchdog + class AdvancedWatchdog : BasicWatchdog { /// /// The for . @@ -30,12 +30,12 @@ class WindowsWatchdog : BasicWatchdog protected SwappableDmbProvider ActiveSwappable { get; private set; } /// - /// The for the pointing to the Game directory. + /// The for the pointing to the Game directory. /// protected IIOManager GameIOManager { get; } /// - /// The for the . + /// The for the . /// protected ISymlinkFactory SymlinkFactory { get; } @@ -55,7 +55,7 @@ class WindowsWatchdog : BasicWatchdog volatile TaskCompletionSource deploymentCleanupGate; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The for the . /// The for the . @@ -73,7 +73,7 @@ class WindowsWatchdog : BasicWatchdog /// The for the . /// The for the . /// The autostart value for the . - public WindowsWatchdog( + public AdvancedWatchdog( IChatManager chat, ISessionControllerFactory sessionControllerFactory, IDmbFactory dmbFactory, @@ -86,7 +86,7 @@ public WindowsWatchdog( IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IIOManager gameIOManager, ISymlinkFactory symlinkFactory, - ILogger logger, + ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, bool autoStart) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index 3bf2927e578..961fee9988a 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -19,9 +19,9 @@ namespace Tgstation.Server.Host.Components.Watchdog { /// - /// A variant of the that works on POSIX systems. + /// A variant of the that works on POSIX systems. /// - sealed class PosixWatchdog : WindowsWatchdog + sealed class PosixWatchdog : AdvancedWatchdog { /// /// The for the . @@ -41,8 +41,8 @@ sealed class PosixWatchdog : WindowsWatchdog /// The for the . /// The for the . /// The for the . - /// The pointing to the game directory for the .. - /// The for the . + /// The pointing to the game directory for the .. + /// The for the . /// The for the . /// The for the . /// The for the . diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index 7909be59aff..fd7bdfe9b29 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -18,7 +18,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { /// - /// for creating s. + /// for creating s. /// class WindowsWatchdogFactory : WatchdogFactory { @@ -78,7 +78,7 @@ public override IWatchdog CreateWatchdog( remoteDeploymentManagerFactory, gameIOManager, SymlinkFactory, - LoggerFactory.CreateLogger(), + LoggerFactory.CreateLogger(), settings, instance, settings.AutoStart ?? throw new ArgumentNullException(nameof(settings))); From 4b02e43c3841c9f1a869a3cc2a8ed74334e0be3c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:10:35 -0400 Subject: [PATCH 023/138] Abstract `AdvancedWatchdog` Move Windows specific functionality out to `WindowsWatchdog`. --- .../Components/Watchdog/AdvancedWatchdog.cs | 16 ++-- .../Components/Watchdog/WindowsWatchdog.cs | 90 +++++++++++++++++++ .../Watchdog/WindowsWatchdogFactory.cs | 2 +- 3 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs diff --git a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 3552149c3cc..07954e5a88f 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -22,7 +22,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// A that, instead of killing servers for updates, uses the wonders of filesystem links to swap out changes without killing the server process. /// - class AdvancedWatchdog : BasicWatchdog + abstract class AdvancedWatchdog : BasicWatchdog { /// /// The for . @@ -125,7 +125,7 @@ public AdvancedWatchdog( } /// - protected override async Task DisposeAndNullControllersImpl() + protected sealed override async Task DisposeAndNullControllersImpl() { await base.DisposeAndNullControllersImpl(); @@ -138,7 +138,7 @@ protected override async Task DisposeAndNullControllersImpl() } /// - protected override async Task HandleNormalReboot(CancellationToken cancellationToken) + protected sealed override async Task HandleNormalReboot(CancellationToken cancellationToken) { if (pendingSwappable != null) { @@ -231,7 +231,7 @@ async Task CleanupLingeringDeployment() } /// - protected override async Task HandleNewDmbAvailable(CancellationToken cancellationToken) + protected sealed override async Task HandleNewDmbAvailable(CancellationToken cancellationToken) { IDmbProvider compileJobProvider = DmbFactory.LockNextDmb(1); bool canSeamlesslySwap = true; @@ -317,18 +317,14 @@ protected sealed override async Task PrepServerForLaunch(IDmbProvi /// /// The for the operation. /// A representing the running operation. - protected virtual async Task ApplyInitialDmb(CancellationToken cancellationToken) - { - Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken); - } + protected abstract Task ApplyInitialDmb(CancellationToken cancellationToken); /// /// Create a for a given . /// /// The to create a for. /// A new . - protected virtual SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) - => new SwappableDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); + protected abstract SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider); /// protected override async Task SessionStartupPersist(CancellationToken cancellationToken) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs new file mode 100644 index 00000000000..917bafa1348 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -0,0 +1,90 @@ +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Host.Components.Chat; +using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.Components.Deployment.Remote; +using Tgstation.Server.Host.Components.Events; +using Tgstation.Server.Host.Components.Session; +using Tgstation.Server.Host.Core; +using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; +using Tgstation.Server.Host.Utils; + +namespace Tgstation.Server.Host.Components.Watchdog +{ + /// + /// A variant of the that works on Windows systems. + /// + sealed class WindowsWatchdog : AdvancedWatchdog + { + /// + /// Initializes a new instance of the class. + /// + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The pointing to the game directory for the .. + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The autostart value for the . + public WindowsWatchdog( + IChatManager chat, + ISessionControllerFactory sessionControllerFactory, + IDmbFactory dmbFactory, + ISessionPersistor sessionPersistor, + IJobManager jobManager, + IServerControl serverControl, + IAsyncDelayer asyncDelayer, + IIOManager diagnosticsIOManager, + IEventConsumer eventConsumer, + IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IIOManager gameIOManager, + ISymlinkFactory symlinkFactory, + ILogger logger, + DreamDaemonLaunchParameters initialLaunchParameters, + Api.Models.Instance instance, + bool autoStart) + : base( + chat, + sessionControllerFactory, + dmbFactory, + sessionPersistor, + jobManager, + serverControl, + asyncDelayer, + diagnosticsIOManager, + eventConsumer, + remoteDeploymentManagerFactory, + gameIOManager, + symlinkFactory, + logger, + initialLaunchParameters, + instance, + autoStart) + { + } + + /// + protected override async Task ApplyInitialDmb(CancellationToken cancellationToken) + { + Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken); + } + + /// + protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) + => new SwappableDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index fd7bdfe9b29..4cbe20577e6 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -78,7 +78,7 @@ public override IWatchdog CreateWatchdog( remoteDeploymentManagerFactory, gameIOManager, SymlinkFactory, - LoggerFactory.CreateLogger(), + LoggerFactory.CreateLogger(), settings, instance, settings.AutoStart ?? throw new ArgumentNullException(nameof(settings))); From afceee3a96a4aa979046f4344901f284f647ffcd Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:24:54 -0400 Subject: [PATCH 024/138] I was unsure about `DefaultIOManager.DeleteDirectory`, so here's another unit test. --- .../IO/TestIOManager.cs | 58 +++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs index f92ffaa04db..5e34e809dc1 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs @@ -1,11 +1,12 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; - -using Remora.Discord.API.Objects; - -using System; +using System; using System.IO; +using System.Linq; +using System.Text; +using System.Threading; using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.IO.Tests @@ -38,6 +39,53 @@ public async Task TestDeleteDirectory() } } + [TestMethod] + public async Task TestDeleteDirectoryWithSymlinkInsideDoesntRecurse() + { + var linkFactory = (ISymlinkFactory)(new PlatformIdentifier().IsWindows + ? new WindowsSymlinkFactory() + : new PosixSymlinkFactory()); + + var tempPath = Path.GetTempFileName(); + File.Delete(tempPath); + Directory.CreateDirectory(tempPath); + try + { + var targetDir = ioManager.ConcatPath(tempPath, "targetdir"); + await ioManager.CreateDirectory(targetDir, CancellationToken.None); + var fileInTargetDir = ioManager.ConcatPath(targetDir, "test1.txt"); + + var expectedBytes = Encoding.UTF8.GetBytes("I want to live"); + await ioManager.WriteAllBytes(fileInTargetDir, expectedBytes, CancellationToken.None); + + var testDir = ioManager.ConcatPath(tempPath, "testdir"); + await ioManager.CreateDirectory(testDir, CancellationToken.None); + var symlinkedFile = ioManager.ConcatPath(testDir, "test1.txt"); + var symlinkedDir = ioManager.ConcatPath(testDir, "linkedDir"); + + await linkFactory.CreateSymbolicLink(targetDir, symlinkedDir, CancellationToken.None); + await linkFactory.CreateSymbolicLink(fileInTargetDir, symlinkedFile, CancellationToken.None); + + Assert.IsTrue(await ioManager.DirectoryExists(symlinkedDir, CancellationToken.None)); + Assert.IsTrue(await ioManager.FileExists(symlinkedFile, CancellationToken.None)); + Assert.IsTrue(await ioManager.FileExists(ioManager.ConcatPath(symlinkedDir, "test1.txt"), CancellationToken.None)); + Assert.IsTrue(await ioManager.FileExists(fileInTargetDir, CancellationToken.None)); + + await ioManager.DeleteDirectory(testDir, CancellationToken.None); + + Assert.IsFalse(await ioManager.DirectoryExists(symlinkedDir, CancellationToken.None)); + Assert.IsFalse(await ioManager.FileExists(symlinkedFile, CancellationToken.None)); + Assert.IsFalse(await ioManager.FileExists(ioManager.ConcatPath(symlinkedDir, "test1.txt"), CancellationToken.None)); + Assert.IsTrue(await ioManager.FileExists(fileInTargetDir, CancellationToken.None)); + Assert.IsTrue(expectedBytes.SequenceEqual(await ioManager.ReadAllBytes(fileInTargetDir, CancellationToken.None))); + } + catch + { + Directory.Delete(tempPath, true); + throw; + } + } + [TestMethod] public async Task TestFileExists() { From 8129773d69d7e8d4c27c3783bf8cf5f962da30e1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:37:01 -0400 Subject: [PATCH 025/138] Extract symlinking behavior to `SymlinkDmbProvider` --- .../Deployment/HardLinkDmbProvider.cs | 16 ++++--- .../Deployment/SwappableDmbProvider.cs | 44 +++++++------------ .../Deployment/SymlinkDmbProvider.cs | 44 +++++++++++++++++++ .../Components/Watchdog/WindowsWatchdog.cs | 2 +- 4 files changed, 70 insertions(+), 36 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs index e5ddede026e..634af753f42 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -111,15 +111,17 @@ async void DisposeOfOldDirectory() goAheadTcs.SetResult(); await IOManager.DeleteDirectory(disposePath, CancellationToken.None); // DCT: We're detached at this point } - catch (Exception ex) + catch (DirectoryNotFoundException ex) { - if (directoryMoved) - logger.LogWarning(ex, "Failed to delete hard linked directory: {disposePath}", disposePath); - else - { - logger.LogDebug(ex, "Live directory appears to not exist"); + logger.LogDebug(ex, "Live directory appears to not exist"); + if (!directoryMoved) goAheadTcs.SetResult(); - } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to delete hard linked directory: {disposePath}", disposePath); + if (!directoryMoved) + goAheadTcs.SetException(ex); } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs index 14cdc268e29..b82f5cc6761 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs @@ -8,29 +8,34 @@ namespace Tgstation.Server.Host.Components.Deployment { /// - /// A that uses symlinks. + /// A that uses filesystem links to change directory structure underneath the server process. /// - class SwappableDmbProvider : IDmbProvider + abstract class SwappableDmbProvider : IDmbProvider { /// - /// The directory where the is symlinked to. + /// The directory where the is symlinked to. /// public const string LiveGameDirectory = "Live"; /// - public string DmbName => baseProvider.DmbName; + public string DmbName => BaseProvider.DmbName; /// public string Directory => IOManager.ResolvePath(LiveGameDirectory); /// - public CompileJob CompileJob => baseProvider.CompileJob; + public CompileJob CompileJob => BaseProvider.CompileJob; /// /// If has been run. /// public bool Swapped => swapped != 0; + /// + /// The we are swapping for. + /// + protected IDmbProvider BaseProvider { get; } + /// /// The to use. /// @@ -41,11 +46,6 @@ class SwappableDmbProvider : IDmbProvider /// protected ISymlinkFactory SymlinkFactory { get; } - /// - /// The we are swapping for. - /// - readonly IDmbProvider baseProvider; - /// /// Backing field for . /// @@ -54,21 +54,21 @@ class SwappableDmbProvider : IDmbProvider /// /// Initializes a new instance of the class. /// - /// The value of . + /// The value of . /// The value of . /// The value of . public SwappableDmbProvider(IDmbProvider baseProvider, IIOManager ioManager, ISymlinkFactory symlinkFactory) { - this.baseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider)); + BaseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider)); IOManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); } /// - public virtual ValueTask DisposeAsync() => baseProvider.DisposeAsync(); + public virtual ValueTask DisposeAsync() => BaseProvider.DisposeAsync(); /// - public void KeepAlive() => baseProvider.KeepAlive(); + public void KeepAlive() => BaseProvider.KeepAlive(); /// /// Make the active by replacing the live link with our . @@ -88,25 +88,13 @@ public Task MakeActive(CancellationToken cancellationToken) /// /// The for the operation. /// A representing the preparation process. - public virtual Task FinishActivationPreparation(CancellationToken cancellationToken) - => Task.CompletedTask; + public abstract Task FinishActivationPreparation(CancellationToken cancellationToken); /// /// Perform the swapping action. /// /// The for the operation. /// A representing the running operation. - protected virtual async Task DoSwap(CancellationToken cancellationToken) - { - if (SymlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) - await IOManager.DeleteFile(LiveGameDirectory, cancellationToken); - else - await IOManager.DeleteDirectory(LiveGameDirectory, cancellationToken); - - await SymlinkFactory.CreateSymbolicLink( - IOManager.ResolvePath(baseProvider.Directory), - IOManager.ResolvePath(LiveGameDirectory), - cancellationToken); - } + protected abstract Task DoSwap(CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs new file mode 100644 index 00000000000..7e0845b95cf --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs @@ -0,0 +1,44 @@ +using System.Threading; +using System.Threading.Tasks; + +using Tgstation.Server.Host.IO; + +namespace Tgstation.Server.Host.Components.Deployment +{ + /// + /// A that uses symlinks. + /// + sealed class SymlinkDmbProvider : SwappableDmbProvider + { + /// + /// Initializes a new instance of the class. + /// + /// The for the . + /// The for the . + /// The for the . + public SymlinkDmbProvider( + IDmbProvider baseProvider, + IIOManager ioManager, + ISymlinkFactory symlinkFactory) + : base(baseProvider, ioManager, symlinkFactory) + { + } + + /// + public override Task FinishActivationPreparation(CancellationToken cancellationToken) => Task.CompletedTask; + + /// + protected override async Task DoSwap(CancellationToken cancellationToken) + { + if (SymlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) + await IOManager.DeleteFile(LiveGameDirectory, cancellationToken); + else + await IOManager.DeleteDirectory(LiveGameDirectory, cancellationToken); + + await SymlinkFactory.CreateSymbolicLink( + IOManager.ResolvePath(BaseProvider.Directory), + IOManager.ResolvePath(LiveGameDirectory), + cancellationToken); + } + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index 917bafa1348..d882296b068 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -85,6 +85,6 @@ protected override async Task ApplyInitialDmb(CancellationToken cancellationToke /// protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) - => new SwappableDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); + => new SymlinkDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); } } From fd6de0fee2d8ab0b2e950de59a94001335952a24 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:42:19 -0400 Subject: [PATCH 026/138] Rename *`SymlinkFactory` to *`FilesystemLinkFactory` to better reflect new behavior --- .../Deployment/HardLinkDmbProvider.cs | 8 +++---- .../Deployment/SwappableDmbProvider.cs | 10 ++++----- .../Deployment/SymlinkDmbProvider.cs | 10 ++++----- .../Components/InstanceFactory.cs | 12 +++++----- .../Components/StaticFiles/Configuration.cs | 12 +++++----- .../Components/Watchdog/AdvancedWatchdog.cs | 10 ++++----- .../Components/Watchdog/PosixWatchdog.cs | 8 +++---- .../Watchdog/PosixWatchdogFactory.cs | 8 +++---- .../Components/Watchdog/WindowsWatchdog.cs | 8 +++---- .../Watchdog/WindowsWatchdogFactory.cs | 12 +++++----- src/Tgstation.Server.Host/Core/Application.cs | 4 ++-- ...nkFactory.cs => IFilesystemLinkFactory.cs} | 2 +- ...ctory.cs => PosixFilesystemLinkFactory.cs} | 4 ++-- ...ory.cs => WindowsFilesystemLinkFactory.cs} | 4 ++-- .../StaticFiles/TestConfiguration.cs | 2 +- ...actory.cs => TestFilesystemLinkFactory.cs} | 22 +++++++++---------- .../IO/TestIOManager.cs | 6 ++--- .../System/TestSymlinkFactory.cs | 6 ++--- 18 files changed, 74 insertions(+), 74 deletions(-) rename src/Tgstation.Server.Host/IO/{ISymlinkFactory.cs => IFilesystemLinkFactory.cs} (97%) rename src/Tgstation.Server.Host/IO/{PosixSymlinkFactory.cs => PosixFilesystemLinkFactory.cs} (92%) rename src/Tgstation.Server.Host/IO/{WindowsSymlinkFactory.cs => WindowsFilesystemLinkFactory.cs} (92%) rename tests/Tgstation.Server.Host.Tests/IO/{TestSymlinkFactory.cs => TestFilesystemLinkFactory.cs} (80%) diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs index 634af753f42..cd9a3f1185a 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -40,19 +40,19 @@ sealed class HardLinkDmbProvider : SwappableDmbProvider /// /// The for the . /// The for the . - /// The for the . + /// The for the . /// The value of . /// The for the . public HardLinkDmbProvider( IDmbProvider baseProvider, IIOManager ioManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, ILogger logger, GeneralConfiguration generalConfiguration) : base( baseProvider, ioManager, - symlinkFactory) + linkFactory) { this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); cancellationTokenSource = new CancellationTokenSource(); @@ -214,7 +214,7 @@ async Task LinkThisFile() using var lockContext = semaphore != null ? await SemaphoreSlimContext.Lock(semaphore, cancellationToken) : null; - await SymlinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken); + await LinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken); } yield return LinkThisFile(); diff --git a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs index b82f5cc6761..b00ba3d518e 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs @@ -42,9 +42,9 @@ abstract class SwappableDmbProvider : IDmbProvider protected IIOManager IOManager { get; } /// - /// The to use. + /// The to use. /// - protected ISymlinkFactory SymlinkFactory { get; } + protected IFilesystemLinkFactory LinkFactory { get; } /// /// Backing field for . @@ -56,12 +56,12 @@ abstract class SwappableDmbProvider : IDmbProvider /// /// The value of . /// The value of . - /// The value of . - public SwappableDmbProvider(IDmbProvider baseProvider, IIOManager ioManager, ISymlinkFactory symlinkFactory) + /// The value of . + public SwappableDmbProvider(IDmbProvider baseProvider, IIOManager ioManager, IFilesystemLinkFactory symlinkFactory) { BaseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider)); IOManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); - SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + LinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); } /// diff --git a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs index 7e0845b95cf..2edc58d37eb 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs @@ -15,12 +15,12 @@ sealed class SymlinkDmbProvider : SwappableDmbProvider /// /// The for the . /// The for the . - /// The for the . + /// The for the . public SymlinkDmbProvider( IDmbProvider baseProvider, IIOManager ioManager, - ISymlinkFactory symlinkFactory) - : base(baseProvider, ioManager, symlinkFactory) + IFilesystemLinkFactory linkFactory) + : base(baseProvider, ioManager, linkFactory) { } @@ -30,12 +30,12 @@ public SymlinkDmbProvider( /// protected override async Task DoSwap(CancellationToken cancellationToken) { - if (SymlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) + if (LinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) await IOManager.DeleteFile(LiveGameDirectory, cancellationToken); else await IOManager.DeleteDirectory(LiveGameDirectory, cancellationToken); - await SymlinkFactory.CreateSymbolicLink( + await LinkFactory.CreateSymbolicLink( IOManager.ResolvePath(BaseProvider.Directory), IOManager.ResolvePath(LiveGameDirectory), cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 8c7a8df85e6..a3c7ba21925 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -66,9 +66,9 @@ sealed class InstanceFactory : IInstanceFactory readonly ISynchronousIOManager synchronousIOManager; /// - /// The for the . + /// The for the . /// - readonly ISymlinkFactory symlinkFactory; + readonly IFilesystemLinkFactory linkFactory; /// /// The for the . @@ -173,7 +173,7 @@ sealed class InstanceFactory : IInstanceFactory /// The value of . /// The value of . /// The value of . - /// The value of . + /// The value of . /// The value of . /// The value of . /// The value of . @@ -199,7 +199,7 @@ public InstanceFactory( ITopicClientFactory topicClientFactory, ICryptographySuite cryptographySuite, ISynchronousIOManager synchronousIOManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, IByondInstaller byondInstaller, IChatManagerFactory chatFactory, IProcessExecutor processExecutor, @@ -225,7 +225,7 @@ public InstanceFactory( this.topicClientFactory = topicClientFactory ?? throw new ArgumentNullException(nameof(topicClientFactory)); this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); this.synchronousIOManager = synchronousIOManager ?? throw new ArgumentNullException(nameof(synchronousIOManager)); - this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + this.linkFactory = linkFactory ?? throw new ArgumentNullException(nameof(linkFactory)); this.byondInstaller = byondInstaller ?? throw new ArgumentNullException(nameof(byondInstaller)); this.chatFactory = chatFactory ?? throw new ArgumentNullException(nameof(chatFactory)); this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); @@ -275,7 +275,7 @@ public async Task CreateInstance(IBridgeRegistrar bridgeRegistrar, Mo var configuration = new StaticFiles.Configuration( configurationIoManager, synchronousIOManager, - symlinkFactory, + linkFactory, processExecutor, postWriteHandler, platformIdentifier, diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 087503d68c4..eeb5cd71c65 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -94,9 +94,9 @@ sealed class Configuration : IConfiguration readonly ISynchronousIOManager synchronousIOManager; /// - /// The for . + /// The for . /// - readonly ISymlinkFactory symlinkFactory; + readonly IFilesystemLinkFactory linkFactory; /// /// The for . @@ -153,7 +153,7 @@ sealed class Configuration : IConfiguration /// /// The value of . /// The value of . - /// The value of . + /// The value of . /// The value of . /// The value of . /// The value of . @@ -164,7 +164,7 @@ sealed class Configuration : IConfiguration public Configuration( IIOManager ioManager, ISynchronousIOManager synchronousIOManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, IProcessExecutor processExecutor, IPostWriteHandler postWriteHandler, IPlatformIdentifier platformIdentifier, @@ -175,7 +175,7 @@ public Configuration( { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.synchronousIOManager = synchronousIOManager ?? throw new ArgumentNullException(nameof(synchronousIOManager)); - this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + this.linkFactory = linkFactory ?? throw new ArgumentNullException(nameof(linkFactory)); this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); this.postWriteHandler = postWriteHandler ?? throw new ArgumentNullException(nameof(postWriteHandler)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); @@ -450,7 +450,7 @@ await Task.WhenAll(entries.Select(async file => var fileExists = await fileExistsTask; if (fileExists) await ioManager.DeleteFile(destPath, cancellationToken); - await symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(file), ioManager.ResolvePath(destPath), cancellationToken); + await linkFactory.CreateSymbolicLink(ioManager.ResolvePath(file), ioManager.ResolvePath(destPath), cancellationToken); })); } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 07954e5a88f..48d2b3e2546 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -35,9 +35,9 @@ abstract class AdvancedWatchdog : BasicWatchdog protected IIOManager GameIOManager { get; } /// - /// The for the . + /// The for the . /// - protected ISymlinkFactory SymlinkFactory { get; } + protected IFilesystemLinkFactory LinkFactory { get; } /// /// of s that are waiting to clean up old deployments. @@ -68,7 +68,7 @@ abstract class AdvancedWatchdog : BasicWatchdog /// The for the . /// The for the . /// The value of . - /// The value of . + /// The value of . /// The for the . /// The for the . /// The for the . @@ -85,7 +85,7 @@ public AdvancedWatchdog( IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IIOManager gameIOManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, @@ -109,7 +109,7 @@ public AdvancedWatchdog( try { GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager)); - SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + LinkFactory = linkFactory ?? throw new ArgumentNullException(nameof(linkFactory)); deploymentCleanupTasks = new List(); } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index 961fee9988a..eb563bd1fa0 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -42,7 +42,7 @@ sealed class PosixWatchdog : AdvancedWatchdog /// The for the . /// The for the . /// The pointing to the game directory for the .. - /// The for the . + /// The for the . /// The for the . /// The for the . /// The for the . @@ -60,7 +60,7 @@ public PosixWatchdog( IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IIOManager gameIOManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, @@ -78,7 +78,7 @@ public PosixWatchdog( eventConsumer, remoteDeploymentManagerFactory, gameIOManager, - symlinkFactory, + linkFactory, logger, initialLaunchParameters, instance, @@ -93,6 +93,6 @@ protected override Task ApplyInitialDmb(CancellationToken cancellationToken) /// protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) - => new HardLinkDmbProvider(dmbProvider, GameIOManager, SymlinkFactory, Logger, generalConfiguration); + => new HardLinkDmbProvider(dmbProvider, GameIOManager, LinkFactory, Logger, generalConfiguration); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs index 787127a05c9..64c86722625 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs @@ -29,21 +29,21 @@ sealed class PosixWatchdogFactory : WindowsWatchdogFactory /// The for the . /// The for the . /// The for the . - /// The for the . + /// The for the . /// The for for the . public PosixWatchdogFactory( IServerControl serverControl, ILoggerFactory loggerFactory, IJobManager jobManager, IAsyncDelayer asyncDelayer, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, IOptions generalConfigurationOptions) : base( serverControl, loggerFactory, jobManager, asyncDelayer, - symlinkFactory, + linkFactory, generalConfigurationOptions) { } @@ -72,7 +72,7 @@ public override IWatchdog CreateWatchdog( eventConsumer, remoteDeploymentManagerFactory, gameIOManager, - SymlinkFactory, + LinkFactory, LoggerFactory.CreateLogger(), settings, instance, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index d882296b068..05c1f100888 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -35,7 +35,7 @@ sealed class WindowsWatchdog : AdvancedWatchdog /// The for the . /// The for the . /// The pointing to the game directory for the .. - /// The for the . + /// The for the . /// The for the . /// The for the . /// The for the . @@ -52,7 +52,7 @@ public WindowsWatchdog( IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IIOManager gameIOManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, @@ -69,7 +69,7 @@ public WindowsWatchdog( eventConsumer, remoteDeploymentManagerFactory, gameIOManager, - symlinkFactory, + linkFactory, logger, initialLaunchParameters, instance, @@ -85,6 +85,6 @@ protected override async Task ApplyInitialDmb(CancellationToken cancellationToke /// protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) - => new SymlinkDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); + => new SymlinkDmbProvider(dmbProvider, GameIOManager, LinkFactory); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index 4cbe20577e6..89ed6b145d7 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -23,9 +23,9 @@ namespace Tgstation.Server.Host.Components.Watchdog class WindowsWatchdogFactory : WatchdogFactory { /// - /// The for the . + /// The for the . /// - protected ISymlinkFactory SymlinkFactory { get; } + protected IFilesystemLinkFactory LinkFactory { get; } /// /// Initializes a new instance of the class. @@ -34,14 +34,14 @@ class WindowsWatchdogFactory : WatchdogFactory /// The for the . /// The for the . /// The for the . - /// The value of . + /// The value of . /// The for for the . public WindowsWatchdogFactory( IServerControl serverControl, ILoggerFactory loggerFactory, IJobManager jobManager, IAsyncDelayer asyncDelayer, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory symlinkFactory, IOptions generalConfigurationOptions) : base( serverControl, @@ -50,7 +50,7 @@ public WindowsWatchdogFactory( asyncDelayer, generalConfigurationOptions) { - SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + LinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); } /// @@ -77,7 +77,7 @@ public override IWatchdog CreateWatchdog( eventConsumer, remoteDeploymentManagerFactory, gameIOManager, - SymlinkFactory, + LinkFactory, LoggerFactory.CreateLogger(), settings, instance, diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index ed1cb97cad0..18abdcacf30 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -336,7 +336,7 @@ void AddTypedContext() { AddWatchdog(services, postSetupServices); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -349,7 +349,7 @@ void AddTypedContext() { AddWatchdog(services, postSetupServices); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs b/src/Tgstation.Server.Host/IO/IFilesystemLinkFactory.cs similarity index 97% rename from src/Tgstation.Server.Host/IO/ISymlinkFactory.cs rename to src/Tgstation.Server.Host/IO/IFilesystemLinkFactory.cs index 08320a3d150..3cf0622801a 100644 --- a/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/IFilesystemLinkFactory.cs @@ -6,7 +6,7 @@ namespace Tgstation.Server.Host.IO /// /// For creating filesystem symbolic links. /// - interface ISymlinkFactory + interface IFilesystemLinkFactory { /// /// If directory symlinks must be deleted as files would in the current environment. diff --git a/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs b/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs similarity index 92% rename from src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs rename to src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs index 05d64a3bf07..b80af9e122a 100644 --- a/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs @@ -8,9 +8,9 @@ namespace Tgstation.Server.Host.IO { /// - /// for posix systems. + /// for POSIX systems. /// - sealed class PosixSymlinkFactory : ISymlinkFactory + sealed class PosixFilesystemLinkFactory : IFilesystemLinkFactory { /// public bool SymlinkedDirectoriesAreDeletedAsFiles => true; diff --git a/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs b/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs similarity index 92% rename from src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs rename to src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs index 864242d992d..1513f4e1666 100644 --- a/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs @@ -9,9 +9,9 @@ namespace Tgstation.Server.Host.IO { /// - /// for windows systems. + /// for windows systems. /// - sealed class WindowsSymlinkFactory : ISymlinkFactory + sealed class WindowsFilesystemLinkFactory : IFilesystemLinkFactory { /// public bool SymlinkedDirectoriesAreDeletedAsFiles => false; diff --git a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs index 68241988d11..ff3bd0b5b83 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs @@ -47,7 +47,7 @@ public async Task TestListOrdering() var configuration = new Configuration( ioManager, new SynchronousIOManager(), - Mock.Of(), + Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of(), diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestSymlinkFactory.cs b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs similarity index 80% rename from tests/Tgstation.Server.Host.Tests/IO/TestSymlinkFactory.cs rename to tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs index 27646a9dc76..e735c485414 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestSymlinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs @@ -8,17 +8,17 @@ namespace Tgstation.Server.Host.IO.Tests { [TestClass] - public sealed class TestSymlinkFactory + public sealed class TestFilesystemLinkFactory { - static ISymlinkFactory symlinkFactory; + static IFilesystemLinkFactory linkFactory; [ClassInitialize] public static void SelectFactory(TestContext _) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - symlinkFactory = new WindowsSymlinkFactory(); + linkFactory = new WindowsFilesystemLinkFactory(); else - symlinkFactory = new PosixSymlinkFactory(); + linkFactory = new PosixFilesystemLinkFactory(); } public static bool HasPermissionToMakeSymlinks() @@ -43,10 +43,10 @@ public async Task TestFileWorks() f2 = f1 + ".linked"; File.WriteAllText(f1, Text); - await Assert.ThrowsExceptionAsync(() => symlinkFactory.CreateSymbolicLink(null, null, default)); - await Assert.ThrowsExceptionAsync(() => symlinkFactory.CreateSymbolicLink(f1, null, default)); + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateSymbolicLink(null, null, default)); + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateSymbolicLink(f1, null, default)); - await symlinkFactory.CreateSymbolicLink(f1, f2, default); + await linkFactory.CreateSymbolicLink(f1, f2, default); Assert.IsTrue(File.Exists(f2)); var f2Contents = File.ReadAllText(f2); @@ -76,10 +76,10 @@ public async Task TestDirectoryWorks() var p1 = Path.Combine(f1, FileName); File.WriteAllText(p1, Text); - await Assert.ThrowsExceptionAsync(() => symlinkFactory.CreateSymbolicLink(null, null, default)); - await Assert.ThrowsExceptionAsync(() => symlinkFactory.CreateSymbolicLink(f1, null, default)); + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateSymbolicLink(null, null, default)); + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateSymbolicLink(f1, null, default)); - await symlinkFactory.CreateSymbolicLink(f1, f2, default); + await linkFactory.CreateSymbolicLink(f1, f2, default); var p2 = Path.Combine(f2, FileName); Assert.IsTrue(File.Exists(p2)); @@ -104,7 +104,7 @@ public async Task TestFailsProperly() try { - await symlinkFactory.CreateSymbolicLink(BadPath, BadPath, default); + await linkFactory.CreateSymbolicLink(BadPath, BadPath, default); Assert.Fail("No exception thrown!"); } catch { } diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs index 5e34e809dc1..00319f4f30e 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs @@ -42,9 +42,9 @@ public async Task TestDeleteDirectory() [TestMethod] public async Task TestDeleteDirectoryWithSymlinkInsideDoesntRecurse() { - var linkFactory = (ISymlinkFactory)(new PlatformIdentifier().IsWindows - ? new WindowsSymlinkFactory() - : new PosixSymlinkFactory()); + var linkFactory = (IFilesystemLinkFactory)(new PlatformIdentifier().IsWindows + ? new WindowsFilesystemLinkFactory() + : new PosixFilesystemLinkFactory()); var tempPath = Path.GetTempFileName(); File.Delete(tempPath); diff --git a/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs b/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs index 8e488f11b0d..4d3c5be7492 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs @@ -12,9 +12,9 @@ namespace Tgstation.Server.Host.System.Tests [TestClass] public sealed class TestSymlinkFactory { - readonly ISymlinkFactory factory = new PlatformIdentifier().IsWindows - ? new WindowsSymlinkFactory() - : new PosixSymlinkFactory(); + readonly IFilesystemLinkFactory factory = new PlatformIdentifier().IsWindows + ? new WindowsFilesystemLinkFactory() + : new PosixFilesystemLinkFactory(); [TestMethod] public async Task TestSymlinks() From 5bb655e3f6d3acc6963054f9517c8cd5257c522b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:48:14 -0400 Subject: [PATCH 027/138] Add test for hard links --- .../IO/TestFilesystemLinkFactory.cs | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs index e735c485414..dbf2aaf2a82 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs @@ -1,10 +1,12 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; +using System; using System.IO; using System.Runtime.InteropServices; using System.Security.Principal; +using System.Threading; using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + namespace Tgstation.Server.Host.IO.Tests { [TestClass] @@ -30,6 +32,57 @@ public static bool HasPermissionToMakeSymlinks() return principal.IsInRole(WindowsBuiltInRole.Administrator); } + [TestMethod] + public async Task TestHardLinkWorks() + { + string f2 = null; + var f1 = Path.GetTempFileName(); + try + { + f2 = f1 + ".linked"; + const string Text = "Hello world"; + File.WriteAllText(f1, Text); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateHardLink(f1, f2, CancellationToken.None)); + Assert.Inconclusive("Windows does not support hardlinks"); + } + + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateHardLink(null, null, CancellationToken.None)); + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateHardLink(f1, null, CancellationToken.None)); + + await linkFactory.CreateHardLink(f1, f2, default); + Assert.IsTrue(File.Exists(f2)); + + var f2Contents = File.ReadAllText(f2); + Assert.AreEqual(Text, f2Contents); + + const string NewText = "asdf"; + File.WriteAllText(f1, NewText); + + f2Contents = File.ReadAllText(f2); + Assert.AreEqual(NewText, f2Contents); + + const string NewText2 = "fdsa"; + File.WriteAllText(f2, NewText2); + + var f1Contents = File.ReadAllText(f1); + Assert.AreEqual(NewText2, f1Contents); + + File.Delete(f1); + Assert.IsFalse(File.Exists(f1)); + Assert.IsTrue(File.Exists(f2)); + f2Contents = File.ReadAllText(f2); + Assert.AreEqual(NewText2, f2Contents); + } + finally + { + File.Delete(f2); + File.Delete(f1); + } + } + [TestMethod] public async Task TestFileWorks() { From abdf8b6bfd620ebad7b973c8bb9b737dd7792338 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 19:06:41 -0400 Subject: [PATCH 028/138] Fix build --- .../Live/Instance/ConfigurationTest.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index 53064c9b7c0..dac5b9b53c9 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -93,7 +93,7 @@ await configurationClient.Write(new ConfigurationFileRequest await configurationClient.CreateDirectory(staticDir, cancellationToken); } - public Task SetupDMApiTests(bool includingRoot, CancellationToken cancellationToken) + public ValueTask SetupDMApiTests(bool includingRoot, CancellationToken cancellationToken) { // just use an I/O manager here var ioManager = new DefaultIOManager(); @@ -110,7 +110,7 @@ public Task SetupDMApiTests(bool includingRoot, CancellationToken cancellationTo "../../../../DMAPI/LongRunning/long_running_test_rooted.dme", ioManager.ConcatPath(instance.Path, "Repository", "long_running_test_rooted.dme"), cancellationToken) - : Task.CompletedTask, + : ValueTask.CompletedTask, ioManager.CopyDirectory( Enumerable.Empty(), null, @@ -182,9 +182,8 @@ async Task SequencedApiTests(CancellationToken cancellationToken) } public Task RunPreWatchdog(CancellationToken cancellationToken) => Task.WhenAll( - SetupDMApiTests(false, cancellationToken), + SetupDMApiTests(false, cancellationToken).AsTask(), SequencedApiTests(cancellationToken), - SetupDMApiTests(cancellationToken), TestPregeneratedFilesExist(cancellationToken)); } } From 53eafffbd07371fddcac177403d7e6d9dbf48ecc Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 00:02:59 -0400 Subject: [PATCH 029/138] Add world.TgsVisibility() to DMAPI --- build/Version.props | 2 +- src/DMAPI/tgs.dm | 13 ++++++++++++- src/DMAPI/tgs/core/core.dm | 5 +++++ src/DMAPI/tgs/core/datum.dm | 3 +++ src/DMAPI/tgs/v5/_defines.dm | 1 + src/DMAPI/tgs/v5/api.dm | 6 ++++++ src/DMAPI/tgs/v5/undefs.dm | 1 + tests/DMAPI/LongRunning/Test.dm | 6 ++++++ 8 files changed, 35 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index 99cef97f0d3..5022bac52e3 100644 --- a/build/Version.props +++ b/build/Version.props @@ -9,7 +9,7 @@ 7.0.0 11.1.2 13.0.0 - 6.5.4 + 6.6.0 5.6.2 1.4.0 1.2.1 diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index d0466b806ff..9825cd118b6 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,6 +1,6 @@ // tgstation-server DMAPI -#define TGS_DMAPI_VERSION "6.5.4" +#define TGS_DMAPI_VERSION "6.6.0" // All functions and datums outside this document are subject to change with any version and should not be relied on. @@ -129,6 +129,13 @@ /// DreamDaemon Ultrasafe security level. #define TGS_SECURITY_ULTRASAFE 2 +/// DreamDaemon public visibility level. +#define TGS_VISIBILITY_PUBLIC 0 +/// DreamDaemon private visibility level. +#define TGS_VISIBILITY_PRIVATE 1 +/// DreamDaemon invisible visibility level. +#define TGS_VISIBILITY_INVISIBLE 2 + //REQUIRED HOOKS /** @@ -458,6 +465,10 @@ /world/proc/TgsSecurityLevel() return +/// Returns the current BYOND visibility level as a TGS_VISIBILITY_ define if TGS is present, null otherwise. Requires TGS to be using interop API version 5 or higher otherwise the string "___unimplemented" wil be returned. This function may sleep if the call to [/world/proc/TgsNew] is sleeping! +/world/proc/TgsVisibility() + return + /// Returns a list of active [/datum/tgs_revision_information/test_merge]s if TGS is present, null otherwise. This function may sleep if the call to [/world/proc/TgsNew] is sleeping! /world/proc/TgsTestMerges() return diff --git a/src/DMAPI/tgs/core/core.dm b/src/DMAPI/tgs/core/core.dm index aa4084904b7..b9a9f27a28a 100644 --- a/src/DMAPI/tgs/core/core.dm +++ b/src/DMAPI/tgs/core/core.dm @@ -154,3 +154,8 @@ var/datum/tgs_api/api = TGS_READ_GLOBAL(tgs) if(api) return api.SecurityLevel() + +/world/TgsVisibility() + var/datum/tgs_api/api = TGS_READ_GLOBAL(tgs) + if(api) + return api.Visibility() diff --git a/src/DMAPI/tgs/core/datum.dm b/src/DMAPI/tgs/core/datum.dm index 68b0330fe86..93377079aa7 100644 --- a/src/DMAPI/tgs/core/datum.dm +++ b/src/DMAPI/tgs/core/datum.dm @@ -57,3 +57,6 @@ TGS_PROTECT_DATUM(/datum/tgs_api) /datum/tgs_api/proc/SecurityLevel() return TGS_UNIMPLEMENTED + +/datum/tgs_api/proc/Visibility() + return TGS_UNIMPLEMENTED diff --git a/src/DMAPI/tgs/v5/_defines.dm b/src/DMAPI/tgs/v5/_defines.dm index f973338daa0..bdcd4e4dd58 100644 --- a/src/DMAPI/tgs/v5/_defines.dm +++ b/src/DMAPI/tgs/v5/_defines.dm @@ -48,6 +48,7 @@ #define DMAPI5_RUNTIME_INFORMATION_REVISION "revision" #define DMAPI5_RUNTIME_INFORMATION_TEST_MERGES "testMerges" #define DMAPI5_RUNTIME_INFORMATION_SECURITY_LEVEL "securityLevel" +#define DMAPI5_RUNTIME_INFORMATION_VISIBILITY "visibility" #define DMAPI5_CHAT_UPDATE_CHANNELS "channels" diff --git a/src/DMAPI/tgs/v5/api.dm b/src/DMAPI/tgs/v5/api.dm index 34cc43f8762..45250efc462 100644 --- a/src/DMAPI/tgs/v5/api.dm +++ b/src/DMAPI/tgs/v5/api.dm @@ -4,6 +4,7 @@ var/instance_name var/security_level + var/visibility var/reboot_mode = TGS_REBOOT_MODE_NORMAL @@ -54,6 +55,7 @@ version = new /datum/tgs_version(runtime_information[DMAPI5_RUNTIME_INFORMATION_SERVER_VERSION]) security_level = runtime_information[DMAPI5_RUNTIME_INFORMATION_SECURITY_LEVEL] + visibility = runtime_information[DMAPI5_RUNTIME_INFORMATION_VISIBILITY] instance_name = runtime_information[DMAPI5_RUNTIME_INFORMATION_INSTANCE_NAME] var/list/revisionData = runtime_information[DMAPI5_RUNTIME_INFORMATION_REVISION] @@ -252,3 +254,7 @@ /datum/tgs_api/v5/SecurityLevel() RequireInitialBridgeResponse() return security_level + +/datum/tgs_api/v5/Visibility() + RequireInitialBridgeResponse() + return visibility diff --git a/src/DMAPI/tgs/v5/undefs.dm b/src/DMAPI/tgs/v5/undefs.dm index c679737dfc4..f163adaaafe 100644 --- a/src/DMAPI/tgs/v5/undefs.dm +++ b/src/DMAPI/tgs/v5/undefs.dm @@ -48,6 +48,7 @@ #undef DMAPI5_RUNTIME_INFORMATION_REVISION #undef DMAPI5_RUNTIME_INFORMATION_TEST_MERGES #undef DMAPI5_RUNTIME_INFORMATION_SECURITY_LEVEL +#undef DMAPI5_RUNTIME_INFORMATION_VISIBILITY #undef DMAPI5_CHAT_UPDATE_CHANNELS diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index a77c3377b5d..68597e10d0c 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -17,6 +17,12 @@ log << "Running in security level: [sec]" + var/vis = TgsVisibility() + if(isnull(vis)) + FailTest("TGS Visibility was null!") + + log << "Running in visibility: [vis]" + if(params["expect_chat_channels"]) var/list/channels = TgsChatChannelInfo() if(!length(channels)) From cc1c955bc8f2b037f257a3eeec977683533e53c5 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 19:16:47 -0400 Subject: [PATCH 030/138] We call it the Advanced Watchdog now --- .github/workflows/ci-pipeline.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 6ce724ea5d3..766e9660b48 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -365,7 +365,7 @@ jobs: fail-fast: false matrix: database-type: [ 'SqlServer', 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ] - watchdog-type: [ 'Basic', 'System' ] + watchdog-type: [ 'Basic', 'Advanced' ] configuration: [ 'Debug', 'Release' ] runs-on: windows-latest steps: @@ -735,25 +735,25 @@ jobs: name: linux-unit-test-coverage-Release path: ./code_coverage/unit_tests/linux_unit_tests_release - - name: Retrieve Linux Integration Test Coverage (Release, System, Sqlite) + - name: Retrieve Linux Integration Test Coverage (Release, Advanced, Sqlite) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Release-System-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_release_system_sqlite - - name: Retrieve Linux Integration Test Coverage (Release, System, PostgresSql) + - name: Retrieve Linux Integration Test Coverage (Release, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Release-System-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_release_system_mariadb - - name: Retrieve Linux Integration Test Coverage (Release, System, MariaDB) + - name: Retrieve Linux Integration Test Coverage (Release, Advanced, MariaDB) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Release-System-MariaDB path: ./code_coverage/integration_tests/linux_integration_tests_release_system_mysql - - name: Retrieve Linux Integration Test Coverage (Release, System, MySql) + - name: Retrieve Linux Integration Test Coverage (Release, Advanced, MySql) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Release-System-MySql @@ -789,13 +789,13 @@ jobs: name: linux-integration-test-coverage-Debug-System-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_sqlite - - name: Retrieve Linux Integration Test Coverage (Debug, System, PostgresSql) + - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Debug-System-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_mariadb - - name: Retrieve Linux Integration Test Coverage (Debug, System, MariaDB) + - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, MariaDB) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Debug-System-MariaDB From 71f5c739d8ba9d68ef691935d3b2e18b2755761f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 19:21:11 -0400 Subject: [PATCH 031/138] Fix tests using `BasicWatchdog` failing --- .../Live/Instance/InstanceTest.cs | 6 ++++-- .../Live/Instance/WatchdogTest.cs | 12 +++++++----- .../Tgstation.Server.Tests/Live/LiveTestingServer.cs | 4 ++++ tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 6 ++++-- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 4457994d792..2940500e206 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -43,6 +43,7 @@ public async Task RunTests( ushort ddPort, bool highPrioDD, bool lowPrioDeployment, + bool usingBasicWatchdog, CancellationToken cancellationToken) { var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, fileDownloader, instanceClient.Metadata); @@ -67,7 +68,7 @@ public async Task RunTests( await byondTask; await new WatchdogTest( - await ByondTest.GetEdgeVersion(fileDownloader, cancellationToken), instanceClient, instanceManager, serverPort, highPrioDD, ddPort).Run(cancellationToken); + await ByondTest.GetEdgeVersion(fileDownloader, cancellationToken), instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog).Run(cancellationToken); } public async Task RunCompatTests( @@ -76,6 +77,7 @@ public async Task RunCompatTests( ushort dmPort, ushort ddPort, bool highPrioDD, + bool usingBasicWatchdog, CancellationToken cancellationToken) { System.Console.WriteLine($"COMPAT TEST START: {compatVersion}"); @@ -189,7 +191,7 @@ await instanceClient.Repository.Update(new RepositoryUpdateRequest await configSetupTask; - await new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort).Run(cancellationToken); + await new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog).Run(cancellationToken); await instanceManagerClient.Update(new InstanceUpdateRequest { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 96ef4d3c201..b59e9532fec 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -63,10 +63,11 @@ sealed class WatchdogTest : JobsRequiredTest readonly bool highPrioDD; readonly TopicClient topicClient; readonly Version testVersion; + readonly bool usingBasicWatchdog; bool ranTimeoutTest = false; - public WatchdogTest(Version testVersion, IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD, ushort ddPort) + public WatchdogTest(Version testVersion, IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD, ushort ddPort, bool usingBasicWatchdog) : base(instanceClient.Jobs) { this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient)); @@ -75,8 +76,9 @@ public WatchdogTest(Version testVersion, IInstanceClient instanceClient, Instanc this.highPrioDD = highPrioDD; this.ddPort = ddPort; this.testVersion = testVersion ?? throw new ArgumentNullException(nameof(testVersion)); + this.usingBasicWatchdog = usingBasicWatchdog; - this.topicClient = new(new SocketParameters + topicClient = new(new SocketParameters { SendTimeout = TimeSpan.FromSeconds(30), ReceiveTimeout = TimeSpan.FromSeconds(30), @@ -476,9 +478,9 @@ async Task RunBasicTest(CancellationToken cancellationToken) Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters); } - void TestLinuxIsntBeingFuckingCheekyAboutFilePaths(DreamDaemonResponse currentStatus, CompileJobResponse previousStatus, CancellationToken cancellationToken) + void TestLinuxIsntBeingFuckingCheekyAboutFilePaths(DreamDaemonResponse currentStatus, CompileJobResponse previousStatus) { - if (new PlatformIdentifier().IsWindows) + if (new PlatformIdentifier().IsWindows || usingBasicWatchdog) return; Assert.IsNotNull(currentStatus.ActiveCompileJob); @@ -947,7 +949,7 @@ async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken) Assert.AreNotEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); Assert.IsNull(daemonStatus.StagedCompileJob); - TestLinuxIsntBeingFuckingCheekyAboutFilePaths(daemonStatus, initialCompileJob, cancellationToken); + TestLinuxIsntBeingFuckingCheekyAboutFilePaths(daemonStatus, initialCompileJob); await instanceClient.DreamDaemon.Shutdown(cancellationToken); diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 56263c17171..33213f6f7c5 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -63,6 +63,8 @@ static async Task Cleanup(string directory) public bool HighPriorityDreamDaemon { get; } public bool LowPriorityDeployments { get; } + public bool UsingBasicWatchdog { get; } + public bool RestartRequested => RealServer.RestartRequested; readonly List args; @@ -89,6 +91,8 @@ public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth var gitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"); var dumpOpenAPISpecPathEnvVar = Environment.GetEnvironmentVariable("TGS_TEST_DUMP_API_SPEC"); + UsingBasicWatchdog = Boolean.TryParse(Environment.GetEnvironmentVariable("General__UseBasicWatchdog"), out var result) && result; + if (String.IsNullOrEmpty(DatabaseType)) Assert.Inconclusive("No database type configured in env var TGS_TEST_DATABASE_TYPE!"); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index a78b984fc2e..21ab8be8442 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1336,6 +1336,7 @@ async Task RunInstanceTests() compatDMPort, compatDDPort, server.HighPriorityDreamDaemon, + server.UsingBasicWatchdog, cancellationToken)); if (TestingUtils.RunningInGitHubActions) // they only have 2 cores, can't handle intense parallelization @@ -1349,6 +1350,7 @@ await FailFast( mainDDPort, server.HighPriorityDreamDaemon, server.LowPriorityDeployments, + server.UsingBasicWatchdog, cancellationToken)); await compatTests; @@ -1529,7 +1531,7 @@ async Task WaitForInitialJobs(IInstanceClient instanceClient) Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); - var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon, mainDDPort); + var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1576,7 +1578,7 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); - var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon, mainDDPort); + var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); currentDD = await wdt.TellWorldToReboot(cancellationToken); Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value); Assert.IsNull(currentDD.StagedCompileJob); From b22368086becd526e59da65aa6dc71a65d3f8d6a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 20:40:04 -0400 Subject: [PATCH 032/138] Additional test logging --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index b59e9532fec..784fce5f67c 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -502,7 +502,7 @@ void TestLinuxIsntBeingFuckingCheekyAboutFilePaths(DreamDaemonResponse currentSt var path = sb.ToString(); - allPaths.Add(path); + allPaths.Add($"Path: {path}"); if (path.Contains($"Game/{previousStatus.DirectoryName}")) failingLinks.Add($"Found fd {fd} resolving to previous absolute path game dir path: {path}"); @@ -516,7 +516,7 @@ void TestLinuxIsntBeingFuckingCheekyAboutFilePaths(DreamDaemonResponse currentSt if (!foundLivePath) failingLinks.Add($"Failed to find a path containing the 'Live' directory!"); - Assert.IsTrue(failingLinks.Count == 0, String.Join(Environment.NewLine, failingLinks)); + Assert.IsTrue(failingLinks.Count == 0, String.Join(Environment.NewLine, failingLinks.Concat(allPaths))); } async Task RunHealthCheckTest(bool checkDump, CancellationToken cancellationToken) From b53b29189b86081b714e2557e2bf298285a10fef Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 23:17:00 -0400 Subject: [PATCH 033/138] Fix naming of CI matrix steps --- .github/workflows/ci-pipeline.yml | 90 +++++++++++++++---------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 766e9660b48..8e75c03afb2 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -382,7 +382,7 @@ jobs: dotnet-version: ${{ env.TGS_DOTNET_VERSION }} - name: Set TGS_TEST_DUMP_API_SPEC - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'SqlServer' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'SqlServer' }} run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV - name: Set General__UseBasicWatchdog @@ -473,7 +473,7 @@ jobs: path: ./TestResults/ - name: Store OpenAPI Spec - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'SqlServer' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'SqlServer' }} uses: actions/upload-artifact@v3 with: name: openapi-spec @@ -562,7 +562,7 @@ jobs: fail-fast: false matrix: database-type: [ 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ] - watchdog-type: [ 'Basic', 'System' ] + watchdog-type: [ 'Basic', 'Advanced' ] configuration: [ 'Debug', 'Release' ] runs-on: ubuntu-latest steps: @@ -645,7 +645,7 @@ jobs: path: ./TestResults/ - name: Package Server Console - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'MariaDB' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'MariaDB' }} run: | cd src/Tgstation.Server.Host.Console dotnet publish -c ${{ matrix.configuration }} -o ../../artifacts/Console @@ -657,7 +657,7 @@ jobs: build/RemoveUnsupportedRuntimes.sh artifacts/Console - name: Package Server Update Package - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'PostgresSql' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'PostgresSql' }} run: | cd src/Tgstation.Server.Host dotnet publish -c ${{ matrix.configuration }}NoWindows --no-build -o ../../artifacts/ServerUpdate @@ -666,14 +666,14 @@ jobs: build/RemoveUnsupportedRuntimes.sh artifacts/ServerUpdate - name: Store Server Console - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'MariaDB' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'MariaDB' }} uses: actions/upload-artifact@v3 with: name: ServerConsole path: artifacts/Console/ - name: Store Server Update Package - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'PostgresSql' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'PostgresSql' }} uses: actions/upload-artifact@v3 with: name: ServerUpdatePackage @@ -738,97 +738,97 @@ jobs: - name: Retrieve Linux Integration Test Coverage (Release, Advanced, Sqlite) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-Sqlite + name: linux-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_release_system_sqlite - name: Retrieve Linux Integration Test Coverage (Release, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-PostgresSql + name: linux-integration-test-coverage-Release-Advanced-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_release_system_mariadb - name: Retrieve Linux Integration Test Coverage (Release, Advanced, MariaDB) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-MariaDB + name: linux-integration-test-coverage-Release-Advanced-MariaDB path: ./code_coverage/integration_tests/linux_integration_tests_release_system_mysql - name: Retrieve Linux Integration Test Coverage (Release, Advanced, MySql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-MySql + name: linux-integration-test-coverage-Release-Advanced-MySql path: ./code_coverage/integration_tests/linux_integration_tests_release_system_mysql - name: Retrieve Linux Integration Test Coverage (Release, Basic, Sqlite) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-Sqlite + name: linux-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_release_basic_sqlite - name: Retrieve Linux Integration Test Coverage (Release, Basic, PostgresSql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-PostgresSql + name: linux-integration-test-coverage-Release-Advanced-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_release_basic_mariadb - name: Retrieve Linux Integration Test Coverage (Release, Basic, MariaDB) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-MariaDB + name: linux-integration-test-coverage-Release-Advanced-MariaDB path: ./code_coverage/integration_tests/linux_integration_tests_release_basic_mysql - name: Retrieve Linux Integration Test Coverage (Release, Basic, MySql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-MySql + name: linux-integration-test-coverage-Release-Advanced-MySql path: ./code_coverage/integration_tests/linux_integration_tests_release_basic_mysql - - name: Retrieve Linux Integration Test Coverage (Debug, System, Sqlite) + - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, Sqlite) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-Sqlite + name: linux-integration-test-coverage-Debug-Advanced-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_sqlite - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-PostgresSql + name: linux-integration-test-coverage-Debug-Advanced-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_mariadb - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, MariaDB) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-MariaDB + name: linux-integration-test-coverage-Debug-Advanced-MariaDB path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_mysql - - name: Retrieve Linux Integration Test Coverage (Debug, System, MySql) + - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, MySql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-MySql + name: linux-integration-test-coverage-Debug-Advanced-MySql path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_mysql - name: Retrieve Linux Integration Test Coverage (Debug, Basic, Sqlite) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-Sqlite + name: linux-integration-test-coverage-Debug-Advanced-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_debug_basic_sqlite - name: Retrieve Linux Integration Test Coverage (Debug, Basic, PostgresSql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-PostgresSql + name: linux-integration-test-coverage-Debug-Advanced-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_debug_basic_mariadb - name: Retrieve Linux Integration Test Coverage (Debug, Basic, MariaDB) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-MariaDB + name: linux-integration-test-coverage-Debug-Advanced-MariaDB path: ./code_coverage/integration_tests/linux_integration_tests_debug_basic_mysql - name: Retrieve Linux Integration Test Coverage (Debug, Basic, MySql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-MySql + name: linux-integration-test-coverage-Debug-Advanced-MySql path: ./code_coverage/integration_tests/linux_integration_tests_debug_basic_mysql - name: Retrieve Windows Unit Test Coverage (Release) @@ -849,16 +849,16 @@ jobs: name: windows-integration-test-coverage-Release-Basic-SqlServer path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_sqlserver - - name: Retrieve Windows Integration Test Coverage (Debug, System, SqlServer) + - name: Retrieve Windows Integration Test Coverage (Debug, Advanced, SqlServer) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System-SqlServer + name: windows-integration-test-coverage-Debug-Advanced-SqlServer path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_sqlserver - - name: Retrieve Windows Integration Test Coverage (Release, System, SqlServer) + - name: Retrieve Windows Integration Test Coverage (Release, Advanced, SqlServer) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System-SqlServer + name: windows-integration-test-coverage-Release-Advanced-SqlServer path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlserver - name: Retrieve Windows Integration Test Coverage (Debug, Basic, MariaDB) @@ -873,16 +873,16 @@ jobs: name: windows-integration-test-coverage-Release-Basic-MariaDB path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_mariadb - - name: Retrieve Windows Integration Test Coverage (Debug, System, MariaDB) + - name: Retrieve Windows Integration Test Coverage (Debug, Advanced, MariaDB) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System-MariaDB + name: windows-integration-test-coverage-Debug-Advanced-MariaDB path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_mariadb - - name: Retrieve Windows Integration Test Coverage (Release, System, MariaDB) + - name: Retrieve Windows Integration Test Coverage (Release, Advanced, MariaDB) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System-MariaDB + name: windows-integration-test-coverage-Release-Advanced-MariaDB path: ./code_coverage/integration_tests/windows_integration_tests_release_system_mariadb - name: Retrieve Windows Integration Test Coverage (Debug, Basic, MySql) @@ -897,16 +897,16 @@ jobs: name: windows-integration-test-coverage-Release-Basic-MySql path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_mysql - - name: Retrieve Windows Integration Test Coverage (Debug, System, MySql) + - name: Retrieve Windows Integration Test Coverage (Debug, Advanced, MySql) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System-MySql + name: windows-integration-test-coverage-Debug-Advanced-MySql path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_mysql - - name: Retrieve Windows Integration Test Coverage (Release, System, MySql) + - name: Retrieve Windows Integration Test Coverage (Release, Advanced, MySql) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System-MySql + name: windows-integration-test-coverage-Release-Advanced-MySql path: ./code_coverage/integration_tests/windows_integration_tests_release_system_mysql - name: Retrieve Windows Integration Test Coverage (Debug, Basic, PostgresSql) @@ -921,16 +921,16 @@ jobs: name: windows-integration-test-coverage-Release-Basic-PostgresSql path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_postgressql - - name: Retrieve Windows Integration Test Coverage (Debug, System, PostgresSql) + - name: Retrieve Windows Integration Test Coverage (Debug, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System-PostgresSql + name: windows-integration-test-coverage-Debug-Advanced-PostgresSql path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_postgressql - - name: Retrieve Windows Integration Test Coverage (Release, System, PostgresSql) + - name: Retrieve Windows Integration Test Coverage (Release, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System-PostgresSql + name: windows-integration-test-coverage-Release-Advanced-PostgresSql path: ./code_coverage/integration_tests/windows_integration_tests_release_system_postgressql - name: Retrieve Windows Integration Test Coverage (Debug, Basic, Sqlite) @@ -945,16 +945,16 @@ jobs: name: windows-integration-test-coverage-Release-Basic-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_sqlite - - name: Retrieve Windows Integration Test Coverage (Debug, System, Sqlite) + - name: Retrieve Windows Integration Test Coverage (Debug, Advanced, Sqlite) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System-Sqlite + name: windows-integration-test-coverage-Debug-Advanced-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_sqlite - - name: Retrieve Windows Integration Test Coverage (Release, System, Sqlite) + - name: Retrieve Windows Integration Test Coverage (Release, Advanced, Sqlite) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System-Sqlite + name: windows-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlite - name: Upload Coverage to CodeCov From cfca47d999b1090dbc57fd418197075ae94759d8 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 23:20:08 -0400 Subject: [PATCH 034/138] Version bump to 5.16.3 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index 4a4c556f247..c9c9e39ca0a 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 5.16.2 + 5.16.3 4.7.1 9.12.0 6.0.1 From 103adf13d1c580256414e06b415cea1a100e1d1b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 09:12:28 -0400 Subject: [PATCH 035/138] Increase delay before running `--link-winget` Prevents PR content from being overwritten by the MS bot --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 8e75c03afb2..02945d00e41 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1771,5 +1771,5 @@ jobs: - name: Run ReleaseNotes with --link-winget shell: powershell run: | - Sleep 15 + Sleep 600 dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --link-winget ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} From bf23f03b9118a1eb895e017ac3f9cc072f74f0e9 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 09:17:34 -0400 Subject: [PATCH 036/138] Fix DMAPI post validate timeout applying to all sessions --- .../Components/Session/SessionController.cs | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 4672b637571..8e8a4715ec5 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -167,6 +167,11 @@ async Task Wrap() /// readonly object synchronizationLock; + /// + /// If this session is meant to validate the presence of the DMAPI. + /// + readonly bool apiValidationSession; + /// /// The waits on when DreamDaemon currently has it's ports closed. /// @@ -244,7 +249,7 @@ async Task Wrap() /// The returning a to be run after the ends. /// The optional time to wait before failing the . /// If this is a reattached session. - /// If this is a DMAPI validation session. + /// The value of . public SessionController( ReattachInformation reattachInformation, Api.Models.Instance metadata, @@ -276,6 +281,8 @@ public SessionController( this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + apiValidationSession = apiValidate; + portClosedForReboot = false; disposed = false; apiValidationStatus = ApiValidationStatus.NeverValidated; @@ -296,7 +303,7 @@ public SessionController( topicSendSemaphore = new FifoSemaphore(); synchronizationLock = new object(); - if (apiValidate || DMApiAvailable) + if (apiValidationSession || DMApiAvailable) { bridgeRegistration = bridgeRegistrar.RegisterHandler(this); this.chatTrackingContext.SetChannelSink(this); @@ -775,9 +782,16 @@ async Task ProcessBridgeCommand(BridgeParameters parameters, Can break; case BridgeCommandType.Startup: - var proceedTcs = new TaskCompletionSource(); - var firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, PostValidationShutdown(proceedTcs.Task), null) == null; - proceedTcs.SetResult(firstValidationRequest); + bool firstValidationRequest; + if (apiValidationSession) + { + var proceedTcs = new TaskCompletionSource(); + firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, PostValidationShutdown(proceedTcs.Task), null) == null; + proceedTcs.SetResult(firstValidationRequest); + } + else + firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, Task.CompletedTask, null) == null; + apiValidationStatus = ApiValidationStatus.BadValidationRequest; if (!firstValidationRequest) From 812d6a619cf7b75ed3dea2079883265efb949a42 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 10:01:03 -0400 Subject: [PATCH 037/138] Remove the interactive test It's annoying to monitor --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 21ab8be8442..7ae5e35c6d0 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -955,9 +955,6 @@ await Task.WhenAny( new LiveTestingServer(null, false).Dispose(); } - [TestMethod] - public async Task TestTgstationInteractive() => await TestTgstation(true); - [TestMethod] public async Task TestTgstationHeadless() => await TestTgstation(false); From c820296b4ca0e994edd7266118a35497d90cb9c5 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 10:19:52 -0400 Subject: [PATCH 038/138] Fix initial bridge requests not expecting reboots --- .../Components/Session/SessionController.cs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 8e8a4715ec5..42054f24fb2 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -782,20 +782,16 @@ async Task ProcessBridgeCommand(BridgeParameters parameters, Can break; case BridgeCommandType.Startup: - bool firstValidationRequest; + apiValidationStatus = ApiValidationStatus.BadValidationRequest; if (apiValidationSession) { var proceedTcs = new TaskCompletionSource(); - firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, PostValidationShutdown(proceedTcs.Task), null) == null; + var firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, PostValidationShutdown(proceedTcs.Task), null) == null; proceedTcs.SetResult(firstValidationRequest); - } - else - firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, Task.CompletedTask, null) == null; - apiValidationStatus = ApiValidationStatus.BadValidationRequest; - - if (!firstValidationRequest) - return BridgeError("Startup bridge request was repeated!"); + if (!firstValidationRequest) + return BridgeError("Startup bridge request was repeated!"); + } if (parameters.Version == null) return BridgeError("Missing dmApiVersion field!"); From 4fac505a93a4d98892a6fbbc251a36ce5758b794 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 10:37:00 -0400 Subject: [PATCH 039/138] Version bump to 5.16.4 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index c9c9e39ca0a..3f21f8ad4b1 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 5.16.3 + 5.16.4 4.7.1 9.12.0 6.0.1 From f0d5286a206f4ab1d9f289dea892d3bee00d04a7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 11:13:16 -0400 Subject: [PATCH 040/138] Disable this validation until BYOND bug is fixed See https://www.byond.com/forum/post/2894866 --- .../Components/Session/SessionController.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 42054f24fb2..b33d3628a1e 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -783,6 +783,9 @@ async Task ProcessBridgeCommand(BridgeParameters parameters, Can break; case BridgeCommandType.Startup: apiValidationStatus = ApiValidationStatus.BadValidationRequest; + + // This business is is cancelled until this BYOND bug is resolved: https://www.byond.com/forum/post/2894866 +#if FALSE if (apiValidationSession) { var proceedTcs = new TaskCompletionSource(); @@ -792,6 +795,7 @@ async Task ProcessBridgeCommand(BridgeParameters parameters, Can if (!firstValidationRequest) return BridgeError("Startup bridge request was repeated!"); } +#endif if (parameters.Version == null) return BridgeError("Missing dmApiVersion field!"); From 720c42c522cb646fed39cff6d4f61a220bad8ca6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 11:44:07 -0400 Subject: [PATCH 041/138] Test that GameStaticFiles work in live tests --- tests/DMAPI/LongRunning/Test.dm | 17 ++++++ .../LongRunning/long_running_test_rooted.dme | 2 + .../Live/Instance/ConfigurationTest.cs | 38 +++++++++--- .../Live/Instance/WatchdogTest.cs | 61 ++++++++++++++++++- 4 files changed, 108 insertions(+), 10 deletions(-) diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index 657fadbfd07..f605c5ad598 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -31,6 +31,23 @@ if(!fexists("[DME_NAME].rsc")) FailTest("Failed to create .rsc!") +#ifdef RUN_STATIC_FILE_TESTS + if(params["expect_static_files"]) + if(!fexists("test2.txt")) + FailTest("Missing test2.txt") + + var/f2content = file2text("test2.txt") + if(f2content != "bbb") + FailTest("Unexpected test2.txt content: [f2content]") + + if(!fexists("data/test.txt")) + FailTest("Missing data/test.txt") + + var/f1content = file2text("data/test.txt") + if(f1content != "aaa") + FailTest("Unexpected data/test.txt content: [f1content]") +#endif + StartAsync() /proc/dab() diff --git a/tests/DMAPI/LongRunning/long_running_test_rooted.dme b/tests/DMAPI/LongRunning/long_running_test_rooted.dme index d8e9d61c967..a2ff783588e 100644 --- a/tests/DMAPI/LongRunning/long_running_test_rooted.dme +++ b/tests/DMAPI/LongRunning/long_running_test_rooted.dme @@ -11,6 +11,8 @@ // END_PREFERENCES // BEGIN_INCLUDE +#define RUN_STATIC_FILE_TESTS +#define DME_NAME "long_running_test_rooted" #include "tests/DMAPI/LongRunning/Config.dm" #include "tests/DMAPI/test_prelude.dm" #include "tests/DMAPI/LongRunning/Test.dm" diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index 2dea3e5b124..44c2f7246be 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -83,19 +83,40 @@ await configurationClient.Write(new ConfigurationFileRequest var path = Path.Combine(instance.Path, "Configuration", tmp); Assert.IsFalse(Directory.Exists(path)); - // leave a directory there to test the deployment process - var staticDir = new ConfigurationFileRequest - { - Path = "/GameStaticFiles/data" - }; - - await configurationClient.CreateDirectory(staticDir, cancellationToken); } public Task SetupDMApiTests(bool includingRoot, CancellationToken cancellationToken) { // just use an I/O manager here var ioManager = new DefaultIOManager(); + + async Task TestStaticFileAndDir() + { + // leave a file there to test the deployment process + var staticDir = new ConfigurationFileRequest + { + Path = "/GameStaticFiles/data" + }; + + await configurationClient.CreateDirectory(staticDir, cancellationToken); + + var staticFile = new ConfigurationFileRequest + { + Path = "/GameStaticFiles/data/test.txt" + }; + + await using var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes("aaa")); + await configurationClient.Write(staticFile, memoryStream, cancellationToken); + + var staticFile2 = new ConfigurationFileRequest + { + Path = "/GameStaticFiles/test2.txt" + }; + + await using var memoryStream2 = new MemoryStream(Encoding.UTF8.GetBytes("bbb")); + await configurationClient.Write(staticFile2, memoryStream2, cancellationToken); + } + return Task.WhenAll( ioManager.CopyDirectory( Enumerable.Empty(), @@ -110,6 +131,9 @@ public Task SetupDMApiTests(bool includingRoot, CancellationToken cancellationTo ioManager.ConcatPath(instance.Path, "Repository", "long_running_test_rooted.dme"), cancellationToken) : Task.CompletedTask, + includingRoot + ? TestStaticFileAndDir() + : Task.CompletedTask, ioManager.CopyDirectory( Enumerable.Empty(), null, diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 784fce5f67c..35bd51fd110 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -114,7 +114,7 @@ await Task.WhenAll( Port = ddPort, MapThreads = 2, LogOutput = false, - AdditionalParameters = "expect_chat_channels=1" + AdditionalParameters = "expect_chat_channels=1&expect_static_files=1" }, cancellationToken), CheckByondVersions(), ApiAssert.ThrowsException(() => instanceClient.DreamDaemon.Update(new DreamDaemonRequest @@ -157,8 +157,64 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest System.Console.WriteLine($"TEST: END WATCHDOG TESTS {instanceClient.Metadata.Name}"); } + async ValueTask RegressionTest1686(CancellationToken cancellationToken) + { + async ValueTask RunTest(bool useTrusted) + { + System.Console.WriteLine($"TEST: RegressionTest1686 {useTrusted}..."); + var ddUpdateTask = instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + SecurityLevel = useTrusted ? DreamDaemonSecurity.Trusted : DreamDaemonSecurity.Safe, + AdditionalParameters = "expect_chat_channels=1&expect_static_files=1", + }, cancellationToken); + var currentStatus = await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); + await ddUpdateTask; + + Assert.AreEqual(WatchdogStatus.Offline, currentStatus.Status); + + var startJob = await StartDD(cancellationToken); + + await WaitForJob(startJob, 40, false, null, cancellationToken); + + currentStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + SoftShutdown = true, + }, cancellationToken); + + Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); + + // reimplement TellWorldToReboot because it expects a new deployment and we don't care + System.Console.WriteLine("TEST: Hack world reboot topic..."); + var result = await topicClient.SendTopic(IPAddress.Loopback, "tgs_integration_test_special_tactics=1", ddPort, cancellationToken); + Assert.AreEqual("ack", result.StringData); + + using var tempCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var tempToken = tempCts.Token; + using (tempToken.Register(() => System.Console.WriteLine("TEST ERROR: Timeout in RegressionTest1686!"))) + { + tempCts.CancelAfter(TimeSpan.FromMinutes(2)); + + do + { + await Task.Delay(TimeSpan.FromSeconds(1), tempToken); + currentStatus = await instanceClient.DreamDaemon.Read(tempToken); + } + while (currentStatus.Status != WatchdogStatus.Offline); + } + + await CheckDMApiFail(currentStatus.ActiveCompileJob, cancellationToken); + } + + await RunTest(true); + + if (new PlatformIdentifier().IsWindows || !usingBasicWatchdog) + await RunTest(false); + } + async Task InteropTestsForLongRunningDme(CancellationToken cancellationToken) { + await RegressionTest1686(cancellationToken); + await StartAndLeaveRunning(cancellationToken); await RegressionTest1550(cancellationToken); @@ -191,8 +247,7 @@ async Task InteropTestsForLongRunningDme(CancellationToken cancellationToken) async ValueTask RegressionTest1550(CancellationToken cancellationToken) { // we need to cycle deployments twice because TGS holds the initial deployment - await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); - var currentStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); Assert.IsNotNull(currentStatus.StagedCompileJob); From 5f663b419eb4e671fff52909a53683ce826fdcef Mon Sep 17 00:00:00 2001 From: Hawk Date: Sun, 22 Oct 2023 19:02:37 +0100 Subject: [PATCH 042/138] Update Caddy Instructions Provide Caddy 2 Caddyfile. Update Caddy Documenation Link. Clarify PublicPath settings. --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ab7a2556b7d..dcceecf82ba 100644 --- a/README.md +++ b/README.md @@ -401,14 +401,15 @@ Once complete, test that your configuration worked by visiting your proxy site f #### Caddy (Reccommended for Linux, or those unfamilar with configuring NGINX or Apache) 1. Setup a basic website configuration. Instructions on how to do so are out of scope. -2. In your Caddyfile, under a server entry, add the following (replace 8080 with the port TGS is hosted on): +2. In your Caddyfile, under a server entry, add the following (replace 5000 with the port TGS is hosted on): ``` -proxy /tgs localhost:8080 { - transparent +https://your.site.here { + reverse_proxy localhost:5000 } ``` +3. For this setup, your PublicPath needs to be blank. If you have a path in PublicPath, it needs to be in "reverse_proxy PublicPathHere localhost:5000". -See https://caddyserver.com/docs/proxy +See https://caddyserver.com/docs/caddyfile/directives/reverse_proxy #### NGINX (Reccommended for Linux) From 9d674618a90e7ca5deda34039fd48a13db7538e7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 14:09:42 -0400 Subject: [PATCH 043/138] Fix `HardLinkDmbProvider` not mirroring `GameStaticFiles` and other symlinks Also update readme about conditional requirements for GameStaticFiles to function Fixes #1686 --- README.md | 22 ++++++- .../Deployment/HardLinkDmbProvider.cs | 65 ++++++++++++++++--- .../Components/Watchdog/PosixWatchdog.cs | 1 + 3 files changed, 78 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ab7a2556b7d..b7636f5c855 100644 --- a/README.md +++ b/README.md @@ -570,7 +570,26 @@ This folder can contain anything. But, when certain events occur in the instance #### GameStaticFiles -Any files and folders contained in this root level of this folder will be symbolically linked to all deployments at the time they are created. This allows persistent game data (BYOND `.sav`s or code configuration files for example) to persist across all deployments. This folder contains a .tgsignore file which can be used to prevent symlinks from being generated by entering the names of files and folders (1 per line) +Any files and folders contained in this root level of this folder will be symbolically linked to all deployments at the time they are created. This allows persistent game data (BYOND `.sav`s or code configuration files for example) to persist across all deployments. This folder contains a .tgsignore file which can be used to prevent symlinks from being generated by entering the names of files and folders (1 per line). + +This functionality has the following prerequisites: + +- You are using Windows. + +**OR** + +- Your world uses the TGS DreamMaker API. +- Your world runs with the `Trusted` security level. + +**OR** + +- You are NOT using the basic watchdog. +- The contents of the `GameStaticFiles` directory are on the same filesystem as the instance's `Game` directory. + +**OR** + +- You are using the basic watchdog. +- Your world runs with the `Trusted` security level. ### Clients @@ -614,4 +633,3 @@ Feel free to ask for help [on the discussions page](https://github.com/tgstation * The remainder of the project is licensed under [GNU AGPL v3](http://www.gnu.org/licenses/agpl-3.0.html) See the files in the `/src/DMAPI` tree for the MIT license - diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs index cd9a3f1185a..263333f6bb0 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -3,11 +3,13 @@ using System.Diagnostics; using System.Globalization; using System.IO; +using System.Runtime.Versioning; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; @@ -18,6 +20,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// A that uses hard links. /// + [UnsupportedOSPlatform("windows")] sealed class HardLinkDmbProvider : SwappableDmbProvider { /// @@ -96,6 +99,7 @@ public override Task FinishActivationPreparation(CancellationToken cancellationT /// protected override async Task DoSwap(CancellationToken cancellationToken) { + logger.LogTrace("Begin DoSwap, mirroring task complete: {complete}...", mirroringTask.IsCompleted); var mirroredDir = await mirroringTask.WaitAsync(cancellationToken); var goAheadTcs = new TaskCompletionSource(); @@ -103,12 +107,15 @@ protected override async Task DoSwap(CancellationToken cancellationToken) async void DisposeOfOldDirectory() { var directoryMoved = false; - var disposePath = Guid.NewGuid().ToString(); + var disposeGuid = Guid.NewGuid(); + var disposePath = disposeGuid.ToString(); + logger.LogTrace("Moving Live directory to {path} for deletion...", disposeGuid); try { await IOManager.MoveDirectory(LiveGameDirectory, disposePath, cancellationToken); directoryMoved = true; goAheadTcs.SetResult(); + logger.LogTrace("Deleting old Live directory {path}...", disposePath); await IOManager.DeleteDirectory(disposePath, CancellationToken.None); // DCT: We're detached at this point } catch (DirectoryNotFoundException ex) @@ -127,7 +134,9 @@ async void DisposeOfOldDirectory() DisposeOfOldDirectory(); await goAheadTcs.Task; + logger.LogTrace("Moving mirror directory {path} to Live...", mirroredDir); await IOManager.MoveDirectory(mirroredDir, LiveGameDirectory, cancellationToken); + logger.LogTrace("Swap complete!"); } /// @@ -173,17 +182,40 @@ IEnumerable MirrorDirectoryImpl(string src, string dest, SemaphoreSlim sem { var dir = new DirectoryInfo(src); Task subdirCreationTask = null; + var dreamDaemonWillAcceptOutOfDirectorySymlinks = CompileJob.MinimumSecurityLevel == DreamDaemonSecurity.Trusted; foreach (var subDirectory in dir.EnumerateDirectories()) { + var mirroredName = Path.Combine(dest, subDirectory.Name); + // check if we are a symbolic link - if (!subDirectory.Attributes.HasFlag(FileAttributes.Directory) || subDirectory.Attributes.HasFlag(FileAttributes.ReparsePoint)) - { - logger.LogTrace("Skipping symlink to {subdir}", subDirectory.Name); - continue; - } + if (subDirectory.Attributes.HasFlag(FileAttributes.ReparsePoint)) + if (dreamDaemonWillAcceptOutOfDirectorySymlinks) + { + var target = subDirectory.ResolveLinkTarget(false); + logger.LogDebug("Recreating directory {name} as symlink to {target}", subDirectory.Name, target); + if (subdirCreationTask == null) + { + subdirCreationTask = IOManager.CreateDirectory(dest, cancellationToken); + yield return subdirCreationTask; + } + + async Task CopyLink() + { + await subdirCreationTask.WaitAsync(cancellationToken); + using var lockContext = semaphore != null + ? await SemaphoreSlimContext.Lock(semaphore, cancellationToken) + : null; + await LinkFactory.CreateSymbolicLink(target.FullName, mirroredName, cancellationToken); + } + + yield return CopyLink(); + continue; + } + else + logger.LogDebug("Recreating symlinked directory {name} as hard links...", subDirectory.Name); var checkingSubdirCreationTask = true; - foreach (var copyTask in MirrorDirectoryImpl(subDirectory.FullName, Path.Combine(dest, subDirectory.Name), semaphore, cancellationToken)) + foreach (var copyTask in MirrorDirectoryImpl(subDirectory.FullName, mirroredName, semaphore, cancellationToken)) { if (subdirCreationTask == null) { @@ -214,7 +246,24 @@ async Task LinkThisFile() using var lockContext = semaphore != null ? await SemaphoreSlimContext.Lock(semaphore, cancellationToken) : null; - await LinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken); + + if (fileInfo.Attributes.HasFlag(FileAttributes.ReparsePoint)) + { + // AHHHHHHHHHHHHH + var target = fileInfo.ResolveLinkTarget(!dreamDaemonWillAcceptOutOfDirectorySymlinks); + if (dreamDaemonWillAcceptOutOfDirectorySymlinks) + { + logger.LogDebug("Recreating symlinked file {name} as symlink to {target}", fileInfo.Name, target.FullName); + await LinkFactory.CreateSymbolicLink(target.FullName, destFile, cancellationToken); + } + else + { + logger.LogDebug("Recreating symlinked file {name} as hard link to {target}", fileInfo.Name, target.FullName); + await LinkFactory.CreateHardLink(target.FullName, destFile, cancellationToken); + } + } + else + await LinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken); } yield return LinkThisFile(); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index eb563bd1fa0..2fb5ad1d544 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Versioning; using System.Threading; using System.Threading.Tasks; From 50072ae2227b48b7f0f638378e4087df765e01df Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 14:20:24 -0400 Subject: [PATCH 044/138] Run milestone check on labeling --- .github/workflows/check-pr-has-milestone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-pr-has-milestone.yml b/.github/workflows/check-pr-has-milestone.yml index 954d539927a..7f6ed97d2d7 100644 --- a/.github/workflows/check-pr-has-milestone.yml +++ b/.github/workflows/check-pr-has-milestone.yml @@ -2,7 +2,7 @@ name: "Check PR Has Milestone" on: pull_request: - types: [ opened, edited, synchronize, reopened ] + types: [ opened, edited, synchronize, reopened, labeled ] branches: - dev - master From cc0b46205f0bf0fb663eabaf8fe74e61883bce60 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 14:28:04 -0400 Subject: [PATCH 045/138] Fix warnings with OS restricted classes --- .../Components/Deployment/SymlinkDmbProvider.cs | 4 +++- .../Components/Watchdog/PosixWatchdog.cs | 1 + .../Components/Watchdog/PosixWatchdogFactory.cs | 2 ++ .../Components/Watchdog/WindowsWatchdog.cs | 4 +++- .../Components/Watchdog/WindowsWatchdogFactory.cs | 2 ++ 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs index 2edc58d37eb..0054e7fa4b5 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs @@ -1,4 +1,5 @@ -using System.Threading; +using System.Runtime.Versioning; +using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.IO; @@ -8,6 +9,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// A that uses symlinks. /// + [SupportedOSPlatform("windows")] sealed class SymlinkDmbProvider : SwappableDmbProvider { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index 2fb5ad1d544..818dea9d7c4 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -22,6 +22,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// A variant of the that works on POSIX systems. /// + [UnsupportedOSPlatform("windows")] sealed class PosixWatchdog : AdvancedWatchdog { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs index 64c86722625..1d0d140bc05 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Versioning; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -20,6 +21,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// for creating s. /// + [UnsupportedOSPlatform("windows")] sealed class PosixWatchdogFactory : WindowsWatchdogFactory { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index 05c1f100888..8ff6bf33624 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -1,4 +1,5 @@ -using System.Threading; +using System.Runtime.Versioning; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -19,6 +20,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// A variant of the that works on Windows systems. /// + [SupportedOSPlatform("windows")] sealed class WindowsWatchdog : AdvancedWatchdog { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index 89ed6b145d7..0888294aa41 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Versioning; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -20,6 +21,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// for creating s. /// + [SupportedOSPlatform("windows")] class WindowsWatchdogFactory : WatchdogFactory { /// From ad2a5d2b9b131575a67ec2b709ec98fcd44f89c6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 14:30:13 -0400 Subject: [PATCH 046/138] Fix release build warnings --- .../Components/Session/SessionController.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index b33d3628a1e..1ee53dfb57c 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -795,6 +795,8 @@ async Task ProcessBridgeCommand(BridgeParameters parameters, Can if (!firstValidationRequest) return BridgeError("Startup bridge request was repeated!"); } +#else + postValidationShutdownTask = Task.CompletedTask; #endif if (parameters.Version == null) From a97001231ff6b1606b54adc16f4ee19a01e04069 Mon Sep 17 00:00:00 2001 From: Hawk-v3 Date: Sun, 22 Oct 2023 19:32:55 +0100 Subject: [PATCH 047/138] Update README.md Accept correction for definition of PublicPath Co-authored-by: Jordan Dominion --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dcceecf82ba..64221f5028b 100644 --- a/README.md +++ b/README.md @@ -407,7 +407,7 @@ https://your.site.here { reverse_proxy localhost:5000 } ``` -3. For this setup, your PublicPath needs to be blank. If you have a path in PublicPath, it needs to be in "reverse_proxy PublicPathHere localhost:5000". +3. For this setup, your configuration's `ControlPanel:PublicPath` needs to be blank. If you have a path in `PublicPath`, it needs to be in "reverse_proxy PublicPathHere localhost:5000". See https://caddyserver.com/docs/caddyfile/directives/reverse_proxy From 5afc74c4872ca4c49fe7990c316fecd4b0f46603 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 16:01:52 -0400 Subject: [PATCH 048/138] Fix more Release build warnings Not being able to build locally in release mode because of the .NET 8 RC SDK is painful --- .../Components/Deployment/SymlinkDmbProvider.cs | 4 +--- .../Components/Watchdog/WindowsWatchdog.cs | 4 +--- .../Components/Watchdog/WindowsWatchdogFactory.cs | 2 -- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs index 0054e7fa4b5..2edc58d37eb 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs @@ -1,5 +1,4 @@ -using System.Runtime.Versioning; -using System.Threading; +using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.IO; @@ -9,7 +8,6 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// A that uses symlinks. /// - [SupportedOSPlatform("windows")] sealed class SymlinkDmbProvider : SwappableDmbProvider { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index 8ff6bf33624..05c1f100888 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -1,5 +1,4 @@ -using System.Runtime.Versioning; -using System.Threading; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -20,7 +19,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// A variant of the that works on Windows systems. /// - [SupportedOSPlatform("windows")] sealed class WindowsWatchdog : AdvancedWatchdog { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index 0888294aa41..89ed6b145d7 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -1,5 +1,4 @@ using System; -using System.Runtime.Versioning; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -21,7 +20,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// for creating s. /// - [SupportedOSPlatform("windows")] class WindowsWatchdogFactory : WatchdogFactory { /// From 4c439cb2dc67a65dfbb603e70aadeb19124ad1f4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 22:54:54 -0400 Subject: [PATCH 049/138] Switch to using modern endpoint routing --- src/Tgstation.Server.Host/Core/Application.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 18abdcacf30..811c77d7080 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -247,7 +247,6 @@ public void ConfigureServices( services .AddMvc(options => { - options.EnableEndpointRouting = false; options.ReturnHttpNotAcceptable = true; options.RespectBrowserAcceptHeader = true; }) @@ -517,14 +516,24 @@ public void Configure( // Do not cache a single thing beyond this point, it's all API applicationBuilder.UseDisabledClientCache(); + // Stack overflow said this needs to go here and removing it breaks things: https://stackoverflow.com/questions/73736879/invalidoperationexception-endpointroutingmiddleware-matches-endpoints-setup-by + applicationBuilder.UseRouting(); + // authenticate JWT tokens using our security pipeline if present, returns 401 if bad applicationBuilder.UseAuthentication(); + // enable authorization on endpoints + applicationBuilder.UseAuthorization(); + // suppress and log database exceptions applicationBuilder.UseDbConflictHandling(); - // majority of handling is done in the controllers - applicationBuilder.UseMvc(); + // setup endpoints + applicationBuilder.UseEndpoints(endpoints => + { + // majority of handling is done in the controllers + endpoints.MapControllers(); + }); // 404 anything that gets this far // End of request pipeline setup From af2570ef790f666c2b384d234987e941a012bcbc Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 20:03:57 -0400 Subject: [PATCH 050/138] Move `TgsAuthorizeAttribute` from `Controllers` namespace to `Security` --- src/Tgstation.Server.Host/Security/IClaimsInjector.cs | 2 +- .../{Controllers => Security}/TgsAuthorizeAttribute.cs | 2 +- src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) rename src/Tgstation.Server.Host/{Controllers => Security}/TgsAuthorizeAttribute.cs (98%) diff --git a/src/Tgstation.Server.Host/Security/IClaimsInjector.cs b/src/Tgstation.Server.Host/Security/IClaimsInjector.cs index 52dbebe6af0..94899d81ffb 100644 --- a/src/Tgstation.Server.Host/Security/IClaimsInjector.cs +++ b/src/Tgstation.Server.Host/Security/IClaimsInjector.cs @@ -6,7 +6,7 @@ namespace Tgstation.Server.Host.Security { /// - /// For injecting s that can look for. + /// For injecting s that can look for. /// interface IClaimsInjector { diff --git a/src/Tgstation.Server.Host/Controllers/TgsAuthorizeAttribute.cs b/src/Tgstation.Server.Host/Security/TgsAuthorizeAttribute.cs similarity index 98% rename from src/Tgstation.Server.Host/Controllers/TgsAuthorizeAttribute.cs rename to src/Tgstation.Server.Host/Security/TgsAuthorizeAttribute.cs index a16cf7646dd..2f6b62a848b 100644 --- a/src/Tgstation.Server.Host/Controllers/TgsAuthorizeAttribute.cs +++ b/src/Tgstation.Server.Host/Security/TgsAuthorizeAttribute.cs @@ -4,7 +4,7 @@ using Tgstation.Server.Api.Rights; -namespace Tgstation.Server.Host.Controllers +namespace Tgstation.Server.Host.Security { /// /// Helper for using the with the system. diff --git a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs index 3bd803b3623..89f77167329 100644 --- a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs +++ b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs @@ -18,6 +18,7 @@ using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Common.Extensions; using Tgstation.Server.Host.Controllers; +using Tgstation.Server.Host.Security; namespace Tgstation.Server.Host.Utils { From 9b405784cc4555e4eed90b72f4f19c939ac23a24 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 19:51:26 -0400 Subject: [PATCH 051/138] Fix a race condition in compat tests --- .../Tgstation.Server.Tests/Live/Instance/InstanceTest.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 657849d25f8..69e6b00802c 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -165,16 +165,19 @@ public async Task RunCompatTests( await chatRequest; await Task.Yield(); - var jobs = await instanceClient.Jobs.List(null, cancellationToken); - var theJobWeWant = jobs.First(x => x.Description.Contains("Reconnect chat bot")); await Task.WhenAll( jrt.WaitForJob(installJob2.InstallJob, 60, false, null, cancellationToken), jrt.WaitForJob(cloneRequest.Result.ActiveJob, 60, false, null, cancellationToken), - jrt.WaitForJob(theJobWeWant, 30, false, null, cancellationToken), dmUpdateRequest.AsTask(), cloneRequest.AsTask()); + var jobs = await instanceClient.Jobs.List(null, cancellationToken); + var theJobWeWant = jobs + .OrderByDescending(x => x.StartedAt) + .First(x => x.Description.Contains("Reconnect chat bot")); + await jrt.WaitForJob(theJobWeWant, 30, false, null, cancellationToken); + var configSetupTask = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata).SetupDMApiTests(true, cancellationToken); if (TestingUtils.RunningInGitHubActions From d2f6dfde8beb95517bbe64fcd860d6adabece366 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 24 Oct 2023 02:43:09 -0400 Subject: [PATCH 052/138] Patch DMAPI to work around del(world) behavior It doesn't actually stop the world until the end of the tick. --- build/Version.props | 2 +- src/DMAPI/tgs.dm | 2 +- src/DMAPI/tgs/core/datum.dm | 4 ++++ src/DMAPI/tgs/v4/api.dm | 10 +++++----- src/DMAPI/tgs/v5/api.dm | 2 +- tests/DMAPI/test_setup.dm | 1 + 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/build/Version.props b/build/Version.props index 37e57bc4ff3..5acebbddfba 100644 --- a/build/Version.props +++ b/build/Version.props @@ -9,7 +9,7 @@ 7.0.0 11.1.2 13.0.0 - 6.6.0 + 6.6.1 5.6.2 1.4.0 1.2.1 diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index 9825cd118b6..d468d604419 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,6 +1,6 @@ // tgstation-server DMAPI -#define TGS_DMAPI_VERSION "6.6.0" +#define TGS_DMAPI_VERSION "6.6.1" // All functions and datums outside this document are subject to change with any version and should not be relied on. diff --git a/src/DMAPI/tgs/core/datum.dm b/src/DMAPI/tgs/core/datum.dm index 93377079aa7..de420a2a325 100644 --- a/src/DMAPI/tgs/core/datum.dm +++ b/src/DMAPI/tgs/core/datum.dm @@ -11,6 +11,10 @@ TGS_DEFINE_AND_SET_GLOBAL(tgs, null) src.event_handler = event_handler src.version = version +/datum/tgs_api/proc/TerminateWorld() + del(world) + sleep(1) // https://www.byond.com/forum/post/2894866 + /datum/tgs_api/latest parent_type = /datum/tgs_api/v5 diff --git a/src/DMAPI/tgs/v4/api.dm b/src/DMAPI/tgs/v4/api.dm index b9a75c4abb4..945e2e41176 100644 --- a/src/DMAPI/tgs/v4/api.dm +++ b/src/DMAPI/tgs/v4/api.dm @@ -73,7 +73,7 @@ if(cached_json["apiValidateOnly"]) TGS_INFO_LOG("Validating API and exiting...") Export(TGS4_COMM_VALIDATE, list(TGS4_PARAMETER_DATA = "[minimum_required_security_level]")) - del(world) + TerminateWorld() security_level = cached_json["securityLevel"] chat_channels_json_path = cached_json["chatChannelsJson"] @@ -188,7 +188,7 @@ requesting_new_port = TRUE if(!world.OpenPort(0)) //open any port TGS_ERROR_LOG("Unable to open random port to retrieve new port![TGS4_PORT_CRITFAIL_MESSAGE]") - del(world) + TerminateWorld() //request a new port export_lock = FALSE @@ -196,16 +196,16 @@ if(!new_port_json) TGS_ERROR_LOG("No new port response from server![TGS4_PORT_CRITFAIL_MESSAGE]") - del(world) + TerminateWorld() var/new_port = new_port_json[TGS4_PARAMETER_DATA] if(!isnum(new_port) || new_port <= 0) TGS_ERROR_LOG("Malformed new port json ([json_encode(new_port_json)])![TGS4_PORT_CRITFAIL_MESSAGE]") - del(world) + TerminateWorld() if(new_port != world.port && !world.OpenPort(new_port)) TGS_ERROR_LOG("Unable to open port [new_port]![TGS4_PORT_CRITFAIL_MESSAGE]") - del(world) + TerminateWorld() requesting_new_port = FALSE while(export_lock) diff --git a/src/DMAPI/tgs/v5/api.dm b/src/DMAPI/tgs/v5/api.dm index 45250efc462..7226f29bba6 100644 --- a/src/DMAPI/tgs/v5/api.dm +++ b/src/DMAPI/tgs/v5/api.dm @@ -51,7 +51,7 @@ if(runtime_information[DMAPI5_RUNTIME_INFORMATION_API_VALIDATE_ONLY]) TGS_INFO_LOG("DMAPI validation, exiting...") - del(world) + TerminateWorld() version = new /datum/tgs_version(runtime_information[DMAPI5_RUNTIME_INFORMATION_SERVER_VERSION]) security_level = runtime_information[DMAPI5_RUNTIME_INFORMATION_SECURITY_LEVEL] diff --git a/tests/DMAPI/test_setup.dm b/tests/DMAPI/test_setup.dm index 250f80fe545..56f4c03eb87 100644 --- a/tests/DMAPI/test_setup.dm +++ b/tests/DMAPI/test_setup.dm @@ -32,3 +32,4 @@ text2file(reason, "test_fail_reason.txt") world.log << "Terminating..." del(world) + sleep(1) // https://www.byond.com/forum/post/2894866 From 9c6b9bba168733dbde1a37f5f8862022a61ab4d3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 24 Oct 2023 02:45:11 -0400 Subject: [PATCH 053/138] Re-enable post validation shutdown timeout --- .../Components/Session/SessionController.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 78160b352da..01ddcea2d41 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -784,8 +784,6 @@ async ValueTask ProcessBridgeCommand(BridgeParameters parameters case BridgeCommandType.Startup: apiValidationStatus = ApiValidationStatus.BadValidationRequest; - // This business is is cancelled until this BYOND bug is resolved: https://www.byond.com/forum/post/2894866 -#if FALSE if (apiValidationSession) { var proceedTcs = new TaskCompletionSource(); @@ -795,9 +793,6 @@ async ValueTask ProcessBridgeCommand(BridgeParameters parameters if (!firstValidationRequest) return BridgeError("Startup bridge request was repeated!"); } -#else - postValidationShutdownTask = Task.CompletedTask; -#endif if (parameters.Version == null) return BridgeError("Missing dmApiVersion field!"); From b21f3a880fa511c3a122f7a580d8ac2240cc17c2 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 24 Oct 2023 04:03:56 -0400 Subject: [PATCH 054/138] Use a higher priority thread here to ensure the dump test runs successfully --- .../Live/Instance/WatchdogTest.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 35bd51fd110..5d0cd5fa6f2 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -394,13 +394,17 @@ async Task DumpTests(CancellationToken cancellationToken) KillDD(true); var jobTcs = new TaskCompletionSource(); var killTaskStarted = new TaskCompletionSource(); - var killTask = Task.Run(() => + var killThread = new Thread(() => { killTaskStarted.SetResult(); while (!jobTcs.Task.IsCompleted) KillDD(false); - }, cancellationToken); + }) + { + Priority = ThreadPriority.AboveNormal + }; + killThread.Start(); try { await killTaskStarted.Task; @@ -410,7 +414,7 @@ async Task DumpTests(CancellationToken cancellationToken) finally { jobTcs.SetResult(); - await killTask; + killThread.Join(); } // these can also happen From 13128abe73125ecf8501a368164a146e8e67a2d1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 25 Oct 2023 02:54:50 -0400 Subject: [PATCH 055/138] Fix overeager required process killing --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 5d0cd5fa6f2..eb9c77047b9 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -1137,9 +1137,11 @@ public async Task StartAndLeaveRunning(CancellationToken cancellationToken) Assert.AreEqual(ddPort, daemonStatus.CurrentPort); // Try killing the DD process to ensure it gets set to the restoring state + bool firstTime = true; do { - KillDD(true); + KillDD(firstTime); + firstTime = false; await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); } From 14ed93c12a7894b36f90bbcf51c751033ee2761d Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 25 Oct 2023 03:33:27 -0400 Subject: [PATCH 056/138] Make every LongRunning DMAPI test compile generate a unique .rsc --- .../PreCompile-GenerateRandomResource.bat | 4 ++++ .../PreCompile-GenerateRandomResource.sh | 8 ++++++++ tests/DMAPI/LongRunning/Test.dm | 10 ++++++---- tests/DMAPI/LongRunning/resource.txt | 8 -------- .../Live/Instance/ConfigurationTest.cs | 14 ++++++++++++++ tgstation-server.sln | 3 ++- 6 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 tests/DMAPI/LongRunning/PreCompile-GenerateRandomResource.bat create mode 100644 tests/DMAPI/LongRunning/PreCompile-GenerateRandomResource.sh delete mode 100644 tests/DMAPI/LongRunning/resource.txt diff --git a/tests/DMAPI/LongRunning/PreCompile-GenerateRandomResource.bat b/tests/DMAPI/LongRunning/PreCompile-GenerateRandomResource.bat new file mode 100644 index 00000000000..9dc4ff99758 --- /dev/null +++ b/tests/DMAPI/LongRunning/PreCompile-GenerateRandomResource.bat @@ -0,0 +1,4 @@ +cd /D "%~dp0" +cd %1 +cd tests\DMAPI\LongRunning +C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe -ExecutionPolicy Bypass -Command "(New-Guid).Guid | Out-File resource.txt; Add-Content -Path resource.txt -Value 'aljsdhfjahsfkjnsalkjdfhskljdackmcnvxkljhvkjsdanv,jdshlkufhklasjeFDhfjkalhdkjlfhalksfdjh'" diff --git a/tests/DMAPI/LongRunning/PreCompile-GenerateRandomResource.sh b/tests/DMAPI/LongRunning/PreCompile-GenerateRandomResource.sh new file mode 100644 index 00000000000..a22163f76ff --- /dev/null +++ b/tests/DMAPI/LongRunning/PreCompile-GenerateRandomResource.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +cd "$1/tests/DMAPI/LongRunning" + +tr -dc A-Za-z0-9 resource.txt +echo -e '\naljsdhfjahsfkjnsalkjdfhskljdackmcnvxkljhvkjsdanv,jdshlkufhklasjeFDhfjkalhdkjlfhalksfdjh\n' >> resource.txt diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index f605c5ad598..7c6ba1674d9 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -22,11 +22,13 @@ if(!length(channels)) FailTest("Expected some chat channels!") - var/test_str = "aljsdhfjahsfkjnsalkjdfhskljdackmcnvxkljhvkjsdanv,jdshlkufhklasjeFDhfjkalhdkjlfhalksfdjh" - var/res_contents = file2text('resource.txt') // we need a .rsc to be generated + var/res = file('resource.txt') + if(!res) + FailTest("Failed to resource!") - if(!findtext(res_contents, test_str)) - FailTest("Failed to resource? Did not find magic: [res_contents]") + var/res_contents = file2text(res) // we need a .rsc to be generated + if(!res_contents) + FailTest("Failed to resource? No contents!") if(!fexists("[DME_NAME].rsc")) FailTest("Failed to create .rsc!") diff --git a/tests/DMAPI/LongRunning/resource.txt b/tests/DMAPI/LongRunning/resource.txt deleted file mode 100644 index 94b848fe2ac..00000000000 --- a/tests/DMAPI/LongRunning/resource.txt +++ /dev/null @@ -1,8 +0,0 @@ - - if(!length(channels)) - FailTest("Expected some chat channels!") - - var/test_str = "aljsdhfjahsfkjnsalkjdfhskljdackmcnvxkljhvkjsdanv,jdshlkufhklasjeFDhfjkalhdkjlfhalksfdjh" - var/self_contents = file2text(file('Test.dm')) // we need a .rsc to be generated - - if(!(test_str in self_contents)) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index 44c2f7246be..2c7295e224a 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -12,6 +12,7 @@ using Tgstation.Server.Client; using Tgstation.Server.Client.Components; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.System; namespace Tgstation.Server.Tests.Live.Instance { @@ -115,6 +116,19 @@ async Task TestStaticFileAndDir() await using var memoryStream2 = new MemoryStream(Encoding.UTF8.GetBytes("bbb")); await configurationClient.Write(staticFile2, memoryStream2, cancellationToken); + + var shellScriptExtension = new PlatformIdentifier().IsWindows ? ".bat" : ".sh"; + var scriptName = $"PreCompile-GenerateRandomResource{shellScriptExtension}"; + var resourcingScript = new ConfigurationFileRequest + { + Path = $"/EventScripts/{scriptName}" + }; + + await using var readStream = ioManager.GetFileStream($"../../../../DMAPI/LongRunning/{scriptName}", false); + await configurationClient.Write( + resourcingScript, + readStream, + cancellationToken); } return Task.WhenAll( diff --git a/tgstation-server.sln b/tgstation-server.sln index c1cde46821b..36d9027e9d6 100644 --- a/tgstation-server.sln +++ b/tgstation-server.sln @@ -158,7 +158,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LongRunning", "LongRunning" tests\DMAPI\LongRunning\long_running_test.dme = tests\DMAPI\LongRunning\long_running_test.dme tests\DMAPI\LongRunning\long_running_test_copy.dme = tests\DMAPI\LongRunning\long_running_test_copy.dme tests\DMAPI\LongRunning\long_running_test_rooted.dme = tests\DMAPI\LongRunning\long_running_test_rooted.dme - tests\DMAPI\LongRunning\resource.txt = tests\DMAPI\LongRunning\resource.txt + tests\DMAPI\LongRunning\PreCompile-GenerateRandomResource.bat = tests\DMAPI\LongRunning\PreCompile-GenerateRandomResource.bat + tests\DMAPI\LongRunning\PreCompile-GenerateRandomResource.sh = tests\DMAPI\LongRunning\PreCompile-GenerateRandomResource.sh tests\DMAPI\LongRunning\Test.dm = tests\DMAPI\LongRunning\Test.dm EndProjectSection EndProject From d46292bbdb9d1cbb7de087a3d65eff1d62f62e75 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 14:37:40 -0400 Subject: [PATCH 057/138] Nuget package updates --- build/TestCommon.props | 4 ++-- .../Tgstation.Server.Host.csproj | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/build/TestCommon.props b/build/TestCommon.props index 5ee30f11474..5562510f72e 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -15,8 +15,8 @@ - - + + diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index f2844ae82bc..e156db8d188 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -75,19 +75,19 @@ - + - + - + - + runtime; build; native; contentfiles; analyzers; buildtransitive - + - + @@ -117,11 +117,11 @@ - + - + From 30f57d28dafc7a83c69fc100da7ead96797c2643 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 14:43:24 -0400 Subject: [PATCH 058/138] Store `TokenResponse` in `ApiHeaders` - Decode it from JWT if necessary. --- src/Tgstation.Server.Api/ApiHeaders.cs | 81 ++++++++++++++++--- .../Models/Response/TokenResponse.cs | 2 +- .../Tgstation.Server.Api.csproj | 2 + src/Tgstation.Server.Client/ApiClient.cs | 2 +- src/Tgstation.Server.Client/ServerClient.cs | 22 +---- .../ServerClientFactory.cs | 26 ++++-- .../Controllers/HomeController.cs | 4 +- .../TestApiHeaders.cs | 5 +- .../TestApiClient.cs | 24 +++++- 9 files changed, 127 insertions(+), 41 deletions(-) diff --git a/src/Tgstation.Server.Api/ApiHeaders.cs b/src/Tgstation.Server.Api/ApiHeaders.cs index 7fe6edafc33..e9b721a3038 100644 --- a/src/Tgstation.Server.Api/ApiHeaders.cs +++ b/src/Tgstation.Server.Api/ApiHeaders.cs @@ -9,9 +9,14 @@ using Microsoft.AspNetCore.Http.Headers; using Microsoft.Extensions.Primitives; +using Microsoft.IdentityModel.JsonWebTokens; using Microsoft.Net.Http.Headers; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + using Tgstation.Server.Api.Models; +using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Api.Properties; using Tgstation.Server.Common.Extensions; @@ -93,9 +98,9 @@ public sealed class ApiHeaders public Version ApiVersion { get; } /// - /// The client's JWT. + /// The client's . /// - public string? Token { get; } + public TokenResponse? Token { get; } /// /// The client's username. @@ -117,6 +122,11 @@ public sealed class ApiHeaders /// public bool IsTokenAuthentication => Token != null && !OAuthProvider.HasValue; + /// + /// The OAuth code in use. + /// + readonly string? oAuthCode; + /// /// Checks if a given is compatible with our own. /// @@ -129,16 +139,31 @@ public sealed class ApiHeaders /// /// The value of . /// The value of . - /// The value of . - public ApiHeaders(ProductHeaderValue userAgent, string token, OAuthProvider? oauthProvider = null) + public ApiHeaders(ProductHeaderValue userAgent, TokenResponse token) : this(userAgent, token, null, null) { if (userAgent == null) throw new ArgumentNullException(nameof(userAgent)); if (token == null) throw new ArgumentNullException(nameof(token)); + if (token.Bearer == null) + throw new InvalidOperationException("token.Bearer must be set!"); + } + + /// + /// Initializes a new instance of the class. Used for token authentication. + /// + /// The value of . + /// The value of . + /// The value of . + public ApiHeaders(ProductHeaderValue userAgent, string oAuthCode, OAuthProvider oAuthProvider) + : this(userAgent, null, null, null) + { + if (userAgent == null) + throw new ArgumentNullException(nameof(userAgent)); - OAuthProvider = oauthProvider; + this.oAuthCode = oAuthCode ?? throw new ArgumentNullException(nameof(oAuthCode)); + OAuthProvider = oAuthProvider; } /// @@ -244,7 +269,36 @@ void AddError(HeaderTypes headerType, string message) goto case BearerAuthenticationScheme; case BearerAuthenticationScheme: - Token = parameter; + var tokenSplits = parameter.Split('.'); + DateTimeOffset? expiresAt = null; + if (tokenSplits.Length != 3) + AddError(HeaderTypes.Authorization, "Invalid JWT!"); + else + try + { + var bytes = Convert.FromBase64String(tokenSplits[1]); + var json = Encoding.UTF8.GetString(bytes); + var jwt = JsonConvert.DeserializeObject(json); + var nbf = jwt?.Value(JwtRegisteredClaimNames.Nbf); + + if (nbf != null) + if (Int64.TryParse(nbf, out var unixTimeSeconds)) + expiresAt = DateTimeOffset.FromUnixTimeSeconds(unixTimeSeconds); + else + AddError(HeaderTypes.Authorization, "'nbf' in JWT could not be parsed!"); + else + AddError(HeaderTypes.Authorization, "Missing 'nbf' in JWT payload!"); + } + catch + { + AddError(HeaderTypes.Authorization, "Invalid JWT payload!"); + } + + Token = new TokenResponse + { + Bearer = parameter, + ExpiresAt = expiresAt, + }; break; case BasicAuthenticationScheme: string badBasicAuthHeaderMessage = $"Invalid basic {HeaderNames.Authorization} header!"; @@ -292,7 +346,7 @@ void AddError(HeaderTypes headerType, string message) /// The value of . /// The value of . /// The value of . - ApiHeaders(ProductHeaderValue userAgent, string? token, string? username, string? password) + ApiHeaders(ProductHeaderValue userAgent, TokenResponse? token, string? username, string? password) { RawUserAgent = userAgent?.ToString(); Token = token; @@ -322,10 +376,10 @@ public void SetRequestHeaders(HttpRequestHeaders headers, long? instanceId = nul headers.Clear(); headers.Accept.Add(new MediaTypeWithQualityHeaderValue(ApplicationJsonMime)); headers.UserAgent.Add(new ProductInfoHeaderValue(UserAgent)); - headers.Add(ApiVersionHeader, new ProductHeaderValue(AssemblyName.Name, ApiVersion.ToString()).ToString()); + headers.Add(ApiVersionHeader, CreateApiVersionHeader()); if (OAuthProvider.HasValue) { - headers.Authorization = new AuthenticationHeaderValue(OAuthAuthenticationScheme, Token); + headers.Authorization = new AuthenticationHeaderValue(OAuthAuthenticationScheme, Token!.Bearer); headers.Add(OAuthProviderHeader, OAuthProvider.ToString()); } else if (!IsTokenAuthentication) @@ -333,11 +387,18 @@ public void SetRequestHeaders(HttpRequestHeaders headers, long? instanceId = nul BasicAuthenticationScheme, Convert.ToBase64String(Encoding.UTF8.GetBytes($"{Username}:{Password}"))); else - headers.Authorization = new AuthenticationHeaderValue(BearerAuthenticationScheme, Token); + headers.Authorization = new AuthenticationHeaderValue(BearerAuthenticationScheme, Token!.Bearer); instanceId ??= InstanceId; if (instanceId.HasValue) headers.Add(InstanceIdHeader, instanceId.Value.ToString(CultureInfo.InvariantCulture)); } + + /// + /// Create the ified for of the . + /// + /// A representing the . + string CreateApiVersionHeader() + => new ProductHeaderValue(AssemblyName.Name, ApiVersion.ToString()).ToString(); } } diff --git a/src/Tgstation.Server.Api/Models/Response/TokenResponse.cs b/src/Tgstation.Server.Api/Models/Response/TokenResponse.cs index cd143ede8c0..cb24b941a6a 100644 --- a/src/Tgstation.Server.Api/Models/Response/TokenResponse.cs +++ b/src/Tgstation.Server.Api/Models/Response/TokenResponse.cs @@ -15,6 +15,6 @@ public sealed class TokenResponse /// /// When the expires. /// - public DateTimeOffset ExpiresAt { get; set; } + public DateTimeOffset? ExpiresAt { get; set; } } } diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index 582c079b6ed..dfbd15a36d2 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -26,6 +26,8 @@ + + diff --git a/src/Tgstation.Server.Client/ApiClient.cs b/src/Tgstation.Server.Client/ApiClient.cs index a235496592b..cb0bda67dab 100644 --- a/src/Tgstation.Server.Client/ApiClient.cs +++ b/src/Tgstation.Server.Client/ApiClient.cs @@ -411,7 +411,7 @@ async ValueTask RefreshToken(CancellationToken cancellationToken) return true; var token = await RunRequest(Routes.Root, new object(), HttpMethod.Post, null, true, cancellationToken).ConfigureAwait(false); - headers = new ApiHeaders(headers.UserAgent!, token.Bearer!); + headers = new ApiHeaders(headers.UserAgent!, token); } catch (ClientException) { diff --git a/src/Tgstation.Server.Client/ServerClient.cs b/src/Tgstation.Server.Client/ServerClient.cs index 81111d475b2..207a085e289 100644 --- a/src/Tgstation.Server.Client/ServerClient.cs +++ b/src/Tgstation.Server.Client/ServerClient.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; @@ -16,12 +16,8 @@ sealed class ServerClient : IServerClient /// public TokenResponse Token { - get => token; - set - { - token = value ?? throw new InvalidOperationException("Cannot set a null Token!"); - apiClient.Headers = new ApiHeaders(apiClient.Headers.UserAgent!, token.Bearer!); - } + get => apiClient.Headers.Token ?? throw new InvalidOperationException("apiClient.Headers.Token was null!"); + set => apiClient.Headers = new ApiHeaders(apiClient.Headers.UserAgent!, value); } /// @@ -48,23 +44,13 @@ public TimeSpan Timeout /// readonly IApiClient apiClient; - /// - /// Backing field for . - /// - TokenResponse token; - /// /// Initializes a new instance of the class. /// /// The value of . - /// The value of . - public ServerClient(IApiClient apiClient, TokenResponse token) + public ServerClient(IApiClient apiClient) { this.apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); - this.token = token ?? throw new ArgumentNullException(nameof(token)); - - if (Token.Bearer != apiClient.Headers.Token) - throw new ArgumentOutOfRangeException(nameof(token), token, "Provided token does not match apiClient headers!"); Instances = new InstanceManagerClient(apiClient); Users = new UsersClient(apiClient); diff --git a/src/Tgstation.Server.Client/ServerClientFactory.cs b/src/Tgstation.Server.Client/ServerClientFactory.cs index 3dcb6ec67d5..2d464ec0ecc 100644 --- a/src/Tgstation.Server.Client/ServerClientFactory.cs +++ b/src/Tgstation.Server.Client/ServerClientFactory.cs @@ -102,7 +102,15 @@ public IServerClient CreateFromToken(Uri host, TokenResponse token) if (token.Bearer == null) throw new InvalidOperationException("token.Bearer should not be null!"); - return new ServerClient(ApiClientFactory.CreateApiClient(host, new ApiHeaders(productHeaderValue, token.Bearer), null, false), token); + var serverClient = new ServerClient( + ApiClientFactory.CreateApiClient( + host, + new ApiHeaders( + productHeaderValue, + token), + null, + false)); + return serverClient; } /// @@ -112,7 +120,16 @@ public async ValueTask GetServerInformation( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { - using var api = ApiClientFactory.CreateApiClient(host, new ApiHeaders(productHeaderValue, "fake"), null, true); + using var api = ApiClientFactory.CreateApiClient( + host, + new ApiHeaders( + productHeaderValue, + new TokenResponse + { + Bearer = "unused", + }), + null, + true); if (requestLoggers != null) foreach (var requestLogger in requestLoggers) @@ -155,14 +172,13 @@ async ValueTask CreateWithNewToken( token = await api.Update(Routes.Root, cancellationToken).ConfigureAwait(false); } - var apiHeaders = new ApiHeaders(productHeaderValue, token.Bearer!); + var apiHeaders = new ApiHeaders(productHeaderValue, token); var client = new ServerClient( ApiClientFactory.CreateApiClient( host, apiHeaders, attemptLoginRefresh ? loginHeaders : null, - false), - token); + false)); if (timeout.HasValue) client.Timeout = timeout.Value; diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index 8aa6d3d03bc..1c832f5d81b 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -261,7 +261,7 @@ public async ValueTask CreateToken(CancellationToken cancellation return BadRequest(new ErrorMessageResponse(ErrorCode.OAuthProviderDisabled)); externalUserId = await validator - .ValidateResponseCode(ApiHeaders.Token, cancellationToken); + .ValidateResponseCode(ApiHeaders.Token.Bearer!, cancellationToken); Logger.LogTrace("External {oAuthProvider} UID: {externalUserId}", oAuthProvider, externalUserId); } @@ -373,7 +373,7 @@ public async ValueTask CreateToken(CancellationToken cancellation if (usingSystemIdentity) { // expire the identity slightly after the auth token in case of lag - var identExpiry = token.ExpiresAt; + var identExpiry = token.ExpiresAt.Value; identExpiry += tokenFactory.ValidationParameters.ClockSkew; identExpiry += TimeSpan.FromSeconds(15); identityCache.CacheSystemIdentity(user, systemIdentity, identExpiry); diff --git a/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs b/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs index e53a4f8e1ad..5261d8719a5 100644 --- a/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs +++ b/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs @@ -6,6 +6,7 @@ using System.Net.Mime; using Tgstation.Server.Api.Models; +using Tgstation.Server.Api.Models.Response; namespace Tgstation.Server.Api.Tests { @@ -22,7 +23,7 @@ public void TestConstruction() { Assert.ThrowsException(() => new ApiHeaders(null, null)); Assert.ThrowsException(() => new ApiHeaders(productHeaderValue, null)); - var headers = new ApiHeaders(productHeaderValue, String.Empty); + var headers = new ApiHeaders(productHeaderValue, new TokenResponse { Bearer = String.Empty }); headers = new ApiHeaders(productHeaderValue, String.Empty, OAuthProvider.GitHub); } @@ -38,7 +39,7 @@ static ApiHeaders TestHeader(string userAgent) { { "Accept", MediaTypeNames.Application.Json }, { "Api", "Tgstation.Server.Api/4.0.0.0" }, - { "Authorization", "Bearer asdfasdf" }, + { "Authorization", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyLCJuYmYiOjEyMzR9.0CsmEwXt9oNTDisikbZ-MUr1eXSMKD8YKdZIOwMeLoc" }, // fake, but we need a valid token to avoid errors { "User-Agent", userAgent } }; diff --git a/tests/Tgstation.Server.Client.Tests/TestApiClient.cs b/tests/Tgstation.Server.Client.Tests/TestApiClient.cs index d4c81db6c3b..47c5a130583 100644 --- a/tests/Tgstation.Server.Client.Tests/TestApiClient.cs +++ b/tests/Tgstation.Server.Client.Tests/TestApiClient.cs @@ -41,7 +41,17 @@ public async Task TestDeserializingByondModelsWork() var httpClient = new Mock(); httpClient.Setup(x => x.SendAsync(It.IsNotNull(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(response)); - var client = new ApiClient(httpClient.Object, new Uri("http://fake.com"), new ApiHeaders(new ProductHeaderValue("fake"), "fake"), null, false); + var client = new ApiClient( + httpClient.Object, + new Uri("http://fake.com"), + new ApiHeaders( + new ProductHeaderValue("fake"), + new TokenResponse + { + Bearer = "fake", + }), + null, + false); var result = await client.Read(Routes.Byond, default); Assert.AreEqual(sample.Version, result.Version); @@ -66,7 +76,17 @@ public async Task TestUnrecognizedResponse() var httpClient = new Mock(); httpClient.Setup(x => x.SendAsync(It.IsNotNull(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(response)); - var client = new ApiClient(httpClient.Object, new Uri("http://fake.com"), new ApiHeaders(new ProductHeaderValue("fake"), "fake"), null, true); + var client = new ApiClient( + httpClient.Object, + new Uri("http://fake.com"), + new ApiHeaders( + new ProductHeaderValue("fake"), + new TokenResponse + { + Bearer = "fake" + }), + null, + false); await Assert.ThrowsExceptionAsync(() => client.Read(Routes.Byond, default).AsTask()); } From 8ead755799b7c3e208708ecb3a85e84520940cea Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 14:47:30 -0400 Subject: [PATCH 059/138] Clean up line width --- src/Tgstation.Server.Client/IApiClientFactory.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Client/IApiClientFactory.cs b/src/Tgstation.Server.Client/IApiClientFactory.cs index 67e1b2aa528..e49385dcf94 100644 --- a/src/Tgstation.Server.Client/IApiClientFactory.cs +++ b/src/Tgstation.Server.Client/IApiClientFactory.cs @@ -17,6 +17,10 @@ interface IApiClientFactory /// The to use to generate a new . /// If there should be no authentication performed. /// A new . - IApiClient CreateApiClient(Uri url, ApiHeaders apiHeaders, ApiHeaders? tokenRefreshHeaders, bool authless); + IApiClient CreateApiClient( + Uri url, + ApiHeaders apiHeaders, + ApiHeaders? tokenRefreshHeaders, + bool authless); } } From c3168c4e00ba075bab36693e441a7eb34a8fdce2 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 14:47:47 -0400 Subject: [PATCH 060/138] Remove unnecessary braces --- src/Tgstation.Server.Host/Server.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Server.cs b/src/Tgstation.Server.Host/Server.cs index c99506805ba..e6ea8c04b1e 100644 --- a/src/Tgstation.Server.Host/Server.cs +++ b/src/Tgstation.Server.Host/Server.cs @@ -128,7 +128,6 @@ public async ValueTask Run(CancellationToken cancellationToken) try { using (Host = hostBuilder.Build()) - { try { logger = Host.Services.GetRequiredService>(); @@ -155,7 +154,6 @@ public async ValueTask Run(CancellationToken cancellationToken) { logger = null; } - } } finally { From 4f3f12f05dbaaf7378ab737d2262a9cc4d92c28c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 14:48:11 -0400 Subject: [PATCH 061/138] Silence a VS message about a `new()` expression --- .../Components/Chat/Providers/DiscordProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs index 1419bd70c46..de804479533 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs @@ -380,7 +380,7 @@ public override async ValueTask new Embed + Embed CreateUpdatedEmbed(string message, Color color) => new () { Author = embed.Author, Colour = color, From 5a5997492bb9839b5d3dde7de96b91beb0e18a0f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 14:48:57 -0400 Subject: [PATCH 062/138] Fix routing in `ControlPanelController` - This was generating an unnecessary 301 when getting `/app` instead of `/app`. --- src/Tgstation.Server.Host/Controllers/ControlPanelController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs b/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs index c1f35f87e5e..08d8f74dad1 100644 --- a/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs +++ b/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs @@ -123,7 +123,7 @@ public override Task OnActionExecutionAsync(ActionExecutingContext context, Acti /// /// The value of the route. /// The to use. - [Route("{**appRoute}")] + [Route("/{**appRoute}")] [HttpGet] public IActionResult Get([FromRoute] string appRoute) { From 9325e545fd9ab4b93a107792375b43b5f9fdc83a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 14:50:49 -0400 Subject: [PATCH 063/138] Simplify some `Substring` calls --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 0d8fec84887..b7a430b8105 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1106,9 +1106,9 @@ async ValueTask RunGitCommand(string args) var dependenciesSh = Encoding.UTF8.GetString(await depsBytesTask); var lines = dependenciesSh.Split("\n", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); const string MajorPrefix = "export BYOND_MAJOR="; - var major = Int32.Parse(lines.First(x => x.StartsWith(MajorPrefix)).Substring(MajorPrefix.Length)); + var major = Int32.Parse(lines.First(x => x.StartsWith(MajorPrefix))[MajorPrefix.Length..]); const string MinorPrefix = "export BYOND_MINOR="; - var minor = Int32.Parse(lines.First(x => x.StartsWith(MinorPrefix)).Substring(MinorPrefix.Length)); + var minor = Int32.Parse(lines.First(x => x.StartsWith(MinorPrefix))[MinorPrefix.Length..]); var byondJob = await instanceClient.Byond.SetActiveVersion(new ByondVersionRequest { From 10728657df94d987abc44bad42fc174bce63f806 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 14:55:06 -0400 Subject: [PATCH 064/138] Move custom controller results to new namespace --- .../Controllers/AdministrationController.cs | 1 + src/Tgstation.Server.Host/Controllers/ApiController.cs | 1 + src/Tgstation.Server.Host/Controllers/ByondController.cs | 1 + src/Tgstation.Server.Host/Controllers/ChatController.cs | 1 + .../Controllers/ConfigurationController.cs | 1 + src/Tgstation.Server.Host/Controllers/DreamMakerController.cs | 1 + src/Tgstation.Server.Host/Controllers/InstanceController.cs | 1 + .../Controllers/InstancePermissionSetController.cs | 1 + src/Tgstation.Server.Host/Controllers/JobController.cs | 1 + .../Controllers/{ => Results}/LimitedStreamResult.cs | 2 +- .../Controllers/{ => Results}/LimitedStreamResultExecutor.cs | 2 +- .../Controllers/{ => Results}/PaginatableResult.cs | 2 +- src/Tgstation.Server.Host/Controllers/TransferController.cs | 1 + src/Tgstation.Server.Host/Controllers/UserController.cs | 1 + src/Tgstation.Server.Host/Controllers/UserGroupController.cs | 1 + src/Tgstation.Server.Host/Core/Application.cs | 1 + .../Extensions/FileTransferStreamHandlerExtensions.cs | 2 +- tests/Tgstation.Server.Host.Tests/Swarm/SwarmRpcMapper.cs | 1 + 18 files changed, 18 insertions(+), 4 deletions(-) rename src/Tgstation.Server.Host/Controllers/{ => Results}/LimitedStreamResult.cs (96%) rename src/Tgstation.Server.Host/Controllers/{ => Results}/LimitedStreamResultExecutor.cs (97%) rename src/Tgstation.Server.Host/Controllers/{ => Results}/PaginatableResult.cs (96%) diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index b742a2ae18a..567d51bd078 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -18,6 +18,7 @@ using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index c0f37fbf82c..98fd2b401f3 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -23,6 +23,7 @@ using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Common.Extensions; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; diff --git a/src/Tgstation.Server.Host/Controllers/ByondController.cs b/src/Tgstation.Server.Host/Controllers/ByondController.cs index 156ff784e93..2c0281b99be 100644 --- a/src/Tgstation.Server.Host/Controllers/ByondController.cs +++ b/src/Tgstation.Server.Host/Controllers/ByondController.cs @@ -13,6 +13,7 @@ using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Components; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Jobs; diff --git a/src/Tgstation.Server.Host/Controllers/ChatController.cs b/src/Tgstation.Server.Host/Controllers/ChatController.cs index c1365b1f842..b88ab73654e 100644 --- a/src/Tgstation.Server.Host/Controllers/ChatController.cs +++ b/src/Tgstation.Server.Host/Controllers/ChatController.cs @@ -19,6 +19,7 @@ using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Components; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; diff --git a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs index b71703af41f..adebfe01030 100644 --- a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs +++ b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs @@ -12,6 +12,7 @@ using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Components; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Models; diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 3c4431a3e1a..acb4d53dac9 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -13,6 +13,7 @@ using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Components; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Jobs; diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index 0e2ebf234e6..6a30e253998 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -19,6 +19,7 @@ using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Components; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; diff --git a/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs b/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs index ca8fdae67bb..2d9059b2047 100644 --- a/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs @@ -13,6 +13,7 @@ using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Components; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; diff --git a/src/Tgstation.Server.Host/Controllers/JobController.cs b/src/Tgstation.Server.Host/Controllers/JobController.cs index 84f4cc50845..6e0817b5534 100644 --- a/src/Tgstation.Server.Host/Controllers/JobController.cs +++ b/src/Tgstation.Server.Host/Controllers/JobController.cs @@ -11,6 +11,7 @@ using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Host.Components; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Jobs; diff --git a/src/Tgstation.Server.Host/Controllers/LimitedStreamResult.cs b/src/Tgstation.Server.Host/Controllers/Results/LimitedStreamResult.cs similarity index 96% rename from src/Tgstation.Server.Host/Controllers/LimitedStreamResult.cs rename to src/Tgstation.Server.Host/Controllers/Results/LimitedStreamResult.cs index 734a1731a6d..998f5779c5c 100644 --- a/src/Tgstation.Server.Host/Controllers/LimitedStreamResult.cs +++ b/src/Tgstation.Server.Host/Controllers/Results/LimitedStreamResult.cs @@ -10,7 +10,7 @@ using Tgstation.Server.Host.IO; -namespace Tgstation.Server.Host.Controllers +namespace Tgstation.Server.Host.Controllers.Results { /// /// Very similar to except it's contains a fix for https://github.com/dotnet/aspnetcore/issues/28189. diff --git a/src/Tgstation.Server.Host/Controllers/LimitedStreamResultExecutor.cs b/src/Tgstation.Server.Host/Controllers/Results/LimitedStreamResultExecutor.cs similarity index 97% rename from src/Tgstation.Server.Host/Controllers/LimitedStreamResultExecutor.cs rename to src/Tgstation.Server.Host/Controllers/Results/LimitedStreamResultExecutor.cs index 7a8169c1d3e..3178db355b4 100644 --- a/src/Tgstation.Server.Host/Controllers/LimitedStreamResultExecutor.cs +++ b/src/Tgstation.Server.Host/Controllers/Results/LimitedStreamResultExecutor.cs @@ -7,7 +7,7 @@ using Microsoft.AspNetCore.Mvc.Infrastructure; using Microsoft.Extensions.Logging; -namespace Tgstation.Server.Host.Controllers +namespace Tgstation.Server.Host.Controllers.Results { /// /// for s. diff --git a/src/Tgstation.Server.Host/Controllers/PaginatableResult.cs b/src/Tgstation.Server.Host/Controllers/Results/PaginatableResult.cs similarity index 96% rename from src/Tgstation.Server.Host/Controllers/PaginatableResult.cs rename to src/Tgstation.Server.Host/Controllers/Results/PaginatableResult.cs index ae19ba913e8..b936d0f3e6e 100644 --- a/src/Tgstation.Server.Host/Controllers/PaginatableResult.cs +++ b/src/Tgstation.Server.Host/Controllers/Results/PaginatableResult.cs @@ -3,7 +3,7 @@ using Microsoft.AspNetCore.Mvc; -namespace Tgstation.Server.Host.Controllers +namespace Tgstation.Server.Host.Controllers.Results { /// /// Helper for returning paginated models. diff --git a/src/Tgstation.Server.Host/Controllers/TransferController.cs b/src/Tgstation.Server.Host/Controllers/TransferController.cs index be12cfda911..489b3525cb8 100644 --- a/src/Tgstation.Server.Host/Controllers/TransferController.cs +++ b/src/Tgstation.Server.Host/Controllers/TransferController.cs @@ -9,6 +9,7 @@ using Tgstation.Server.Api; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Security; diff --git a/src/Tgstation.Server.Host/Controllers/UserController.cs b/src/Tgstation.Server.Host/Controllers/UserController.cs index 04559bde3c4..50c071ee997 100644 --- a/src/Tgstation.Server.Host/Controllers/UserController.cs +++ b/src/Tgstation.Server.Host/Controllers/UserController.cs @@ -15,6 +15,7 @@ using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; diff --git a/src/Tgstation.Server.Host/Controllers/UserGroupController.cs b/src/Tgstation.Server.Host/Controllers/UserGroupController.cs index 14269e3254f..a23be84706c 100644 --- a/src/Tgstation.Server.Host/Controllers/UserGroupController.cs +++ b/src/Tgstation.Server.Host/Controllers/UserGroupController.cs @@ -14,6 +14,7 @@ using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 811c77d7080..1e4a7458358 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -40,6 +40,7 @@ using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Controllers; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Extensions.Converters; diff --git a/src/Tgstation.Server.Host/Extensions/FileTransferStreamHandlerExtensions.cs b/src/Tgstation.Server.Host/Extensions/FileTransferStreamHandlerExtensions.cs index 8028f8ab8db..a7e84abac19 100644 --- a/src/Tgstation.Server.Host/Extensions/FileTransferStreamHandlerExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/FileTransferStreamHandlerExtensions.cs @@ -11,7 +11,7 @@ using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; -using Tgstation.Server.Host.Controllers; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Extensions diff --git a/tests/Tgstation.Server.Host.Tests/Swarm/SwarmRpcMapper.cs b/tests/Tgstation.Server.Host.Tests/Swarm/SwarmRpcMapper.cs index 167c058c19d..df37c87ca0e 100644 --- a/tests/Tgstation.Server.Host.Tests/Swarm/SwarmRpcMapper.cs +++ b/tests/Tgstation.Server.Host.Tests/Swarm/SwarmRpcMapper.cs @@ -24,6 +24,7 @@ using Tgstation.Server.Common.Http; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Controllers; +using Tgstation.Server.Host.Controllers.Results; using Tgstation.Server.Host.Transfer; namespace Tgstation.Server.Host.Swarm.Tests From 548d8ccf092d36b2a04adff51ec919036b396417 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 14:57:00 -0400 Subject: [PATCH 065/138] Minor code cleanups in `TokenFactory` --- src/Tgstation.Server.Host/Security/TokenFactory.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Security/TokenFactory.cs b/src/Tgstation.Server.Host/Security/TokenFactory.cs index 81a96876f60..9ceac820cac 100644 --- a/src/Tgstation.Server.Host/Security/TokenFactory.cs +++ b/src/Tgstation.Server.Host/Security/TokenFactory.cs @@ -112,7 +112,7 @@ public async ValueTask CreateToken(Models.User user, bool oAuth, var nowUnix = now.ToUnixTimeSeconds(); // this prevents validation conflicts down the line - // tldr we can (theoretically) send a token the same second we receive it + // tldr we can (theoretically) receive a token the same second after we generate it // since unix time rounds down, it looks like it came from before the user changed their password // this happens occasionally in unit tests // just delay a second so we can force a round up @@ -125,9 +125,9 @@ public async ValueTask CreateToken(Models.User user, bool oAuth, : securityConfiguration.TokenExpiryMinutes); var claims = new Claim[] { - new Claim(JwtRegisteredClaimNames.Sub, user.Id.Value.ToString(CultureInfo.InvariantCulture)), - new Claim(JwtRegisteredClaimNames.Exp, expiry.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture)), - new Claim(JwtRegisteredClaimNames.Nbf, nowUnix.ToString(CultureInfo.InvariantCulture)), + new (JwtRegisteredClaimNames.Sub, user.Id.Value.ToString(CultureInfo.InvariantCulture)), + new (JwtRegisteredClaimNames.Exp, expiry.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture)), + new (JwtRegisteredClaimNames.Nbf, nowUnix.ToString(CultureInfo.InvariantCulture)), issuerClaim, audienceClaim, }; From 49a351ecc6b259ae3343a7a154357f5372b8d74e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 14:59:44 -0400 Subject: [PATCH 066/138] Move client cache disabling to new `ApiControllerBase` It's probably not in a good middleware spot anymore with endpoint routing. Also disable CA1501 (Excessive inheritance) because our use cases appear to be consistently valid. --- build/analyzers.ruleset | 8 +-- .../Models/Request/UserCreateRequest.cs | 1 - .../Controllers/ApiController.cs | 50 ++++++------------- .../Controllers/ApiControllerBase.cs | 48 ++++++++++++++++++ .../Controllers/BridgeController.cs | 4 +- .../Controllers/SwarmController.cs | 22 +++----- src/Tgstation.Server.Host/Core/Application.cs | 3 -- .../ApplicationBuilderExtensions.cs | 16 ------ 8 files changed, 76 insertions(+), 76 deletions(-) create mode 100644 src/Tgstation.Server.Host/Controllers/ApiControllerBase.cs diff --git a/build/analyzers.ruleset b/build/analyzers.ruleset index 365db1e57f0..0f86cbd400e 100644 --- a/build/analyzers.ruleset +++ b/build/analyzers.ruleset @@ -1,4 +1,4 @@ - + @@ -88,7 +88,7 @@ - + @@ -971,7 +971,7 @@ - + @@ -1043,4 +1043,4 @@ - \ No newline at end of file + diff --git a/src/Tgstation.Server.Api/Models/Request/UserCreateRequest.cs b/src/Tgstation.Server.Api/Models/Request/UserCreateRequest.cs index 5f3b4e09da4..9c35ee12ce7 100644 --- a/src/Tgstation.Server.Api/Models/Request/UserCreateRequest.cs +++ b/src/Tgstation.Server.Api/Models/Request/UserCreateRequest.cs @@ -3,7 +3,6 @@ /// /// For creating a user. /// -#pragma warning disable CA1501 public sealed class UserCreateRequest : UserUpdateRequest { } diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index 98fd2b401f3..3feebe9c263 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -3,13 +3,11 @@ using System.Globalization; using System.Linq; using System.Net; -using System.Net.Mime; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Query; using Microsoft.Extensions.Logging; @@ -35,9 +33,7 @@ namespace Tgstation.Server.Host.Controllers /// /// Base for API functions. /// - [Produces(MediaTypeNames.Application.Json)] - [ApiController] - public abstract class ApiController : Controller + public abstract class ApiController : ApiControllerBase { /// /// Default size of results. @@ -102,18 +98,14 @@ protected ApiController( /// #pragma warning disable CA1506 // TODO: Decomplexify - public override async Task OnActionExecutionAsync(ActionExecutingContext context, ActionExecutionDelegate next) + protected override async ValueTask HookExecuteAction(Func executeAction, CancellationToken cancellationToken) { - ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(executeAction); // ALL valid token and login requests that match a route go through this function // 404 is returned before if (AuthenticationContext != null && AuthenticationContext.User == null) - { - // valid token, expired password - await Unauthorized().ExecuteResultAsync(context); - return; - } + return Unauthorized(); // valid token, expired password // validate the headers try @@ -121,29 +113,18 @@ public override async Task OnActionExecutionAsync(ActionExecutingContext context ApiHeaders = new ApiHeaders(Request.GetTypedHeaders()); if (!ApiHeaders.Compatible()) - { - await this.StatusCode( + return this.StatusCode( HttpStatusCode.UpgradeRequired, - new ErrorMessageResponse(ErrorCode.ApiMismatch)) - .ExecuteResultAsync(context); - return; - } + new ErrorMessageResponse(ErrorCode.ApiMismatch)); - var errorCase = await ValidateRequest(context.HttpContext.RequestAborted); + var errorCase = await ValidateRequest(cancellationToken); if (errorCase != null) - { - await errorCase.ExecuteResultAsync(context); - return; - } + return errorCase; } catch (HeadersException) { if (requireHeaders) - { - await HeadersIssue(false) - .ExecuteResultAsync(context); - return; - } + return HeadersIssue(false); } if (ModelState?.IsValid == false) @@ -159,15 +140,11 @@ await HeadersIssue(false) .Where(x => !x.EndsWith(" field is required.", StringComparison.Ordinal)); if (errorMessages.Any()) - { - await BadRequest( + return BadRequest( new ErrorMessageResponse(ErrorCode.ModelValidationFailure) { AdditionalData = String.Join(Environment.NewLine, errorMessages), - }) - .ExecuteResultAsync(context); - return; - } + }); ModelState.Clear(); } @@ -202,8 +179,11 @@ await BadRequest( Logger.LogDebug( "Starting unauthorized API request. No {userAgentHeaderName}!", HeaderNames.UserAgent); - await base.OnActionExecutionAsync(context, next); + + await executeAction(); } + + return null; } #pragma warning restore CA1506 diff --git a/src/Tgstation.Server.Host/Controllers/ApiControllerBase.cs b/src/Tgstation.Server.Host/Controllers/ApiControllerBase.cs new file mode 100644 index 00000000000..9a8b1198082 --- /dev/null +++ b/src/Tgstation.Server.Host/Controllers/ApiControllerBase.cs @@ -0,0 +1,48 @@ +using System; +using System.Net.Mime; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.Extensions.Primitives; +using Microsoft.Net.Http.Headers; + +namespace Tgstation.Server.Host.Controllers +{ + /// + /// Base class for all API style controllers. + /// + [Produces(MediaTypeNames.Application.Json)] + [ApiController] + public abstract class ApiControllerBase : Controller + { + /// + public sealed override async Task OnActionExecutionAsync(ActionExecutingContext context, ActionExecutionDelegate next) + { + // never cache an API response + Response.Headers.Add(HeaderNames.CacheControl, new StringValues("no-cache")); + + var errorCase = await HookExecuteAction( + () => base.OnActionExecutionAsync(context, next), + Request.HttpContext.RequestAborted); + + if (errorCase != null) + await errorCase.ExecuteResultAsync(context); + } + + /// + /// Hook for executing a request. + /// + /// A that should be invoked and its response awaited to continue normal execution of the request. Should NOT be called if this method returns a non- value. + /// The for the operation. + /// A resulting in an that, if not , is executed. + protected virtual async ValueTask HookExecuteAction(Func executeAction, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(executeAction); + + await executeAction(); + return null; + } + } +} diff --git a/src/Tgstation.Server.Host/Controllers/BridgeController.cs b/src/Tgstation.Server.Host/Controllers/BridgeController.cs index 1f53b43ec20..6afb32bc1d3 100644 --- a/src/Tgstation.Server.Host/Controllers/BridgeController.cs +++ b/src/Tgstation.Server.Host/Controllers/BridgeController.cs @@ -20,10 +20,8 @@ namespace Tgstation.Server.Host.Controllers /// for recieving DMAPI requests from DreamDaemon. /// [Route("/Bridge")] - [Produces(MediaTypeNames.Application.Json)] - [ApiController] [ApiExplorerSettings(IgnoreApi = true)] - public class BridgeController : Controller + public class BridgeController : ApiControllerBase { /// /// If the content of bridge requests and responses should be logged. diff --git a/src/Tgstation.Server.Host/Controllers/SwarmController.cs b/src/Tgstation.Server.Host/Controllers/SwarmController.cs index 8b941d425c9..fb81784b12c 100644 --- a/src/Tgstation.Server.Host/Controllers/SwarmController.cs +++ b/src/Tgstation.Server.Host/Controllers/SwarmController.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -25,10 +24,8 @@ namespace Tgstation.Server.Host.Controllers /// For swarm server communication. /// [Route(SwarmConstants.ControllerRoute)] - [Produces(MediaTypeNames.Application.Json)] - [ApiController] [ApiExplorerSettings(IgnoreApi = true)] - public sealed class SwarmController : Controller + public sealed class SwarmController : ApiControllerBase { /// /// Get the current registration from the . @@ -211,7 +208,7 @@ public async ValueTask AbortUpdate() } /// - public override async Task OnActionExecutionAsync(ActionExecutingContext context, ActionExecutionDelegate next) + protected override async ValueTask HookExecuteAction(Func executeAction, CancellationToken cancellationToken) { using (LogContext.PushProperty(SerilogContextHelper.RequestPathContextProperty, $"{Request.Method} {Request.Path}")) { @@ -219,8 +216,7 @@ public override async Task OnActionExecutionAsync(ActionExecutingContext context if (swarmConfiguration.PrivateKey == null) { logger.LogDebug("Attempted swarm request without private key configured!"); - await Forbid().ExecuteResultAsync(context); - return; + return Forbid(); } if (!(Request.Headers.TryGetValue(SwarmConstants.ApiKeyHeader, out var apiKeyHeaderValues) @@ -228,8 +224,7 @@ public override async Task OnActionExecutionAsync(ActionExecutingContext context && apiKeyHeaderValues.First() == swarmConfiguration.PrivateKey)) { logger.LogDebug("Unauthorized swarm request!"); - await Unauthorized().ExecuteResultAsync(context); - return; + return Unauthorized(); } if (!(Request.Headers.TryGetValue(SwarmConstants.RegistrationIdHeader, out var registrationHeaderValues) @@ -237,8 +232,7 @@ public override async Task OnActionExecutionAsync(ActionExecutingContext context && Guid.TryParse(registrationHeaderValues.First(), out var registrationId))) { logger.LogDebug("Swarm request without registration ID!"); - await BadRequest().ExecuteResultAsync(context); - return; + return BadRequest(); } // we validate the registration itself on a case-by-case basis @@ -252,12 +246,12 @@ public override async Task OnActionExecutionAsync(ActionExecutingContext context "Swarm request model validation failed!{newLine}{messages}", Environment.NewLine, String.Join(Environment.NewLine, errorMessages)); - await BadRequest().ExecuteResultAsync(context); - return; + return BadRequest(); } logger.LogTrace("Starting swarm request processing..."); - await base.OnActionExecutionAsync(context, next); + await executeAction(); + return null; } } diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 1e4a7458358..938ce47fed7 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -514,9 +514,6 @@ public void Configure( logger.LogTrace("Web control panel disabled!"); #endif - // Do not cache a single thing beyond this point, it's all API - applicationBuilder.UseDisabledClientCache(); - // Stack overflow said this needs to go here and removing it breaks things: https://stackoverflow.com/questions/73736879/invalidoperationexception-endpointroutingmiddleware-matches-endpoints-setup-by applicationBuilder.UseRouting(); diff --git a/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs b/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs index 42bb3ad8fb1..ae8eb65ae61 100644 --- a/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs @@ -8,8 +8,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Primitives; -using Microsoft.Net.Http.Headers; using Serilog.Context; @@ -66,20 +64,6 @@ public static void UseDbConflictHandling(this IApplicationBuilder applicationBui }); } - /// - /// Suppress any client side caching of API calls. - /// - /// The to configure. - public static void UseDisabledClientCache(this IApplicationBuilder applicationBuilder) - { - ArgumentNullException.ThrowIfNull(applicationBuilder); - applicationBuilder.Use(async (context, next) => - { - context.Response.Headers.Add(HeaderNames.CacheControl, new StringValues("no-cache")); - await next(); - }); - } - /// /// Suppress warnings when a user aborts a request. /// From 5b615ea04b489d1de481e2d5ad9c72327ea779b9 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 15:07:33 -0400 Subject: [PATCH 067/138] Upgrade "webpanel not built" message to `Debug` --- src/Tgstation.Server.Host/Core/Application.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 938ce47fed7..329b2262702 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -509,7 +509,7 @@ public void Configure( } else #if NO_WEBPANEL - logger.LogTrace("Web control panel was not included in TGS build!"); + logger.LogDebug("Web control panel was not included in TGS build!"); #else logger.LogTrace("Web control panel disabled!"); #endif From 78669f486c1936a5360ae858853e55ba075df20d Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 15:08:07 -0400 Subject: [PATCH 068/138] Fix issue with CORS not working properly --- src/Tgstation.Server.Host/Core/Application.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 329b2262702..459272d77ed 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -472,6 +472,9 @@ public void Configure( logger.LogTrace("Swagger API generation enabled"); } + // Enable endpoint routing + applicationBuilder.UseRouting(); + // Set up CORS based on configuration if necessary Action corsBuilder = null; if (controlPanelConfiguration.AllowAnyOrigin) @@ -514,9 +517,6 @@ public void Configure( logger.LogTrace("Web control panel disabled!"); #endif - // Stack overflow said this needs to go here and removing it breaks things: https://stackoverflow.com/questions/73736879/invalidoperationexception-endpointroutingmiddleware-matches-endpoints-setup-by - applicationBuilder.UseRouting(); - // authenticate JWT tokens using our security pipeline if present, returns 401 if bad applicationBuilder.UseAuthentication(); From a7cb72ca8472495d85fa45d3d3e8a4aee4cb011b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 15:11:08 -0400 Subject: [PATCH 069/138] Fix BOM in `ServerClient.cs` --- src/Tgstation.Server.Client/ServerClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Client/ServerClient.cs b/src/Tgstation.Server.Client/ServerClient.cs index 207a085e289..5826b76bdc6 100644 --- a/src/Tgstation.Server.Client/ServerClient.cs +++ b/src/Tgstation.Server.Client/ServerClient.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; From 4b39bde25d69a4bffa4a64cad52b13359d43650b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 15:12:27 -0400 Subject: [PATCH 070/138] Fix `HardFailLogger`/provider logging non-errors --- tests/Tgstation.Server.Tests/Live/HardFailLogger.cs | 1 - tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs index f5b5707bf71..eca7cf4ef58 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -30,7 +30,6 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except && !(logMessage.StartsWith("An exception occurred in the database while saving changes for context type") && (exception is OperationCanceledException || exception?.InnerException is OperationCanceledException))) || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) { - Console.WriteLine("UNEXPECTED ERROR OCCURS NEAR HERE"); failureSink(new AssertFailedException("TGS logged an unexpected error!")); } } diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs b/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs index 59a677abd89..9f8ae24a051 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -15,7 +16,10 @@ sealed class HardFailLoggerProvider : ILoggerProvider public ILogger CreateLogger(string categoryName) => new HardFailLogger(ex => { if (!BlockFails) + { + Console.WriteLine("UNEXPECTED ERROR OCCURS NEAR HERE"); failureSink.TrySetException(ex); + } }); public void Dispose() { } From 590f5c6a5510a511cc30291436e5dadeba4cfdac Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 15:26:34 -0400 Subject: [PATCH 071/138] Fix `ApiHeaders` improperly storing `OAuthCode` - Store in its own property instead of `Token` --- src/Tgstation.Server.Api/ApiHeaders.cs | 19 ++++++++++--------- .../Controllers/HomeController.cs | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Tgstation.Server.Api/ApiHeaders.cs b/src/Tgstation.Server.Api/ApiHeaders.cs index e9b721a3038..5670abb8a30 100644 --- a/src/Tgstation.Server.Api/ApiHeaders.cs +++ b/src/Tgstation.Server.Api/ApiHeaders.cs @@ -112,6 +112,11 @@ public sealed class ApiHeaders /// public string? Password { get; } + /// + /// The OAuth code in use. + /// + public string? OAuthCode { get; } + /// /// The the is for, if any. /// @@ -122,11 +127,6 @@ public sealed class ApiHeaders /// public bool IsTokenAuthentication => Token != null && !OAuthProvider.HasValue; - /// - /// The OAuth code in use. - /// - readonly string? oAuthCode; - /// /// Checks if a given is compatible with our own. /// @@ -154,7 +154,7 @@ public ApiHeaders(ProductHeaderValue userAgent, TokenResponse token) /// Initializes a new instance of the class. Used for token authentication. /// /// The value of . - /// The value of . + /// The value of . /// The value of . public ApiHeaders(ProductHeaderValue userAgent, string oAuthCode, OAuthProvider oAuthProvider) : this(userAgent, null, null, null) @@ -162,7 +162,7 @@ public ApiHeaders(ProductHeaderValue userAgent, string oAuthCode, OAuthProvider if (userAgent == null) throw new ArgumentNullException(nameof(userAgent)); - this.oAuthCode = oAuthCode ?? throw new ArgumentNullException(nameof(oAuthCode)); + OAuthCode = oAuthCode ?? throw new ArgumentNullException(nameof(oAuthCode)); OAuthProvider = oAuthProvider; } @@ -267,7 +267,8 @@ void AddError(HeaderTypes headerType, string message) else AddError(HeaderTypes.OAuthProvider, $"Missing {OAuthProviderHeader} header!"); - goto case BearerAuthenticationScheme; + OAuthCode = parameter; + break; case BearerAuthenticationScheme: var tokenSplits = parameter.Split('.'); DateTimeOffset? expiresAt = null; @@ -379,7 +380,7 @@ public void SetRequestHeaders(HttpRequestHeaders headers, long? instanceId = nul headers.Add(ApiVersionHeader, CreateApiVersionHeader()); if (OAuthProvider.HasValue) { - headers.Authorization = new AuthenticationHeaderValue(OAuthAuthenticationScheme, Token!.Bearer); + headers.Authorization = new AuthenticationHeaderValue(OAuthAuthenticationScheme, OAuthCode!); headers.Add(OAuthProviderHeader, OAuthProvider.ToString()); } else if (!IsTokenAuthentication) diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index 1c832f5d81b..af1d4b90dd7 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -261,7 +261,7 @@ public async ValueTask CreateToken(CancellationToken cancellation return BadRequest(new ErrorMessageResponse(ErrorCode.OAuthProviderDisabled)); externalUserId = await validator - .ValidateResponseCode(ApiHeaders.Token.Bearer!, cancellationToken); + .ValidateResponseCode(ApiHeaders.OAuthCode!, cancellationToken); Logger.LogTrace("External {oAuthProvider} UID: {externalUserId}", oAuthProvider, externalUserId); } From fd44b61a24541f6c7d7b392550a4d3c4b5fb29ba Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 15:15:26 -0400 Subject: [PATCH 072/138] Authentication pipeline rewrite If you're looking for the future CVE it probably came from here. --- .../Controllers/AdministrationController.cs | 12 ++- .../Controllers/ApiController.cs | 45 ++++---- .../Controllers/BridgeController.cs | 4 +- .../Controllers/ByondController.cs | 14 ++- .../Controllers/ChatController.cs | 15 +-- .../ComponentInterfacingController.cs | 11 +- .../Controllers/ConfigurationController.cs | 14 ++- .../Controllers/DreamDaemonController.cs | 13 ++- .../Controllers/DreamMakerController.cs | 13 ++- .../Controllers/HomeController.cs | 14 ++- .../Controllers/InstanceController.cs | 16 +-- .../InstancePermissionSetController.cs | 14 ++- .../Controllers/InstanceRequiredController.cs | 12 ++- .../Controllers/JobController.cs | 16 +-- .../Controllers/RepositoryController.cs | 14 ++- .../Controllers/SwarmController.cs | 2 + .../Controllers/TransferController.cs | 12 ++- .../Controllers/UserController.cs | 14 ++- .../Controllers/UserGroupController.cs | 20 ++-- src/Tgstation.Server.Host/Core/Application.cs | 101 +++++++++++------- .../Security/AuthenticationContext.cs | 32 ++++-- ...uthenticationContextAuthorizationFilter.cs | 53 +++++++++ ...thenticationContextClaimsTransformation.cs | 97 +++++++++++++++++ .../Security/AuthenticationContextFactory.cs | 63 ++++++++--- .../Security/ClaimsInjector.cs | 95 ---------------- .../Security/IAuthenticationContext.cs | 11 +- .../Security/IAuthenticationContextFactory.cs | 19 ++-- .../Security/IClaimsInjector.cs | 21 ---- .../Utils/ApiHeadersProvider.cs | 39 +++++++ .../Utils/IApiHeadersProvider.cs | 15 +++ .../Security/TestAuthenticationContext.cs | 23 ++-- 31 files changed, 528 insertions(+), 316 deletions(-) create mode 100644 src/Tgstation.Server.Host/Security/AuthenticationContextAuthorizationFilter.cs create mode 100644 src/Tgstation.Server.Host/Security/AuthenticationContextClaimsTransformation.cs delete mode 100644 src/Tgstation.Server.Host/Security/ClaimsInjector.cs delete mode 100644 src/Tgstation.Server.Host/Security/IClaimsInjector.cs create mode 100644 src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs create mode 100644 src/Tgstation.Server.Host/Utils/IApiHeadersProvider.cs diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index 567d51bd078..c1da69a69d6 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -26,6 +26,7 @@ using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Transfer; +using Tgstation.Server.Host.Utils; using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Controllers @@ -85,7 +86,7 @@ public sealed class AdministrationController : ApiController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The value of . /// The value of . /// The value of . @@ -95,9 +96,10 @@ public sealed class AdministrationController : ApiController /// The value of . /// The for the . /// The containing value of . + /// The for the . public AdministrationController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, IGitHubServiceFactory gitHubServiceFactory, IServerControl serverControl, IServerUpdateInitiator serverUpdateInitiator, @@ -106,10 +108,12 @@ public AdministrationController( IPlatformIdentifier platformIdentifier, IFileTransferTicketProvider fileTransferService, ILogger logger, - IOptions fileLoggingConfigurationOptions) + IOptions fileLoggingConfigurationOptions, + IApiHeadersProvider apiHeadersProvider) : base( databaseContext, - authenticationContextFactory, + authenticationContext, + apiHeadersProvider, logger, true) { diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index 3feebe9c263..9fd4b6154ad 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -48,7 +48,7 @@ public abstract class ApiController : ApiControllerBase /// /// The for the operation. /// - protected ApiHeaders ApiHeaders { get; private set; } + protected ApiHeaders ApiHeaders { get; } /// /// The for the operation. @@ -79,20 +79,24 @@ public abstract class ApiController : ApiControllerBase /// Initializes a new instance of the class. /// /// The value of . - /// The for the . + /// The for the . /// The value of . + /// The containing value of . /// The value of . protected ApiController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, + IApiHeadersProvider apiHeadersProvider, ILogger logger, bool requireHeaders) { DatabaseContext = databaseContext ?? throw new ArgumentNullException(nameof(databaseContext)); - ArgumentNullException.ThrowIfNull(authenticationContextFactory); + AuthenticationContext = authenticationContext ?? throw new ArgumentNullException(nameof(authenticationContext)); + ArgumentNullException.ThrowIfNull(apiHeadersProvider); Logger = logger ?? throw new ArgumentNullException(nameof(logger)); - AuthenticationContext = authenticationContextFactory.CurrentAuthenticationContext; + Instance = AuthenticationContext?.InstancePermissionSet?.Instance; + ApiHeaders = apiHeadersProvider.ApiHeaders; this.requireHeaders = requireHeaders; } @@ -102,30 +106,20 @@ protected override async ValueTask HookExecuteAction(Func e { ArgumentNullException.ThrowIfNull(executeAction); - // ALL valid token and login requests that match a route go through this function - // 404 is returned before - if (AuthenticationContext != null && AuthenticationContext.User == null) - return Unauthorized(); // valid token, expired password - // validate the headers - try - { - ApiHeaders = new ApiHeaders(Request.GetTypedHeaders()); - - if (!ApiHeaders.Compatible()) - return this.StatusCode( - HttpStatusCode.UpgradeRequired, - new ErrorMessageResponse(ErrorCode.ApiMismatch)); - - var errorCase = await ValidateRequest(cancellationToken); - if (errorCase != null) - return errorCase; - } - catch (HeadersException) + if (ApiHeaders == null) { if (requireHeaders) return HeadersIssue(false); } + else if (!ApiHeaders.Compatible()) + return this.StatusCode( + HttpStatusCode.UpgradeRequired, + new ErrorMessageResponse(ErrorCode.ApiMismatch)); + + var errorCase = await ValidateRequest(cancellationToken); + if (errorCase != null) + return errorCase; if (ModelState?.IsValid == false) { @@ -152,7 +146,7 @@ protected override async ValueTask HookExecuteAction(Func e using (ApiHeaders?.InstanceId != null ? LogContext.PushProperty(SerilogContextHelper.InstanceIdContextProperty, ApiHeaders.InstanceId) : null) - using (AuthenticationContext != null + using (AuthenticationContext.Valid ? LogContext.PushProperty(SerilogContextHelper.UserIdContextProperty, AuthenticationContext.User.Id) : null) using (LogContext.PushProperty(SerilogContextHelper.RequestPathContextProperty, $"{Request.Method} {Request.Path}")) @@ -252,6 +246,7 @@ protected IActionResult HeadersIssue(bool ignoreMissingAuth) HeadersException headersException; try { + // TODO: Move this somewhere saner? _ = new ApiHeaders(Request.GetTypedHeaders(), ignoreMissingAuth); throw new InvalidOperationException("Expected a header parse exception!"); } diff --git a/src/Tgstation.Server.Host/Controllers/BridgeController.cs b/src/Tgstation.Server.Host/Controllers/BridgeController.cs index 6afb32bc1d3..4ae2e0497c8 100644 --- a/src/Tgstation.Server.Host/Controllers/BridgeController.cs +++ b/src/Tgstation.Server.Host/Controllers/BridgeController.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -21,7 +22,7 @@ namespace Tgstation.Server.Host.Controllers /// [Route("/Bridge")] [ApiExplorerSettings(IgnoreApi = true)] - public class BridgeController : ApiControllerBase + public sealed class BridgeController : ApiControllerBase { /// /// If the content of bridge requests and responses should be logged. @@ -74,6 +75,7 @@ public BridgeController(IBridgeDispatcher bridgeDispatcher, IHostApplicationLife /// The for the operation. /// A resulting in the for the operation. [HttpGet] + [AllowAnonymous] public async ValueTask Process([FromQuery] string data, CancellationToken cancellationToken) { // Nothing to see here diff --git a/src/Tgstation.Server.Host/Controllers/ByondController.cs b/src/Tgstation.Server.Host/Controllers/ByondController.cs index 2c0281b99be..41cccc1584d 100644 --- a/src/Tgstation.Server.Host/Controllers/ByondController.cs +++ b/src/Tgstation.Server.Host/Controllers/ByondController.cs @@ -20,6 +20,7 @@ using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.Transfer; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Controllers { @@ -50,23 +51,26 @@ public sealed class ByondController : InstanceRequiredController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . /// The value of . /// The value of . + /// The for the . public ByondController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, IInstanceManager instanceManager, IJobManager jobManager, - IFileTransferTicketProvider fileTransferService) + IFileTransferTicketProvider fileTransferService, + IApiHeadersProvider apiHeadersProvider) : base( databaseContext, - authenticationContextFactory, + authenticationContext, logger, - instanceManager) + instanceManager, + apiHeadersProvider) { this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); diff --git a/src/Tgstation.Server.Host/Controllers/ChatController.cs b/src/Tgstation.Server.Host/Controllers/ChatController.cs index b88ab73654e..7f9682c43aa 100644 --- a/src/Tgstation.Server.Host/Controllers/ChatController.cs +++ b/src/Tgstation.Server.Host/Controllers/ChatController.cs @@ -24,7 +24,7 @@ using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; - +using Tgstation.Server.Host.Utils; using Z.EntityFramework.Plus; namespace Tgstation.Server.Host.Controllers @@ -40,19 +40,22 @@ public sealed class ChatController : InstanceRequiredController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . + /// The for the . public ChatController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, - IInstanceManager instanceManager) + IInstanceManager instanceManager, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, logger, - instanceManager) + instanceManager, + apiHeaders) { } diff --git a/src/Tgstation.Server.Host/Controllers/ComponentInterfacingController.cs b/src/Tgstation.Server.Host/Controllers/ComponentInterfacingController.cs index 45e3829bbb2..eef1af8b180 100644 --- a/src/Tgstation.Server.Host/Controllers/ComponentInterfacingController.cs +++ b/src/Tgstation.Server.Host/Controllers/ComponentInterfacingController.cs @@ -40,18 +40,21 @@ public abstract class ComponentInterfacingController : ApiController /// /// The value of . /// The for the . - /// The for the . + /// The for the . /// The for the . + /// The for the . /// The value of . protected ComponentInterfacingController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, IInstanceManager instanceManager, - bool useInstanceRequestHeader = false) + IApiHeadersProvider apiHeaders, + bool useInstanceRequestHeader) : base( databaseContext, - authenticationContextFactory, + authenticationContext, + apiHeaders, logger, true) { diff --git a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs index adebfe01030..ea26922fd88 100644 --- a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs +++ b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs @@ -17,6 +17,7 @@ using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Controllers { @@ -35,21 +36,24 @@ public sealed class ConfigurationController : InstanceRequiredController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . /// The value of . + /// The for the . public ConfigurationController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, IInstanceManager instanceManager, - IIOManager ioManager) + IIOManager ioManager, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, logger, - instanceManager) + instanceManager, + apiHeaders) { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); } diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index f4613f81d00..5bceb479d5c 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -47,23 +47,26 @@ public sealed class DreamDaemonController : InstanceRequiredController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . /// The value of . /// The value of . + /// The for the . public DreamDaemonController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, IInstanceManager instanceManager, IJobManager jobManager, - IPortAllocator portAllocator) + IPortAllocator portAllocator, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, logger, - instanceManager) + instanceManager, + apiHeaders) { this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); this.portAllocator = portAllocator ?? throw new ArgumentNullException(nameof(portAllocator)); diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index acb4d53dac9..c2e4874ea4b 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -44,23 +44,26 @@ public sealed class DreamMakerController : InstanceRequiredController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . /// The value of . /// The value of . + /// The for the . public DreamMakerController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, IInstanceManager instanceManager, IJobManager jobManager, - IPortAllocator portAllocator) + IPortAllocator portAllocator, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, logger, - instanceManager) + instanceManager, + apiHeaders) { this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); this.portAllocator = portAllocator ?? throw new ArgumentNullException(nameof(portAllocator)); diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index af1d4b90dd7..dda8e793369 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Net; using System.Threading; @@ -27,6 +27,7 @@ using Tgstation.Server.Host.Security.OAuth; using Tgstation.Server.Host.Swarm; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Controllers { @@ -95,7 +96,7 @@ public sealed class HomeController : ApiController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The value of . /// The value of . /// The value of . @@ -108,9 +109,10 @@ public sealed class HomeController : ApiController /// The containing the value of . /// The containing the value of . /// The for the . + /// The for the . public HomeController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ITokenFactory tokenFactory, ISystemIdentityFactory systemIdentityFactory, ICryptographySuite cryptographySuite, @@ -122,10 +124,12 @@ public HomeController( IServerControl serverControl, IOptions generalConfigurationOptions, IOptions controlPanelConfigurationOptions, - ILogger logger) + ILogger logger, + IApiHeadersProvider apiHeadersProvider) : base( databaseContext, - authenticationContextFactory, + authenticationContext, + apiHeadersProvider, logger, false) { diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index 6a30e253998..c5e472b7ade 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -82,7 +82,7 @@ public sealed class InstanceController : ComponentInterfacingController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . /// The value of . @@ -91,9 +91,10 @@ public sealed class InstanceController : ComponentInterfacingController /// The value of . /// The containing the value of . /// The containing the value of . + /// The for the . public InstanceController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, IInstanceManager instanceManager, IJobManager jobManager, @@ -101,12 +102,15 @@ public InstanceController( IPortAllocator portAllocator, IPlatformIdentifier platformIdentifier, IOptions generalConfigurationOptions, - IOptions swarmConfigurationOptions) + IOptions swarmConfigurationOptions, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, logger, - instanceManager) + instanceManager, + apiHeaders, + false) { this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); diff --git a/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs b/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs index 2d9059b2047..5c1a91e6ba4 100644 --- a/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs @@ -18,6 +18,7 @@ using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Utils; using Z.EntityFramework.Plus; @@ -33,19 +34,22 @@ public sealed class InstancePermissionSetController : InstanceRequiredController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . + /// The for the . public InstancePermissionSetController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, - IInstanceManager instanceManager) + IInstanceManager instanceManager, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, logger, - instanceManager) + instanceManager, + apiHeaders) { } diff --git a/src/Tgstation.Server.Host/Controllers/InstanceRequiredController.cs b/src/Tgstation.Server.Host/Controllers/InstanceRequiredController.cs index 1f119500f59..cdf3f0e139c 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceRequiredController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceRequiredController.cs @@ -3,6 +3,7 @@ using Tgstation.Server.Host.Components; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Controllers { @@ -15,19 +16,22 @@ public abstract class InstanceRequiredController : ComponentInterfacingControlle /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . + /// The for the . protected InstanceRequiredController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, - IInstanceManager instanceManager) + IInstanceManager instanceManager, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, logger, instanceManager, + apiHeaders, true) { } diff --git a/src/Tgstation.Server.Host/Controllers/JobController.cs b/src/Tgstation.Server.Host/Controllers/JobController.cs index 6e0817b5534..9b468546bcb 100644 --- a/src/Tgstation.Server.Host/Controllers/JobController.cs +++ b/src/Tgstation.Server.Host/Controllers/JobController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -17,6 +17,7 @@ using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Controllers { @@ -35,21 +36,24 @@ public sealed class JobController : InstanceRequiredController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . /// The value of . + /// The for the . public JobController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, IInstanceManager instanceManager, - IJobManager jobManager) + IJobManager jobManager, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, logger, - instanceManager) + instanceManager, + apiHeaders) { this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); } diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 0b02d7d0c15..d701142bfd6 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -22,6 +22,7 @@ using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Controllers { @@ -46,23 +47,26 @@ public sealed class RepositoryController : InstanceRequiredController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . /// The value of . /// The value of . + /// The for the . public RepositoryController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ILogger logger, IInstanceManager instanceManager, ILoggerFactory loggerFactory, - IJobManager jobManager) + IJobManager jobManager, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, logger, - instanceManager) + instanceManager, + apiHeaders) { this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); diff --git a/src/Tgstation.Server.Host/Controllers/SwarmController.cs b/src/Tgstation.Server.Host/Controllers/SwarmController.cs index fb81784b12c..15b3bae421a 100644 --- a/src/Tgstation.Server.Host/Controllers/SwarmController.cs +++ b/src/Tgstation.Server.Host/Controllers/SwarmController.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -25,6 +26,7 @@ namespace Tgstation.Server.Host.Controllers /// [Route(SwarmConstants.ControllerRoute)] [ApiExplorerSettings(IgnoreApi = true)] + [AllowAnonymous] // We have custom private key auth public sealed class SwarmController : ApiControllerBase { /// diff --git a/src/Tgstation.Server.Host/Controllers/TransferController.cs b/src/Tgstation.Server.Host/Controllers/TransferController.cs index 489b3525cb8..8a0a2ebcd3f 100644 --- a/src/Tgstation.Server.Host/Controllers/TransferController.cs +++ b/src/Tgstation.Server.Host/Controllers/TransferController.cs @@ -14,6 +14,7 @@ using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.Transfer; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Controllers { @@ -33,17 +34,20 @@ public sealed class TransferController : ApiController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The value of . /// The for the . + /// The for the . public TransferController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, IFileTransferStreamHandler fileTransferService, - ILogger logger) + ILogger logger, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, + apiHeaders, logger, true) { diff --git a/src/Tgstation.Server.Host/Controllers/UserController.cs b/src/Tgstation.Server.Host/Controllers/UserController.cs index 50c071ee997..88ce9d1abb9 100644 --- a/src/Tgstation.Server.Host/Controllers/UserController.cs +++ b/src/Tgstation.Server.Host/Controllers/UserController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -20,6 +20,7 @@ using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Controllers { @@ -48,21 +49,24 @@ public sealed class UserController : ApiController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The value of . /// The value of . /// The for the . /// The containing the value of . + /// The for the . public UserController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, ISystemIdentityFactory systemIdentityFactory, ICryptographySuite cryptographySuite, ILogger logger, - IOptions generalConfigurationOptions) + IOptions generalConfigurationOptions, + IApiHeadersProvider apiHeaders) : base( databaseContext, - authenticationContextFactory, + authenticationContext, + apiHeaders, logger, true) { diff --git a/src/Tgstation.Server.Host/Controllers/UserGroupController.cs b/src/Tgstation.Server.Host/Controllers/UserGroupController.cs index a23be84706c..589c1ecb462 100644 --- a/src/Tgstation.Server.Host/Controllers/UserGroupController.cs +++ b/src/Tgstation.Server.Host/Controllers/UserGroupController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -19,6 +19,7 @@ using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Utils; using Z.EntityFramework.Plus; @@ -39,19 +40,22 @@ public class UserGroupController : ApiController /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The containing the value of . /// The for the . + /// The for the . public UserGroupController( IDatabaseContext databaseContext, - IAuthenticationContextFactory authenticationContextFactory, + IAuthenticationContext authenticationContext, IOptions generalConfigurationOptions, - ILogger logger) + ILogger logger, + IApiHeadersProvider apiHeaders) : base( - databaseContext, - authenticationContextFactory, - logger, - true) + databaseContext, + authenticationContext, + apiHeaders, + logger, + true) { generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); } diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 459272d77ed..7253e90d687 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -3,16 +3,19 @@ using System.Globalization; using System.IdentityModel.Tokens.Jwt; using System.Linq; +using System.Threading.Tasks; using Cyberboss.AspNetCore.AsyncInitializer; using Elastic.CommonSchema.Serilog; +using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Cors.Infrastructure; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Identity; +using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.AspNetCore.Mvc.Infrastructure; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -115,7 +118,7 @@ public Application( } /// - /// Configure the 's services. + /// Configure the 's . /// /// The to configure. /// The needed for configuration. @@ -217,34 +220,24 @@ public void ConfigureServices( postSetupServices.InternalConfiguration, postSetupServices.FileLoggingConfiguration); - // configure bearer token validation - services - .AddAuthentication(JwtBearerDefaults.AuthenticationScheme) - .AddJwtBearer(jwtBearerOptions => - { - // this line isn't actually run until the first request is made - // at that point tokenFactory will be populated - jwtBearerOptions.TokenValidationParameters = tokenFactory.ValidationParameters; - jwtBearerOptions.Events = new JwtBearerEvents - { - // Application is our composition root so this monstrosity of a line is okay - // At least, that's what I tell myself to sleep at night - OnTokenValidated = ctx => ctx - .HttpContext - .RequestServices - .GetRequiredService() - .InjectClaimsIntoContext( - ctx, - ctx.HttpContext.RequestAborted), - }; - }); - - // WARNING: STATIC CODE - // fucking prevents converting 'sub' to M$ bs - // can't be done in the above lambda, that's too late - JwtSecurityTokenHandler.DefaultInboundClaimTypeMap.Clear(); + // configure authentication pipeline + ConfigureAuthenticationPipeline(services); // add mvc, configure the json serializer settings + var jsonVersionConverterList = new List + { + new VersionConverter(), + }; + + void ConfigureNewtonsoftJsonSerializerSettingsForApi(JsonSerializerSettings settings) + { + settings.NullValueHandling = NullValueHandling.Ignore; + settings.CheckAdditionalContent = true; + settings.MissingMemberHandling = MissingMemberHandling.Error; + settings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; + settings.Converters = jsonVersionConverterList; + } + services .AddMvc(options => { @@ -254,14 +247,7 @@ public void ConfigureServices( .AddNewtonsoftJson(options => { options.AllowInputFormatterExceptionMessages = true; - options.SerializerSettings.NullValueHandling = NullValueHandling.Ignore; - options.SerializerSettings.CheckAdditionalContent = true; - options.SerializerSettings.MissingMemberHandling = MissingMemberHandling.Error; - options.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; - options.SerializerSettings.Converters = new List - { - new VersionConverter(), - }; + ConfigureNewtonsoftJsonSerializerSettingsForApi(options.SerializerSettings); }); if (postSetupServices.GeneralConfiguration.HostApiDocumentation) @@ -322,9 +308,7 @@ void AddTypedContext() services.AddSingleton(); services.AddSingleton(); - // configure security services - services.AddScoped(); - services.AddScoped(); + // configure other security services services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -546,5 +530,46 @@ public void Configure( /// protected override void ConfigureHostedService(IServiceCollection services) => services.AddSingleton(x => x.GetRequiredService()); + + /// + /// Configure the for the authentication pipeline. + /// + /// The to configure. + void ConfigureAuthenticationPipeline(IServiceCollection services) + { + services.AddHttpContextAccessor(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(provider => provider.GetRequiredService()); + services.AddScoped(provider => + { + return provider.GetRequiredService().CurrentAuthenticationContext; + }); + services.AddScoped(); + services.AddScoped(); + + services + .AddAuthentication(JwtBearerDefaults.AuthenticationScheme) + .AddJwtBearer(jwtBearerOptions => + { + // this line isn't actually run until the first request is made + // at that point tokenFactory will be populated + jwtBearerOptions.TokenValidationParameters = tokenFactory?.ValidationParameters ?? throw new InvalidOperationException("tokenFactory not initialized!"); + jwtBearerOptions.Events = new JwtBearerEvents + { + OnTokenValidated = tokenValidatedContext => + { + var acf = tokenValidatedContext.HttpContext.RequestServices.GetRequiredService(); + acf.SetTokenNbf(tokenValidatedContext.SecurityToken.ValidFrom); + return Task.CompletedTask; + }, + }; + }); + + // WARNING: STATIC CODE + // fucking prevents converting 'sub' to M$ bs + // can't be done in the above lambda, that's too late + JwtSecurityTokenHandler.DefaultInboundClaimTypeMap.Clear(); + } } } diff --git a/src/Tgstation.Server.Host/Security/AuthenticationContext.cs b/src/Tgstation.Server.Host/Security/AuthenticationContext.cs index 105c7f98371..5ed4dafe3f1 100644 --- a/src/Tgstation.Server.Host/Security/AuthenticationContext.cs +++ b/src/Tgstation.Server.Host/Security/AuthenticationContext.cs @@ -7,19 +7,22 @@ namespace Tgstation.Server.Host.Security { /// - sealed class AuthenticationContext : IAuthenticationContext + sealed class AuthenticationContext : IAuthenticationContext, IDisposable { /// - public User User { get; } + public bool Valid { get; private set; } /// - public PermissionSet PermissionSet { get; } + public User User { get; private set; } /// - public InstancePermissionSet InstancePermissionSet { get; } + public PermissionSet PermissionSet { get; private set; } /// - public ISystemIdentity SystemIdentity { get; } + public InstancePermissionSet InstancePermissionSet { get; private set; } + + /// + public ISystemIdentity SystemIdentity { get; private set; } /// /// Initializes a new instance of the class. @@ -28,13 +31,16 @@ public AuthenticationContext() { } + /// + public void Dispose() => SystemIdentity?.Dispose(); + /// - /// Initializes a new instance of the class. + /// Initializes the . /// /// The value of . /// The value of . /// The value of . - public AuthenticationContext(ISystemIdentity systemIdentity, User user, InstancePermissionSet instanceUser) + public void Initialize(ISystemIdentity systemIdentity, User user, InstancePermissionSet instanceUser) { User = user ?? throw new ArgumentNullException(nameof(user)); if (systemIdentity == null && User.SystemIdentifier != null) @@ -44,10 +50,9 @@ public AuthenticationContext(ISystemIdentity systemIdentity, User user, Instance ?? throw new ArgumentException("No PermissionSet provider", nameof(user)); InstancePermissionSet = instanceUser; SystemIdentity = systemIdentity; - } - /// - public void Dispose() => SystemIdentity?.Dispose(); + Valid = true; + } /// public ulong GetRight(RightsType rightsType) @@ -69,10 +74,15 @@ public ulong GetRight(RightsType rightsType) var prop = typeToCheck.GetProperties().Where(x => x.PropertyType == nullableRightsType).First(); - var right = prop.GetMethod.Invoke(isInstance ? InstancePermissionSet : PermissionSet, Array.Empty()); + var right = prop.GetMethod.Invoke( + isInstance + ? InstancePermissionSet + : PermissionSet, + Array.Empty()); if (right == null) throw new InvalidOperationException("A user right was null!"); + return (ulong)right; } } diff --git a/src/Tgstation.Server.Host/Security/AuthenticationContextAuthorizationFilter.cs b/src/Tgstation.Server.Host/Security/AuthenticationContextAuthorizationFilter.cs new file mode 100644 index 00000000000..562f49f443c --- /dev/null +++ b/src/Tgstation.Server.Host/Security/AuthenticationContextAuthorizationFilter.cs @@ -0,0 +1,53 @@ +using System; +using System.Security.Claims; + +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.Extensions.Logging; + +namespace Tgstation.Server.Host.Security +{ + /// + /// An that maps s using an . + /// + sealed class AuthenticationContextAuthorizationFilter : IAuthorizationFilter + { + /// + /// The for the . + /// + readonly IAuthenticationContext authenticationContext; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + public AuthenticationContextAuthorizationFilter(IAuthenticationContext authenticationContext, ILogger logger) + { + this.authenticationContext = authenticationContext ?? throw new ArgumentNullException(nameof(authenticationContext)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public void OnAuthorization(AuthorizationFilterContext context) + { + if (!authenticationContext.Valid) + { + logger.LogTrace("authenticationContext is invalid!"); + context.Result = new UnauthorizedResult(); + return; + } + + if (authenticationContext.User.Enabled.Value) + return; + + logger.LogTrace("authenticationContext is for a disabled user!"); + context.Result = new ForbidResult(); + } + } +} diff --git a/src/Tgstation.Server.Host/Security/AuthenticationContextClaimsTransformation.cs b/src/Tgstation.Server.Host/Security/AuthenticationContextClaimsTransformation.cs new file mode 100644 index 00000000000..6cd5f303053 --- /dev/null +++ b/src/Tgstation.Server.Host/Security/AuthenticationContextClaimsTransformation.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IdentityModel.Tokens.Jwt; +using System.Security.Claims; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Authentication; + +using Tgstation.Server.Api; +using Tgstation.Server.Api.Rights; +using Tgstation.Server.Host.Utils; + +namespace Tgstation.Server.Host.Security +{ + /// + /// A that maps s using an . + /// + sealed class AuthenticationContextClaimsTransformation : IClaimsTransformation + { + /// + /// The for the . + /// + readonly IAuthenticationContextFactory authenticationContextFactory; + + /// + /// The for the . + /// + readonly ApiHeaders apiHeaders; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The containing the value of . + public AuthenticationContextClaimsTransformation(IAuthenticationContextFactory authenticationContextFactory, IApiHeadersProvider apiHeadersProvider) + { + this.authenticationContextFactory = authenticationContextFactory ?? throw new ArgumentNullException(nameof(authenticationContextFactory)); + ArgumentNullException.ThrowIfNull(apiHeadersProvider); + apiHeaders = apiHeadersProvider.ApiHeaders; + } + + /// + public async Task TransformAsync(ClaimsPrincipal principal) + { + ArgumentNullException.ThrowIfNull(principal); + + var userIdClaim = principal.FindFirst(JwtRegisteredClaimNames.Sub); + if (userIdClaim == default) + throw new InvalidOperationException("Missing required claim!"); + + long userId; + try + { + userId = Int64.Parse(userIdClaim.Value, CultureInfo.InvariantCulture); + } + catch (Exception e) + { + throw new InvalidOperationException("Failed to parse user ID!", e); + } + + var authenticationContext = await authenticationContextFactory.CreateAuthenticationContext( + userId, + apiHeaders?.InstanceId, + CancellationToken.None); // DCT: None available + + if (authenticationContext.Valid) + { + var enumerator = Enum.GetValues(typeof(RightsType)); + var claims = new List(); + foreach (RightsType rightType in enumerator) + { + // if there's no instance user, do a weird thing and add all the instance roles + // we need it so we can get to OnActionExecutionAsync where we can properly decide between BadRequest and Forbid + // if user is null that means they got the token with an expired password + var rightAsULong = authenticationContext.User == null + || (RightsHelper.IsInstanceRight(rightType) && authenticationContext.InstancePermissionSet == null) + ? ~0UL + : authenticationContext.GetRight(rightType); + var rightEnum = RightsHelper.RightToType(rightType); + var right = (Enum)Enum.ToObject(rightEnum, rightAsULong); + foreach (Enum enumeratedRight in Enum.GetValues(rightEnum)) + if (right.HasFlag(enumeratedRight)) + claims.Add( + new Claim( + ClaimTypes.Role, + RightsHelper.RoleName(rightType, enumeratedRight))); + } + + principal.AddIdentity(new ClaimsIdentity(claims)); + } + + return principal; + } + } +} diff --git a/src/Tgstation.Server.Host/Security/AuthenticationContextFactory.cs b/src/Tgstation.Server.Host/Security/AuthenticationContextFactory.cs index 7288dc00cfb..3b5035e69c2 100644 --- a/src/Tgstation.Server.Host/Security/AuthenticationContextFactory.cs +++ b/src/Tgstation.Server.Host/Security/AuthenticationContextFactory.cs @@ -13,11 +13,13 @@ namespace Tgstation.Server.Host.Security { - /// + /// sealed class AuthenticationContextFactory : IAuthenticationContextFactory, IDisposable { - /// - public IAuthenticationContext CurrentAuthenticationContext { get; private set; } + /// + /// The the created. + /// + public IAuthenticationContext CurrentAuthenticationContext => currentAuthenticationContext; /// /// The for the . @@ -39,6 +41,21 @@ sealed class AuthenticationContextFactory : IAuthenticationContextFactory, IDisp /// readonly SwarmConfiguration swarmConfiguration; + /// + /// Backing field for . + /// + readonly AuthenticationContext currentAuthenticationContext; + + /// + /// The the request's token must be valid after. + /// + DateTimeOffset? validAfter; + + /// + /// 1 if was initialized, 0 otherwise. + /// + int initialized; + /// /// Initializes a new instance of the class. /// @@ -56,17 +73,34 @@ public AuthenticationContextFactory( this.identityCache = identityCache ?? throw new ArgumentNullException(nameof(identityCache)); swarmConfiguration = swarmConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(swarmConfigurationOptions)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + currentAuthenticationContext = new AuthenticationContext(); } /// - public void Dispose() => CurrentAuthenticationContext?.Dispose(); + public void Dispose() => currentAuthenticationContext.Dispose(); + + /// + /// Populate with a given . + /// + /// The an issued token is not valid before. + public void SetTokenNbf(DateTimeOffset tokenNbf) + { + if (validAfter.HasValue) + throw new InvalidOperationException("SetTokenNbf called multiple times!"); + + validAfter = tokenNbf; + } /// - public async ValueTask CreateAuthenticationContext(long userId, long? instanceId, DateTimeOffset validAfter, CancellationToken cancellationToken) + public async ValueTask CreateAuthenticationContext(long userId, long? instanceId, CancellationToken cancellationToken) { - if (CurrentAuthenticationContext != null) + if (Interlocked.Exchange(ref initialized, 1) != 0) throw new InvalidOperationException("Authentication context has already been loaded"); + if (!validAfter.HasValue) + throw new InvalidOperationException("SetTokenNbf has not been called!"); + var user = await databaseContext .Users .AsQueryable() @@ -79,9 +113,8 @@ public async ValueTask CreateAuthenticationContext(long userId, long? instanceId .FirstOrDefaultAsync(cancellationToken); if (user == default) { - logger.LogWarning("Unable to find user with ID {0}!", userId); - CurrentAuthenticationContext = new AuthenticationContext(); - return; + logger.LogWarning("Unable to find user with ID {userId}!", userId); + return currentAuthenticationContext; } ISystemIdentity systemIdentity; @@ -89,11 +122,10 @@ public async ValueTask CreateAuthenticationContext(long userId, long? instanceId systemIdentity = identityCache.LoadCachedIdentity(user); else { - if (user.LastPasswordUpdate.HasValue && user.LastPasswordUpdate > validAfter) + if (user.LastPasswordUpdate.HasValue && user.LastPasswordUpdate > validAfter.Value) { - logger.LogDebug("Rejecting token for user {0} created before last password update: {1}", userId, user.LastPasswordUpdate.Value); - CurrentAuthenticationContext = new AuthenticationContext(); - return; + logger.LogDebug("Rejecting token for user {userId} created before last password update: {lastPasswordUpdate}", userId, user.LastPasswordUpdate.Value); + return currentAuthenticationContext; } systemIdentity = null; @@ -112,13 +144,14 @@ public async ValueTask CreateAuthenticationContext(long userId, long? instanceId .FirstOrDefaultAsync(cancellationToken); if (instancePermissionSet == null) - logger.LogDebug("User {0} does not have permissions on instance {1}!", userId, instanceId.Value); + logger.LogDebug("User {userId} does not have permissions on instance {instanceId}!", userId, instanceId.Value); } - CurrentAuthenticationContext = new AuthenticationContext( + currentAuthenticationContext.Initialize( systemIdentity, user, instancePermissionSet); + return currentAuthenticationContext; } catch { diff --git a/src/Tgstation.Server.Host/Security/ClaimsInjector.cs b/src/Tgstation.Server.Host/Security/ClaimsInjector.cs deleted file mode 100644 index a441af0f16c..00000000000 --- a/src/Tgstation.Server.Host/Security/ClaimsInjector.cs +++ /dev/null @@ -1,95 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IdentityModel.Tokens.Jwt; -using System.Security.Claims; -using System.Threading; -using System.Threading.Tasks; - -using Microsoft.AspNetCore.Authentication.JwtBearer; -using Microsoft.AspNetCore.Http; - -using Tgstation.Server.Api; -using Tgstation.Server.Api.Rights; - -namespace Tgstation.Server.Host.Security -{ - /// - sealed class ClaimsInjector : IClaimsInjector - { - /// - /// The for the . - /// - readonly IAuthenticationContextFactory authenticationContextFactory; - - /// - /// Initializes a new instance of the class. - /// - /// The value of . - public ClaimsInjector(IAuthenticationContextFactory authenticationContextFactory) - { - this.authenticationContextFactory = authenticationContextFactory ?? throw new ArgumentNullException(nameof(authenticationContextFactory)); - } - - /// - public async Task InjectClaimsIntoContext(TokenValidatedContext tokenValidatedContext, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(tokenValidatedContext); - - // Find the user id in the token - var userIdClaim = tokenValidatedContext.Principal.FindFirst(JwtRegisteredClaimNames.Sub); - if (userIdClaim == default) - throw new InvalidOperationException("Missing required claim!"); - - long userId; - try - { - userId = Int64.Parse(userIdClaim.Value, CultureInfo.InvariantCulture); - } - catch (Exception e) - { - throw new InvalidOperationException("Failed to parse user ID!", e); - } - - ApiHeaders apiHeaders; - try - { - apiHeaders = new ApiHeaders(tokenValidatedContext.HttpContext.Request.GetTypedHeaders()); - } - catch (HeadersException) - { - // we are not responsible for handling header validation issues - return; - } - - // This populates the CurrentAuthenticationContext field for use by us and subsequent controllers - await authenticationContextFactory.CreateAuthenticationContext( - userId, - apiHeaders.InstanceId, - tokenValidatedContext.SecurityToken.ValidFrom, - cancellationToken); - - var authenticationContext = authenticationContextFactory.CurrentAuthenticationContext; - - var enumerator = Enum.GetValues(typeof(RightsType)); - var claims = new List(); - foreach (RightsType rightType in enumerator) - { - // if there's no instance user, do a weird thing and add all the instance roles - // we need it so we can get to OnActionExecutionAsync where we can properly decide between BadRequest and Forbid - // if user is null that means they got the token with an expired password - var rightAsULong = authenticationContext.User == null - || (RightsHelper.IsInstanceRight(rightType) && authenticationContext.InstancePermissionSet == null) - ? ~0UL - : authenticationContext.GetRight(rightType); - var rightEnum = RightsHelper.RightToType(rightType); - var right = (Enum)Enum.ToObject(rightEnum, rightAsULong); - foreach (Enum enumeratedRight in Enum.GetValues(rightEnum)) - if (right.HasFlag(enumeratedRight)) - claims.Add(new Claim(ClaimTypes.Role, RightsHelper.RoleName(rightType, enumeratedRight))); - } - - tokenValidatedContext.Principal.AddIdentity(new ClaimsIdentity(claims)); - } - } -} diff --git a/src/Tgstation.Server.Host/Security/IAuthenticationContext.cs b/src/Tgstation.Server.Host/Security/IAuthenticationContext.cs index 55e68a4cc19..36abee2b2c5 100644 --- a/src/Tgstation.Server.Host/Security/IAuthenticationContext.cs +++ b/src/Tgstation.Server.Host/Security/IAuthenticationContext.cs @@ -1,6 +1,4 @@ -using System; - -using Tgstation.Server.Api.Rights; +using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Security @@ -8,8 +6,13 @@ namespace Tgstation.Server.Host.Security /// /// Represents the currently authenticated . /// - public interface IAuthenticationContext : IDisposable + public interface IAuthenticationContext { + /// + /// If the is for a valid login. + /// + public bool Valid { get; } + /// /// The authenticated user. /// diff --git a/src/Tgstation.Server.Host/Security/IAuthenticationContextFactory.cs b/src/Tgstation.Server.Host/Security/IAuthenticationContextFactory.cs index 9017765615f..02337005850 100644 --- a/src/Tgstation.Server.Host/Security/IAuthenticationContextFactory.cs +++ b/src/Tgstation.Server.Host/Security/IAuthenticationContextFactory.cs @@ -1,5 +1,4 @@ -using System; -using System.Threading; +using System.Threading; using System.Threading.Tasks; namespace Tgstation.Server.Host.Security @@ -10,18 +9,12 @@ namespace Tgstation.Server.Host.Security public interface IAuthenticationContextFactory { /// - /// The the created. + /// Create an in the request pipeline for a given and . /// - IAuthenticationContext CurrentAuthenticationContext { get; } - - /// - /// Create an to populate . - /// - /// The of the . - /// The of the operation. - /// The the resulting 's password must be valid after. + /// The of the . + /// The of the for the operation. /// The for the operation. - /// A representing the running operation. - ValueTask CreateAuthenticationContext(long userId, long? instanceId, DateTimeOffset validAfter, CancellationToken cancellationToken); + /// A resulting in the created . + ValueTask CreateAuthenticationContext(long userId, long? instanceId, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Security/IClaimsInjector.cs b/src/Tgstation.Server.Host/Security/IClaimsInjector.cs deleted file mode 100644 index 94899d81ffb..00000000000 --- a/src/Tgstation.Server.Host/Security/IClaimsInjector.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; - -using Microsoft.AspNetCore.Authentication.JwtBearer; - -namespace Tgstation.Server.Host.Security -{ - /// - /// For injecting s that can look for. - /// - interface IClaimsInjector - { - /// - /// Setup the s for a given . - /// - /// The containing the and of the request and the to add s to. - /// The for the operation. - /// A representing the running operation. - Task InjectClaimsIntoContext(TokenValidatedContext tokenValidatedContext, CancellationToken cancellationToken); - } -} diff --git a/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs b/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs new file mode 100644 index 00000000000..c59d2d67d0a --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs @@ -0,0 +1,39 @@ +using System; + +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Tgstation.Server.Api; + +namespace Tgstation.Server.Host.Utils +{ + /// + sealed class ApiHeadersProvider : IApiHeadersProvider + { + /// + public ApiHeaders ApiHeaders { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The for accessing the . + /// The to write to. + public ApiHeadersProvider(IHttpContextAccessor httpContextAccessor, ILogger logger) + { + ArgumentNullException.ThrowIfNull(httpContextAccessor); + + if (httpContextAccessor.HttpContext == null) + throw new InvalidOperationException("httpContextAccessor has no HttpContext!"); + + var request = httpContextAccessor.HttpContext.Request; + try + { + ApiHeaders = new ApiHeaders(request.GetTypedHeaders()); + } + catch (HeadersException ex) + { + // we are not responsible for handling header validation issues + logger.LogTrace(ex, "Failed to validated API request headers!"); + } + } + } +} diff --git a/src/Tgstation.Server.Host/Utils/IApiHeadersProvider.cs b/src/Tgstation.Server.Host/Utils/IApiHeadersProvider.cs new file mode 100644 index 00000000000..31747862be0 --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/IApiHeadersProvider.cs @@ -0,0 +1,15 @@ +using Tgstation.Server.Api; + +namespace Tgstation.Server.Host.Utils +{ + /// + /// Provides . + /// + public interface IApiHeadersProvider + { + /// + /// The created , if any. + /// + ApiHeaders ApiHeaders { get; } + } +} diff --git a/tests/Tgstation.Server.Host.Tests/Security/TestAuthenticationContext.cs b/tests/Tgstation.Server.Host.Tests/Security/TestAuthenticationContext.cs index 090f10f1ec3..9145401b347 100644 --- a/tests/Tgstation.Server.Host.Tests/Security/TestAuthenticationContext.cs +++ b/tests/Tgstation.Server.Host.Tests/Security/TestAuthenticationContext.cs @@ -1,4 +1,4 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; @@ -16,7 +16,7 @@ public sealed class TestAuthenticationContext [TestMethod] public void TestConstruction() { - Assert.ThrowsException(() => new AuthenticationContext(null, null, null)); + Assert.ThrowsException(() => new AuthenticationContext().Initialize(null, null, null)); var mockSystemIdentity = new Mock(); var user = new User() @@ -24,18 +24,18 @@ public void TestConstruction() PermissionSet = new PermissionSet() }; - var authContext = new AuthenticationContext(null, user, null); - Assert.ThrowsException(() => new AuthenticationContext(mockSystemIdentity.Object, null, null)); + var authContext = new AuthenticationContext(); + Assert.ThrowsException(() => new AuthenticationContext().Initialize(mockSystemIdentity.Object, null, null)); var instanceUser = new InstancePermissionSet(); - Assert.ThrowsException(() => new AuthenticationContext(null, null, instanceUser)); - Assert.ThrowsException(() => new AuthenticationContext(mockSystemIdentity.Object, null, instanceUser)); - authContext = new AuthenticationContext(mockSystemIdentity.Object, user, null); - authContext = new AuthenticationContext(null, user, instanceUser); - authContext = new AuthenticationContext(mockSystemIdentity.Object, user, instanceUser); + Assert.ThrowsException(() => new AuthenticationContext().Initialize(null, null, instanceUser)); + Assert.ThrowsException(() => new AuthenticationContext().Initialize(mockSystemIdentity.Object, null, instanceUser)); + new AuthenticationContext().Initialize(mockSystemIdentity.Object, user, null); + new AuthenticationContext().Initialize(null, user, instanceUser); + new AuthenticationContext().Initialize(mockSystemIdentity.Object, user, instanceUser); user.SystemIdentifier = "root"; - Assert.ThrowsException(() => new AuthenticationContext(null, user, null)); + Assert.ThrowsException(() => new AuthenticationContext().Initialize(null, user, null)); } @@ -47,7 +47,8 @@ public void TestGetRightsGeneric() PermissionSet = new PermissionSet() }; var instanceUser = new InstancePermissionSet(); - var authContext = new AuthenticationContext(null, user, instanceUser); + var authContext = new AuthenticationContext(); + authContext.Initialize(null, user, instanceUser); user.PermissionSet.AdministrationRights = AdministrationRights.WriteUsers; instanceUser.ByondRights = ByondRights.InstallOfficialOrChangeActiveVersion | ByondRights.ReadActive; From 6dbf357b8a0e52f4545feadf9dbcfb1048d2e671 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 16:23:29 -0400 Subject: [PATCH 073/138] Silence VS message about a `new()` expression --- .../Components/Chat/Providers/IrcProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs index c4449e00ea7..533f5010a0e 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs @@ -352,7 +352,7 @@ await SendMessage( dbChannel, new List { - new ChannelRepresentation + new () { RealId = id.Value, IsAdminChannel = dbChannel.IsAdminChannel == true, From 1d0371f84a98c3f75dd1b5961af974d094cadd6d Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 16:23:52 -0400 Subject: [PATCH 074/138] Make `TestServerService` more analyzer friendly --- .../TestServerService.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Tgstation.Server.Host.Service.Tests/TestServerService.cs b/tests/Tgstation.Server.Host.Service.Tests/TestServerService.cs index e1f604a5334..855870d4c82 100644 --- a/tests/Tgstation.Server.Host.Service.Tests/TestServerService.cs +++ b/tests/Tgstation.Server.Host.Service.Tests/TestServerService.cs @@ -35,7 +35,7 @@ public void TestRun() var mockWatchdog = new Mock(); var args = Array.Empty(); CancellationToken cancellationToken = default; - ValueTask? signalCheckerTask = null; + Task signalCheckerTask = null; var childStarted = false; ISignalChecker signalChecker = null; @@ -46,8 +46,8 @@ public void TestRun() { childStarted = true; return (123, Task.CompletedTask); - }, cancellationToken); - }).Returns(ValueTask.FromResult(true)).Verifiable(); + }, cancellationToken).AsTask(); + }).ReturnsAsync(true).Verifiable(); var mockWatchdogFactory = new Mock(); mockWatchdogFactory.Setup(x => x.CreateWatchdog(It.IsNotNull(), It.IsNotNull())) @@ -67,7 +67,7 @@ public void TestRun() } mockWatchdogFactory.VerifyAll(); - Assert.IsTrue(signalCheckerTask.Value.IsCompleted); + Assert.IsTrue(signalCheckerTask.IsCompleted); } } } From ed618de29ec33f5578a42d9e613eda2491ed2652 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 16:39:56 -0400 Subject: [PATCH 075/138] ODR use of "Bearer" string --- src/Tgstation.Server.Host/Controllers/HomeController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index dda8e793369..76598ef4f1b 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Net; using System.Threading; @@ -227,7 +227,7 @@ public async ValueTask CreateToken(CancellationToken cancellation { if (ApiHeaders == null) { - Response.Headers.Add(HeaderNames.WWWAuthenticate, new StringValues("basic realm=\"Create TGS bearer token\"")); + Response.Headers.Add(HeaderNames.WWWAuthenticate, new StringValues($"basic realm=\"Create TGS {ApiHeaders.BearerAuthenticationScheme} token\"")); return HeadersIssue(false); } From 23a4f5758f4a0cbf2c7f1f3745ff5e7999e5fb24 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 16:40:47 -0400 Subject: [PATCH 076/138] Include `InstanceId` in `JobResponse` --- src/Tgstation.Server.Api/Models/Response/JobResponse.cs | 5 +++++ .../Components/Deployment/DmbFactory.cs | 2 ++ .../Controllers/DreamMakerController.cs | 2 ++ src/Tgstation.Server.Host/Controllers/InstanceController.cs | 6 ++++-- src/Tgstation.Server.Host/Controllers/JobController.cs | 6 +++++- src/Tgstation.Server.Host/Models/Job.cs | 1 + 6 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Api/Models/Response/JobResponse.cs b/src/Tgstation.Server.Api/Models/Response/JobResponse.cs index eb194af4734..053fe1f9aaa 100644 --- a/src/Tgstation.Server.Api/Models/Response/JobResponse.cs +++ b/src/Tgstation.Server.Api/Models/Response/JobResponse.cs @@ -5,6 +5,11 @@ /// public sealed class JobResponse : Internal.Job { + /// + /// The of the . + /// + public long? InstanceId { get; set; } + /// /// The that started the job. /// diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 31bef5dfdb3..fdb223da823 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -239,6 +239,8 @@ await databaseContextFactory.UseContext( .Where(x => x.Id == compileJob.Id) .Include(x => x.Job) .ThenInclude(x => x.StartedBy) + .Include(x => x.Job) + .ThenInclude(x => x.Instance) .Include(x => x.RevisionInformation) .ThenInclude(x => x.PrimaryTestMerge) .ThenInclude(x => x.MergedBy) diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index c2e4874ea4b..105a64d8b97 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -261,6 +261,8 @@ IQueryable BaseCompileJobsQuery() => DatabaseContext .AsQueryable() .Include(x => x.Job) .ThenInclude(x => x.StartedBy) + .Include(x => x.Job) + .ThenInclude(x => x.Instance) .Include(x => x.RevisionInformation) .ThenInclude(x => x.PrimaryTestMerge) .ThenInclude(x => x.MergedBy) diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index c5e472b7ade..d4ae7122817 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -627,7 +627,9 @@ public async ValueTask GetId(long id, CancellationToken cancellat var moveJob = await QueryForUser() .SelectMany(x => x.Jobs) .Where(x => !x.StoppedAt.HasValue && x.Description.StartsWith(MoveInstanceJobPrefix)) - .Include(x => x.StartedBy).ThenInclude(x => x.CreatedBy) + .Include(x => x.StartedBy) + .ThenInclude(x => x.CreatedBy) + .Include(x => x.Instance) .FirstOrDefaultAsync(cancellationToken); api.MoveJob = moveJob?.ToApi(); await CheckAccessible(api, cancellationToken); diff --git a/src/Tgstation.Server.Host/Controllers/JobController.cs b/src/Tgstation.Server.Host/Controllers/JobController.cs index 9b468546bcb..521a560a0b8 100644 --- a/src/Tgstation.Server.Host/Controllers/JobController.cs +++ b/src/Tgstation.Server.Host/Controllers/JobController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -78,6 +78,7 @@ public ValueTask Read([FromQuery] int? page, [FromQuery] int? pag .AsQueryable() .Include(x => x.StartedBy) .Include(x => x.CancelledBy) + .Include(x => x.Instance) .Where(x => x.Instance.Id == Instance.Id && !x.StoppedAt.HasValue) .OrderByDescending(x => x.StartedAt))), AddJobProgressResponseTransformer, @@ -105,6 +106,7 @@ public ValueTask List([FromQuery] int? page, [FromQuery] int? pag .AsQueryable() .Include(x => x.StartedBy) .Include(x => x.CancelledBy) + .Include(x => x.Instance) .Where(x => x.Instance.Id == Instance.Id) .OrderByDescending(x => x.StartedAt))), AddJobProgressResponseTransformer, @@ -132,6 +134,7 @@ public async ValueTask Delete(long id, CancellationToken cancella .Jobs .AsQueryable() .Include(x => x.StartedBy) + .Include(x => x.Instance) .Where(x => x.Id == id && x.Instance.Id == Instance.Id) .FirstOrDefaultAsync(cancellationToken); if (job == default) @@ -167,6 +170,7 @@ public async ValueTask GetId(long id, CancellationToken cancellat .Where(x => x.Id == id && x.Instance.Id == Instance.Id) .Include(x => x.StartedBy) .Include(x => x.CancelledBy) + .Include(x => x.Instance) .FirstOrDefaultAsync(cancellationToken); if (job == default) return NotFound(); diff --git a/src/Tgstation.Server.Host/Models/Job.cs b/src/Tgstation.Server.Host/Models/Job.cs index 1a8fca57c0b..a05b83b811b 100644 --- a/src/Tgstation.Server.Host/Models/Job.cs +++ b/src/Tgstation.Server.Host/Models/Job.cs @@ -30,6 +30,7 @@ public sealed class Job : Api.Models.Internal.Job, IApiTransformable new () { Id = Id, + InstanceId = Instance.Id.Value, StartedAt = StartedAt, StoppedAt = StoppedAt, Cancelled = Cancelled, From b86bd51e1fa6cb02a1bc18f08b91eab0dcd49f7b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 16:41:32 -0400 Subject: [PATCH 077/138] Fix some logging formatter messages --- src/Tgstation.Server.Host/Controllers/UserController.cs | 6 +++--- .../Controllers/UserGroupController.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/UserController.cs b/src/Tgstation.Server.Host/Controllers/UserController.cs index 88ce9d1abb9..38c75043021 100644 --- a/src/Tgstation.Server.Host/Controllers/UserController.cs +++ b/src/Tgstation.Server.Host/Controllers/UserController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -285,7 +285,7 @@ public async ValueTask Update([FromBody] UserUpdateRequest model, DatabaseContext.Groups.Attach(originalUser.Group); if (originalUser.PermissionSet != null) { - Logger.LogInformation("Deleting permission set {0}...", originalUser.PermissionSet.Id); + Logger.LogInformation("Deleting permission set {permissionSetId}...", originalUser.PermissionSet.Id); DatabaseContext.PermissionSets.Remove(originalUser.PermissionSet); originalUser.PermissionSet = null; } @@ -313,7 +313,7 @@ public async ValueTask Update([FromBody] UserUpdateRequest model, await DatabaseContext.Save(cancellationToken); - Logger.LogInformation("Updated user {0} ({1})", originalUser.Name, originalUser.Id); + Logger.LogInformation("Updated user {userName} ({userId})", originalUser.Name, originalUser.Id); // return id only if not a self update and cannot read users var canReadBack = AuthenticationContext.User.Id == originalUser.Id diff --git a/src/Tgstation.Server.Host/Controllers/UserGroupController.cs b/src/Tgstation.Server.Host/Controllers/UserGroupController.cs index 589c1ecb462..3f2c5ea3a24 100644 --- a/src/Tgstation.Server.Host/Controllers/UserGroupController.cs +++ b/src/Tgstation.Server.Host/Controllers/UserGroupController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -98,7 +98,7 @@ public async ValueTask Create([FromBody] UserGroupCreateRequest m DatabaseContext.Groups.Add(dbGroup); await DatabaseContext.Save(cancellationToken); - Logger.LogInformation("Created new user group {0} ({1})", dbGroup.Name, dbGroup.Id); + Logger.LogInformation("Created new user group {groupName} ({groupId})", dbGroup.Name, dbGroup.Id); return Created(dbGroup.ToApi(true)); } From 808f96b9681cd42589decf9cfa9690680aefaf25 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 16:43:50 -0400 Subject: [PATCH 078/138] Clean up some using ordering --- .../Components/TestDreamDaemonClient.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/Tgstation.Server.Client.Tests/Components/TestDreamDaemonClient.cs b/tests/Tgstation.Server.Client.Tests/Components/TestDreamDaemonClient.cs index efd4bccc292..42f90a47bc8 100644 --- a/tests/Tgstation.Server.Client.Tests/Components/TestDreamDaemonClient.cs +++ b/tests/Tgstation.Server.Client.Tests/Components/TestDreamDaemonClient.cs @@ -1,13 +1,12 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; -using System; -using System.Collections.Generic; -using System.Text; +using System; using System.Threading; using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using Moq; + using Tgstation.Server.Api; -using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; namespace Tgstation.Server.Client.Components.Tests From ad460d9ae46bdacd4b523975af011d25ae765412 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 16:45:48 -0400 Subject: [PATCH 079/138] Silence VS messages about `new()` expressions --- src/Tgstation.Server.Host/Models/CompileJob.cs | 2 +- src/Tgstation.Server.Host/Models/UserGroup.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Models/CompileJob.cs b/src/Tgstation.Server.Host/Models/CompileJob.cs index 9ddde1344c1..5ee95f96ef5 100644 --- a/src/Tgstation.Server.Host/Models/CompileJob.cs +++ b/src/Tgstation.Server.Host/Models/CompileJob.cs @@ -82,7 +82,7 @@ public override Version DMApiVersion } /// - public CompileJobResponse ToApi() => new CompileJobResponse + public CompileJobResponse ToApi() => new () { DirectoryName = DirectoryName, DmeName = DmeName, diff --git a/src/Tgstation.Server.Host/Models/UserGroup.cs b/src/Tgstation.Server.Host/Models/UserGroup.cs index d914fe872bc..11329e9397d 100644 --- a/src/Tgstation.Server.Host/Models/UserGroup.cs +++ b/src/Tgstation.Server.Host/Models/UserGroup.cs @@ -28,7 +28,7 @@ public sealed class UserGroup : NamedEntity, IApiTransformable /// If should be populated. /// A new . - public UserGroupResponse ToApi(bool showUsers) => new UserGroupResponse + public UserGroupResponse ToApi(bool showUsers) => new () { Id = Id, Name = Name, From 7bb565d0da0539e52d1b790af449bb71f1077295 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 18:10:19 -0400 Subject: [PATCH 080/138] Better message formatting for `HeadersException` --- src/Tgstation.Server.Api/ApiHeaders.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Api/ApiHeaders.cs b/src/Tgstation.Server.Api/ApiHeaders.cs index 5670abb8a30..c912e582651 100644 --- a/src/Tgstation.Server.Api/ApiHeaders.cs +++ b/src/Tgstation.Server.Api/ApiHeaders.cs @@ -197,11 +197,15 @@ public ApiHeaders(RequestHeaders requestHeaders, bool ignoreMissingAuth = false) var badHeaders = HeaderTypes.None; var errorBuilder = new StringBuilder(); - + var multipleErrors = false; void AddError(HeaderTypes headerType, string message) { if (badHeaders != HeaderTypes.None) + { + multipleErrors = true; errorBuilder.AppendLine(); + } + badHeaders |= headerType; errorBuilder.Append(message); } @@ -334,7 +338,12 @@ void AddError(HeaderTypes headerType, string message) } if (badHeaders != HeaderTypes.None) + { + if (multipleErrors) + errorBuilder.Insert(0, $"Multiple header validation errors occurred:{Environment.NewLine}"); + throw new HeadersException(badHeaders, errorBuilder.ToString()); + } ApiVersion = apiVersion!.Semver(); } From 3901512693ed575943a24ff3add10da9cfc4dec3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 18:28:56 -0400 Subject: [PATCH 081/138] The `del world` issue is caused by `sleep_offline` Just do an infinite loop and ensure `sleep_offline` is off. It can't keep getting away with this. Bump DMAPI version --- build/Version.props | 2 +- src/DMAPI/tgs.dm | 2 +- src/DMAPI/tgs/core/datum.dm | 8 ++++++-- tests/DMAPI/BasicOperation/Config.dm | 9 --------- tests/DMAPI/LongRunning/Config.dm | 10 ---------- tests/DMAPI/test_prelude.dm | 11 +++++++++++ 6 files changed, 19 insertions(+), 23 deletions(-) diff --git a/build/Version.props b/build/Version.props index 5acebbddfba..ecaea3d8d97 100644 --- a/build/Version.props +++ b/build/Version.props @@ -9,7 +9,7 @@ 7.0.0 11.1.2 13.0.0 - 6.6.1 + 6.6.2 5.6.2 1.4.0 1.2.1 diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index d468d604419..0cc106ec9cf 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,6 +1,6 @@ // tgstation-server DMAPI -#define TGS_DMAPI_VERSION "6.6.1" +#define TGS_DMAPI_VERSION "6.6.2" // All functions and datums outside this document are subject to change with any version and should not be relied on. diff --git a/src/DMAPI/tgs/core/datum.dm b/src/DMAPI/tgs/core/datum.dm index de420a2a325..935263cc955 100644 --- a/src/DMAPI/tgs/core/datum.dm +++ b/src/DMAPI/tgs/core/datum.dm @@ -12,8 +12,12 @@ TGS_DEFINE_AND_SET_GLOBAL(tgs, null) src.version = version /datum/tgs_api/proc/TerminateWorld() - del(world) - sleep(1) // https://www.byond.com/forum/post/2894866 + while(TRUE) + TGS_DEBUG_LOG("About to terminate world. Tick: [world.time], sleep_offline: [world.sleep_offline]") + del(world) + world.sleep_offline = FALSE // https://www.byond.com/forum/post/2894866 + sleep(1) + TGS_DEBUG_LOG("BYOND DIDN'T TERMINATE THE WORLD!!! TICK IS: [world.time], sleep_offline: [world.sleep_offline]") /datum/tgs_api/latest parent_type = /datum/tgs_api/v5 diff --git a/tests/DMAPI/BasicOperation/Config.dm b/tests/DMAPI/BasicOperation/Config.dm index 3cd3f332de2..45d12ef665f 100644 --- a/tests/DMAPI/BasicOperation/Config.dm +++ b/tests/DMAPI/BasicOperation/Config.dm @@ -1,12 +1,3 @@ -#define TGS_EXTERNAL_CONFIGURATION -#define TGS_DEFINE_AND_SET_GLOBAL(Name, Value) var/##Name = ##Value -#define TGS_READ_GLOBAL(Name) global.##Name -#define TGS_WRITE_GLOBAL(Name, Value) global.##Name = ##Value -#define TGS_PROTECT_DATUM(Path) -#define TGS_WORLD_ANNOUNCE(message) world << ##message #define TGS_INFO_LOG(message) world.log << "Info: [##message]" -#define TGS_WARNING_LOG(message) world.log << "Warn: [##message]" #define TGS_ERROR_LOG(message) world.log << "Err: [##message]" -#define TGS_NOTIFY_ADMINS(event) -#define TGS_CLIENT_COUNT 0 #define TGS_V3_API diff --git a/tests/DMAPI/LongRunning/Config.dm b/tests/DMAPI/LongRunning/Config.dm index de593ce90f1..3aa6b8a01cb 100644 --- a/tests/DMAPI/LongRunning/Config.dm +++ b/tests/DMAPI/LongRunning/Config.dm @@ -1,12 +1,2 @@ -#define TGS_EXTERNAL_CONFIGURATION -#define TGS_DEFINE_AND_SET_GLOBAL(Name, Value) var/##Name = ##Value -#define TGS_READ_GLOBAL(Name) global.##Name -#define TGS_WRITE_GLOBAL(Name, Value) global.##Name = ##Value -#define TGS_PROTECT_DATUM(Path) -#define TGS_WORLD_ANNOUNCE(message) world << ##message #define TGS_INFO_LOG(message) TgsInfo(##message) -#define TGS_WARNING_LOG(message) world.log << "Warn: [##message]" #define TGS_ERROR_LOG(message) TgsError(##message) -#define TGS_NOTIFY_ADMINS(event) -#define TGS_CLIENT_COUNT 0 -#define TGS_DEBUG_LOG(message) world.log << "TGS DEBUG: [##message]" diff --git a/tests/DMAPI/test_prelude.dm b/tests/DMAPI/test_prelude.dm index 18b7192ec10..efb84d46ae3 100644 --- a/tests/DMAPI/test_prelude.dm +++ b/tests/DMAPI/test_prelude.dm @@ -1,3 +1,14 @@ +#define TGS_EXTERNAL_CONFIGURATION +#define TGS_DEFINE_AND_SET_GLOBAL(Name, Value) var/##Name = ##Value +#define TGS_READ_GLOBAL(Name) global.##Name +#define TGS_WRITE_GLOBAL(Name, Value) global.##Name = ##Value +#define TGS_PROTECT_DATUM(Path) +#define TGS_WORLD_ANNOUNCE(message) world << ##message +#define TGS_WARNING_LOG(message) world.log << "Warn: [##message]" +#define TGS_NOTIFY_ADMINS(event) +#define TGS_CLIENT_COUNT 0 +#define TGS_DEBUG_LOG(message) world.log << "TGS DEBUG: [##message]" + #include "..\..\src\DMAPI\tgs.dm" #include "..\..\src\DMAPI\tgs\includes.dm" #include "test_setup.dm" From eaebe733bd29478c64920ef41ad3afe4cff406bb Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 19:13:51 -0400 Subject: [PATCH 082/138] Improve `HeadersException` handling Avoid recreating ApiHeaders in request pipeline where possible --- src/Tgstation.Server.Api/ApiHeaders.cs | 44 ++++++------- src/Tgstation.Server.Api/HeaderErrorTypes.cs | 46 +++++++++++++ src/Tgstation.Server.Api/HeaderTypes.cs | 41 ------------ src/Tgstation.Server.Api/HeadersException.cs | 10 +-- .../Controllers/ApiController.cs | 31 ++++----- .../Controllers/HomeController.cs | 8 +-- .../Utils/ApiHeadersProvider.cs | 64 ++++++++++++++++--- .../Utils/IApiHeadersProvider.cs | 13 ++++ .../TestApiHeaders.cs | 2 +- 9 files changed, 160 insertions(+), 99 deletions(-) create mode 100644 src/Tgstation.Server.Api/HeaderErrorTypes.cs delete mode 100644 src/Tgstation.Server.Api/HeaderTypes.cs diff --git a/src/Tgstation.Server.Api/ApiHeaders.cs b/src/Tgstation.Server.Api/ApiHeaders.cs index c912e582651..b23d890abeb 100644 --- a/src/Tgstation.Server.Api/ApiHeaders.cs +++ b/src/Tgstation.Server.Api/ApiHeaders.cs @@ -190,17 +190,17 @@ public ApiHeaders(ProductHeaderValue userAgent, string username, string password /// If a missing should be ignored. /// Thrown if the constitue invalid . #pragma warning disable CA1502 // TODO: Decomplexify - public ApiHeaders(RequestHeaders requestHeaders, bool ignoreMissingAuth = false) + public ApiHeaders(RequestHeaders requestHeaders, bool ignoreMissingAuth) { if (requestHeaders == null) throw new ArgumentNullException(nameof(requestHeaders)); - var badHeaders = HeaderTypes.None; + var badHeaders = HeaderErrorTypes.None; var errorBuilder = new StringBuilder(); var multipleErrors = false; - void AddError(HeaderTypes headerType, string message) + void AddError(HeaderErrorTypes headerType, string message) { - if (badHeaders != HeaderTypes.None) + if (badHeaders != HeaderErrorTypes.None) { multipleErrors = true; errorBuilder.AppendLine(); @@ -212,28 +212,28 @@ void AddError(HeaderTypes headerType, string message) var jsonAccept = new Microsoft.Net.Http.Headers.MediaTypeHeaderValue(ApplicationJsonMime); if (!requestHeaders.Accept.Any(x => jsonAccept.IsSubsetOf(x))) - AddError(HeaderTypes.Accept, $"Client does not accept {ApplicationJsonMime}!"); + AddError(HeaderErrorTypes.Accept, $"Client does not accept {ApplicationJsonMime}!"); if (!requestHeaders.Headers.TryGetValue(HeaderNames.UserAgent, out var userAgentValues) || userAgentValues.Count == 0) - AddError(HeaderTypes.UserAgent, $"Missing {HeaderNames.UserAgent} header!"); + AddError(HeaderErrorTypes.UserAgent, $"Missing {HeaderNames.UserAgent} header!"); else { RawUserAgent = userAgentValues.First(); if (String.IsNullOrWhiteSpace(RawUserAgent)) - AddError(HeaderTypes.UserAgent, $"Malformed {HeaderNames.UserAgent} header!"); + AddError(HeaderErrorTypes.UserAgent, $"Malformed {HeaderNames.UserAgent} header!"); } // make sure the api header matches ours Version? apiVersion = null; if (!requestHeaders.Headers.TryGetValue(ApiVersionHeader, out var apiUserAgentHeaderValues) || !ProductInfoHeaderValue.TryParse(apiUserAgentHeaderValues.FirstOrDefault(), out var apiUserAgent) || apiUserAgent.Product.Name != AssemblyName.Name) - AddError(HeaderTypes.Api, $"Missing {ApiVersionHeader} header!"); + AddError(HeaderErrorTypes.Api, $"Missing {ApiVersionHeader} header!"); else if (!Version.TryParse(apiUserAgent.Product.Version, out apiVersion)) - AddError(HeaderTypes.Api, $"Malformed {ApiVersionHeader} header!"); + AddError(HeaderErrorTypes.Api, $"Malformed {ApiVersionHeader} header!"); if (!requestHeaders.Headers.TryGetValue(HeaderNames.Authorization, out StringValues authorization)) { if (!ignoreMissingAuth) - AddError(HeaderTypes.Authorization, $"Missing {HeaderNames.Authorization} header!"); + AddError(HeaderErrorTypes.AuthorizationMissing, $"Missing {HeaderNames.Authorization} header!"); } else { @@ -241,13 +241,13 @@ void AddError(HeaderTypes headerType, string message) var splits = new List(auth.Split(' ')); var scheme = splits.First(); if (String.IsNullOrWhiteSpace(scheme)) - AddError(HeaderTypes.Authorization, "Missing authentication scheme!"); + AddError(HeaderErrorTypes.AuthorizationInvalid, "Missing authentication scheme!"); else { splits.RemoveAt(0); var parameter = String.Concat(splits); if (String.IsNullOrEmpty(parameter)) - AddError(HeaderTypes.Authorization, "Missing authentication parameter!"); + AddError(HeaderErrorTypes.AuthorizationInvalid, "Missing authentication parameter!"); else { if (requestHeaders.Headers.TryGetValue(InstanceIdHeader, out var instanceIdValues)) @@ -266,10 +266,10 @@ void AddError(HeaderTypes headerType, string message) if (Enum.TryParse(oauthProviderString, out var oauthProvider)) OAuthProvider = oauthProvider; else - AddError(HeaderTypes.OAuthProvider, "Invalid OAuth provider!"); + AddError(HeaderErrorTypes.OAuthProvider, "Invalid OAuth provider!"); } else - AddError(HeaderTypes.OAuthProvider, $"Missing {OAuthProviderHeader} header!"); + AddError(HeaderErrorTypes.OAuthProvider, $"Missing {OAuthProviderHeader} header!"); OAuthCode = parameter; break; @@ -277,7 +277,7 @@ void AddError(HeaderTypes headerType, string message) var tokenSplits = parameter.Split('.'); DateTimeOffset? expiresAt = null; if (tokenSplits.Length != 3) - AddError(HeaderTypes.Authorization, "Invalid JWT!"); + AddError(HeaderErrorTypes.AuthorizationInvalid, "Invalid JWT!"); else try { @@ -290,13 +290,13 @@ void AddError(HeaderTypes headerType, string message) if (Int64.TryParse(nbf, out var unixTimeSeconds)) expiresAt = DateTimeOffset.FromUnixTimeSeconds(unixTimeSeconds); else - AddError(HeaderTypes.Authorization, "'nbf' in JWT could not be parsed!"); + AddError(HeaderErrorTypes.AuthorizationInvalid, "'nbf' in JWT could not be parsed!"); else - AddError(HeaderTypes.Authorization, "Missing 'nbf' in JWT payload!"); + AddError(HeaderErrorTypes.AuthorizationInvalid, "Missing 'nbf' in JWT payload!"); } catch { - AddError(HeaderTypes.Authorization, "Invalid JWT payload!"); + AddError(HeaderErrorTypes.AuthorizationInvalid, "Invalid JWT payload!"); } Token = new TokenResponse @@ -315,14 +315,14 @@ void AddError(HeaderTypes headerType, string message) } catch { - AddError(HeaderTypes.Authorization, badBasicAuthHeaderMessage); + AddError(HeaderErrorTypes.AuthorizationInvalid, badBasicAuthHeaderMessage); break; } var basicAuthSplits = joinedString.Split(ColonSeparator, StringSplitOptions.RemoveEmptyEntries); if (basicAuthSplits.Length < 2) { - AddError(HeaderTypes.Authorization, badBasicAuthHeaderMessage); + AddError(HeaderErrorTypes.AuthorizationInvalid, badBasicAuthHeaderMessage); break; } @@ -330,14 +330,14 @@ void AddError(HeaderTypes headerType, string message) Password = String.Concat(basicAuthSplits.Skip(1)); break; default: - AddError(HeaderTypes.Authorization, "Invalid authentication scheme!"); + AddError(HeaderErrorTypes.AuthorizationInvalid, "Invalid authentication scheme!"); break; } } } } - if (badHeaders != HeaderTypes.None) + if (badHeaders != HeaderErrorTypes.None) { if (multipleErrors) errorBuilder.Insert(0, $"Multiple header validation errors occurred:{Environment.NewLine}"); diff --git a/src/Tgstation.Server.Api/HeaderErrorTypes.cs b/src/Tgstation.Server.Api/HeaderErrorTypes.cs new file mode 100644 index 00000000000..414b1e22d3f --- /dev/null +++ b/src/Tgstation.Server.Api/HeaderErrorTypes.cs @@ -0,0 +1,46 @@ +using System; + +namespace Tgstation.Server.Api +{ + /// + /// Types of individual errors. + /// + [Flags] + public enum HeaderErrorTypes + { + /// + /// No header errors. + /// + None = 0, + + /// + /// The header is missing or invalid. + /// + UserAgent = 1 << 0, + + /// + /// The header is missing or invalid. + /// + Accept = 1 << 1, + + /// + /// The header is missing or invalid. + /// + Api = 1 << 2, + + /// + /// The header is invalid. + /// + AuthorizationInvalid = 1 << 3, + + /// + /// The header is missing or invalid. + /// + OAuthProvider = 1 << 4, + + /// + /// The header is missing. + /// + AuthorizationMissing = 1 << 5, + } +} diff --git a/src/Tgstation.Server.Api/HeaderTypes.cs b/src/Tgstation.Server.Api/HeaderTypes.cs deleted file mode 100644 index a5fdca325d9..00000000000 --- a/src/Tgstation.Server.Api/HeaderTypes.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; - -namespace Tgstation.Server.Api -{ - /// - /// Types of individual . - /// - [Flags] - public enum HeaderTypes - { - /// - /// No headers. - /// - None = 0, - - /// - /// header. - /// - UserAgent = 1 << 0, - - /// - /// header. - /// - Accept = 1 << 1, - - /// - /// . - /// - Api = 1 << 2, - - /// - /// - /// - Authorization = 1 << 3, - - /// - /// . - /// - OAuthProvider = 1 << 4, - } -} diff --git a/src/Tgstation.Server.Api/HeadersException.cs b/src/Tgstation.Server.Api/HeadersException.cs index 6287ee1c88e..352d2ce9d86 100644 --- a/src/Tgstation.Server.Api/HeadersException.cs +++ b/src/Tgstation.Server.Api/HeadersException.cs @@ -8,19 +8,19 @@ namespace Tgstation.Server.Api public sealed class HeadersException : Exception { /// - /// The s that are missing or malformed. + /// The s that are missing or malformed. /// - public HeaderTypes MissingOrMalformedHeaders { get; } + public HeaderErrorTypes ParseErrors { get; } /// /// Initializes a new instance of the class. /// - /// The value of . + /// The value of . /// The error message. - public HeadersException(HeaderTypes missingOrMalformedHeaders, string message) + public HeadersException(HeaderErrorTypes parseErrors, string message) : base(message) { - MissingOrMalformedHeaders = missingOrMalformedHeaders; + ParseErrors = parseErrors; } /// diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index 9fd4b6154ad..d0b72134277 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -48,7 +48,12 @@ public abstract class ApiController : ApiControllerBase /// /// The for the operation. /// - protected ApiHeaders ApiHeaders { get; } + protected ApiHeaders ApiHeaders => ApiHeadersProvider.ApiHeaders; + + /// + /// The containing value of . + /// + protected IApiHeadersProvider ApiHeadersProvider { get; } /// /// The for the operation. @@ -81,7 +86,7 @@ public abstract class ApiController : ApiControllerBase /// The value of . /// The for the . /// The value of . - /// The containing value of . + /// The value of .. /// The value of . protected ApiController( IDatabaseContext databaseContext, @@ -92,11 +97,10 @@ protected ApiController( { DatabaseContext = databaseContext ?? throw new ArgumentNullException(nameof(databaseContext)); AuthenticationContext = authenticationContext ?? throw new ArgumentNullException(nameof(authenticationContext)); - ArgumentNullException.ThrowIfNull(apiHeadersProvider); + ApiHeadersProvider = apiHeadersProvider ?? throw new ArgumentNullException(nameof(apiHeadersProvider)); Logger = logger ?? throw new ArgumentNullException(nameof(logger)); Instance = AuthenticationContext?.InstancePermissionSet?.Instance; - ApiHeaders = apiHeadersProvider.ApiHeaders; this.requireHeaders = requireHeaders; } @@ -110,7 +114,7 @@ protected override async ValueTask HookExecuteAction(Func e if (ApiHeaders == null) { if (requireHeaders) - return HeadersIssue(false); + return HeadersIssue(ApiHeadersProvider.HeadersException); } else if (!ApiHeaders.Compatible()) return this.StatusCode( @@ -239,28 +243,19 @@ protected virtual ValueTask ValidateRequest(CancellationToken can /// /// Response for missing/Invalid headers. /// - /// Whether or not errors due to missing should be thrown. + /// The that occurred while trying to parse the . /// The appropriate . - protected IActionResult HeadersIssue(bool ignoreMissingAuth) + protected IActionResult HeadersIssue(HeadersException headersException) { - HeadersException headersException; - try - { - // TODO: Move this somewhere saner? - _ = new ApiHeaders(Request.GetTypedHeaders(), ignoreMissingAuth); + if (headersException == null) throw new InvalidOperationException("Expected a header parse exception!"); - } - catch (HeadersException ex) - { - headersException = ex; - } var errorMessage = new ErrorMessageResponse(ErrorCode.BadHeaders) { AdditionalData = headersException.Message, }; - if (headersException.MissingOrMalformedHeaders.HasFlag(HeaderTypes.Accept)) + if (headersException.ParseErrors.HasFlag(HeaderErrorTypes.Accept)) return this.StatusCode(HttpStatusCode.NotAcceptable, errorMessage); return BadRequest(errorMessage); diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index 76598ef4f1b..290049ca52e 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -180,15 +180,15 @@ public IActionResult Home() try { // we only allow authorization header issues - var headers = new ApiHeaders(Request.GetTypedHeaders(), true); + var headers = ApiHeadersProvider.CreateAuthlessHeaders(); if (!headers.Compatible()) return this.StatusCode( HttpStatusCode.UpgradeRequired, new ErrorMessageResponse(ErrorCode.ApiMismatch)); } - catch (HeadersException) + catch (HeadersException ex) { - return HeadersIssue(true); + return HeadersIssue(ex); } } @@ -228,7 +228,7 @@ public async ValueTask CreateToken(CancellationToken cancellation if (ApiHeaders == null) { Response.Headers.Add(HeaderNames.WWWAuthenticate, new StringValues($"basic realm=\"Create TGS {ApiHeaders.BearerAuthenticationScheme} token\"")); - return HeadersIssue(false); + return HeadersIssue(ApiHeadersProvider.HeadersException); } if (ApiHeaders.IsTokenAuthentication) diff --git a/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs b/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs index c59d2d67d0a..868a7683c70 100644 --- a/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs +++ b/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs @@ -2,6 +2,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; + using Tgstation.Server.Api; namespace Tgstation.Server.Host.Utils @@ -10,29 +11,76 @@ namespace Tgstation.Server.Host.Utils sealed class ApiHeadersProvider : IApiHeadersProvider { /// - public ApiHeaders ApiHeaders { get; } + public ApiHeaders ApiHeaders => attemptedApiHeadersCreation + ? apiHeaders + : CreateApiHeaders(true); + + /// + public HeadersException HeadersException { get; private set; } + + /// + /// The for the . + /// + readonly IHttpContextAccessor httpContextAccessor; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// Backing field for . + /// + ApiHeaders apiHeaders; + + /// + /// If populating was previously attempted. + /// + bool attemptedApiHeadersCreation; /// /// Initializes a new instance of the class. /// - /// The for accessing the . - /// The to write to. + /// The value of . + /// The value of . public ApiHeadersProvider(IHttpContextAccessor httpContextAccessor, ILogger logger) { - ArgumentNullException.ThrowIfNull(httpContextAccessor); + this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException(nameof(httpContextAccessor)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public ApiHeaders CreateAuthlessHeaders() => CreateApiHeaders(false); + /// + /// Attempt to parse from the , optionally populating the properties. + /// + /// If the error should be ignored and / should be populated. + /// A newly parsed or if was set and the parse failed. + ApiHeaders CreateApiHeaders(bool includeAuthAndSetProperties) + { if (httpContextAccessor.HttpContext == null) throw new InvalidOperationException("httpContextAccessor has no HttpContext!"); var request = httpContextAccessor.HttpContext.Request; + var ignoreMissingAuth = !includeAuthAndSetProperties; + + if (includeAuthAndSetProperties) + attemptedApiHeadersCreation = true; + try { - ApiHeaders = new ApiHeaders(request.GetTypedHeaders()); + var headers = new ApiHeaders(request.GetTypedHeaders(), ignoreMissingAuth); + if (includeAuthAndSetProperties) + apiHeaders = headers; + + return headers; } - catch (HeadersException ex) + catch (HeadersException ex) when (includeAuthAndSetProperties) { - // we are not responsible for handling header validation issues - logger.LogTrace(ex, "Failed to validated API request headers!"); + logger.LogTrace(ex, "Failed to parse API headers!"); + HeadersException = ex; + return null; } } } diff --git a/src/Tgstation.Server.Host/Utils/IApiHeadersProvider.cs b/src/Tgstation.Server.Host/Utils/IApiHeadersProvider.cs index 31747862be0..7de65834d3d 100644 --- a/src/Tgstation.Server.Host/Utils/IApiHeadersProvider.cs +++ b/src/Tgstation.Server.Host/Utils/IApiHeadersProvider.cs @@ -11,5 +11,18 @@ public interface IApiHeadersProvider /// The created , if any. /// ApiHeaders ApiHeaders { get; } + + /// + /// The thrown when attempting to parse the if any. + /// + HeadersException HeadersException { get; } + + /// + /// Attempt to create without checking for the presence of an header. + /// + /// A new . + /// This does not populate the property. + /// Thrown if the requested contain errors other than . + ApiHeaders CreateAuthlessHeaders(); } } diff --git a/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs b/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs index 5261d8719a5..27329a1af4b 100644 --- a/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs +++ b/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs @@ -43,7 +43,7 @@ static ApiHeaders TestHeader(string userAgent) { "User-Agent", userAgent } }; - return new ApiHeaders(new RequestHeaders(headers)); + return new ApiHeaders(new RequestHeaders(headers), false); }; var header = TestHeader(BrowserHeader); From dd8c8df35ffbb1628ec5b268b543dc8c2171a038 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 19:20:04 -0400 Subject: [PATCH 083/138] Fix `NotImplementedException` errors on non-github/gitlab deployments --- .../Deployment/Remote/NoOpRemoteDeploymentManager.cs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/NoOpRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/NoOpRemoteDeploymentManager.cs index b8253d412d2..6751f181849 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/NoOpRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/NoOpRemoteDeploymentManager.cs @@ -32,10 +32,7 @@ public NoOpRemoteDeploymentManager( } /// - public override ValueTask FailDeployment(Models.CompileJob compileJob, string errorMessage, CancellationToken cancellationToken) - { - throw new NotImplementedException(); - } + public override ValueTask FailDeployment(Models.CompileJob compileJob, string errorMessage, CancellationToken cancellationToken) => ValueTask.CompletedTask; /// public override ValueTask> RemoveMergedTestMerges(IRepository repository, Models.RepositorySettings repositorySettings, Models.RevisionInformation revisionInformation, CancellationToken cancellationToken) @@ -69,10 +66,7 @@ protected override string FormatTestMerge( => String.Empty; /// - protected override ValueTask MarkInactiveImpl(Models.CompileJob compileJob, CancellationToken cancellationToken) - { - throw new NotImplementedException(); - } + protected override ValueTask MarkInactiveImpl(Models.CompileJob compileJob, CancellationToken cancellationToken) => ValueTask.CompletedTask; /// protected override ValueTask StageDeploymentImpl(Models.CompileJob compileJob, CancellationToken cancellationToken) => ValueTask.CompletedTask; From e41e59a48aba2af750cad633f05bef7c1be1eb7c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 19:23:45 -0400 Subject: [PATCH 084/138] Remove unnecessary checks for non-existent `PosixSystemIdentity`s --- .../Controllers/ConfigurationController.cs | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs index ea26922fd88..37b7e89ed1d 100644 --- a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs +++ b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs @@ -103,10 +103,6 @@ public async ValueTask Update([FromBody] ConfigurationFileRequest AdditionalData = e.Message, }); } - catch (NotImplementedException ex) - { - return RequiresPosixSystemIdentity(ex); - } } /// @@ -150,10 +146,6 @@ public async ValueTask File(string filePath, CancellationToken ca AdditionalData = e.Message, }); } - catch (NotImplementedException ex) - { - return RequiresPosixSystemIdentity(ex); - } } /// @@ -196,11 +188,6 @@ public ValueTask Directory( return new PaginatableResult(result); } - catch (NotImplementedException ex) - { - return new PaginatableResult( - RequiresPosixSystemIdentity(ex)); - } catch (UnauthorizedAccessException) { return new PaginatableResult( @@ -287,10 +274,6 @@ public async ValueTask CreateDirectory([FromBody] ConfigurationFi Message = e.Message, }); } - catch (NotImplementedException ex) - { - return RequiresPosixSystemIdentity(ex); - } catch (UnauthorizedAccessException) { return Forbid(); @@ -335,10 +318,6 @@ public async ValueTask DeleteDirectory([FromBody] ConfigurationFi : Conflict(new ErrorMessageResponse(ErrorCode.ConfigurationDirectoryNotEmpty)); }); } - catch (NotImplementedException ex) - { - return RequiresPosixSystemIdentity(ex); - } catch (UnauthorizedAccessException) { return Forbid(); From 29579a351d70f433ca3752fb851d7e54baf5944b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 29 Oct 2023 21:05:12 -0400 Subject: [PATCH 085/138] Fix JWT compliance issues. - `nbf` and `exp` should be numbers, not strings. - Parse JWT in `ApiHeaders` fully for errors. - Use `iat` and increment `nbf` by one second if equal and may trigger the `LastPasswordUpdate` bug. - Make client aware of this change and move the delay there. - Deprecated `TokenResponse.ExpiresAt`. --- src/Tgstation.Server.Api/ApiHeaders.cs | 42 ++++---------- .../Models/Response/TokenResponse.cs | 9 +++ .../Tgstation.Server.Api.csproj | 2 +- src/Tgstation.Server.Client/ApiClient.cs | 26 +++++++++ .../Controllers/HomeController.cs | 5 +- .../Security/ITokenFactory.cs | 10 +--- .../Security/TokenFactory.cs | 57 +++++++------------ .../Live/RawRequestTests.cs | 2 + 8 files changed, 76 insertions(+), 77 deletions(-) diff --git a/src/Tgstation.Server.Api/ApiHeaders.cs b/src/Tgstation.Server.Api/ApiHeaders.cs index b23d890abeb..98f14b91dba 100644 --- a/src/Tgstation.Server.Api/ApiHeaders.cs +++ b/src/Tgstation.Server.Api/ApiHeaders.cs @@ -9,12 +9,8 @@ using Microsoft.AspNetCore.Http.Headers; using Microsoft.Extensions.Primitives; -using Microsoft.IdentityModel.JsonWebTokens; using Microsoft.Net.Http.Headers; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Api.Properties; @@ -274,36 +270,22 @@ void AddError(HeaderErrorTypes headerType, string message) OAuthCode = parameter; break; case BearerAuthenticationScheme: - var tokenSplits = parameter.Split('.'); - DateTimeOffset? expiresAt = null; - if (tokenSplits.Length != 3) - AddError(HeaderErrorTypes.AuthorizationInvalid, "Invalid JWT!"); - else - try - { - var bytes = Convert.FromBase64String(tokenSplits[1]); - var json = Encoding.UTF8.GetString(bytes); - var jwt = JsonConvert.DeserializeObject(json); - var nbf = jwt?.Value(JwtRegisteredClaimNames.Nbf); - - if (nbf != null) - if (Int64.TryParse(nbf, out var unixTimeSeconds)) - expiresAt = DateTimeOffset.FromUnixTimeSeconds(unixTimeSeconds); - else - AddError(HeaderErrorTypes.AuthorizationInvalid, "'nbf' in JWT could not be parsed!"); - else - AddError(HeaderErrorTypes.AuthorizationInvalid, "Missing 'nbf' in JWT payload!"); - } - catch - { - AddError(HeaderErrorTypes.AuthorizationInvalid, "Invalid JWT payload!"); - } - Token = new TokenResponse { Bearer = parameter, - ExpiresAt = expiresAt, }; + + try + { +#pragma warning disable CS0618 // Type or member is obsolete + Token.ExpiresAt = Token.ParseJwt().ValidTo; +#pragma warning restore CS0618 // Type or member is obsolete + } + catch (ArgumentException ex) when (ex is not ArgumentNullException) + { + AddError(HeaderErrorTypes.AuthorizationInvalid, $"Invalid JWT: {ex.Message}"); + } + break; case BasicAuthenticationScheme: string badBasicAuthHeaderMessage = $"Invalid basic {HeaderNames.Authorization} header!"; diff --git a/src/Tgstation.Server.Api/Models/Response/TokenResponse.cs b/src/Tgstation.Server.Api/Models/Response/TokenResponse.cs index cb24b941a6a..8f7b01762b8 100644 --- a/src/Tgstation.Server.Api/Models/Response/TokenResponse.cs +++ b/src/Tgstation.Server.Api/Models/Response/TokenResponse.cs @@ -1,5 +1,7 @@ using System; +using Microsoft.IdentityModel.JsonWebTokens; + namespace Tgstation.Server.Api.Models.Response { /// @@ -15,6 +17,13 @@ public sealed class TokenResponse /// /// When the expires. /// + [Obsolete("Will be removed in a future API version")] public DateTimeOffset? ExpiresAt { get; set; } + + /// + /// Parses the as a . + /// + /// A new based on . + public JsonWebToken ParseJwt() => new (Bearer); } } diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index dfbd15a36d2..3573822c52b 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -26,7 +26,7 @@ - + diff --git a/src/Tgstation.Server.Client/ApiClient.cs b/src/Tgstation.Server.Client/ApiClient.cs index cb0bda67dab..ccc5bb9e9bc 100644 --- a/src/Tgstation.Server.Client/ApiClient.cs +++ b/src/Tgstation.Server.Client/ApiClient.cs @@ -336,6 +336,32 @@ protected virtual async ValueTask RunRequest( if (authless) request.Headers.Remove(HeaderNames.Authorization); + else + { + var bearer = headersToUse.Token?.Bearer; + if (bearer != null) + { + try + { + var parsed = headersToUse.Token!.ParseJwt(); + var nbf = parsed.ValidFrom; + var now = DateTime.UtcNow; + if (nbf >= now) + { + var delay = (nbf - now).Add(TimeSpan.FromMilliseconds(1)); + await Task.Delay(delay, cancellationToken); + } + } + catch (ArgumentException ex) when (ex is not ArgumentNullException) + { + // backwards compat, API <=9 put out invalid JWTs, remove in API 10 + } +#if DEBUG + if (ApiHeaders.Version.Major > 9) + throw new NotImplementedException(); +#endif + } + } if (fileDownload) request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(MediaTypeNames.Application.Octet)); diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index 290049ca52e..7a821316833 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -5,7 +5,6 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -373,11 +372,11 @@ public async ValueTask CreateToken(CancellationToken cancellation return Forbid(); } - var token = await tokenFactory.CreateToken(user, oAuthLogin, cancellationToken); + var token = tokenFactory.CreateToken(user, oAuthLogin); if (usingSystemIdentity) { // expire the identity slightly after the auth token in case of lag - var identExpiry = token.ExpiresAt.Value; + var identExpiry = token.ParseJwt().ValidTo; identExpiry += tokenFactory.ValidationParameters.ClockSkew; identExpiry += TimeSpan.FromSeconds(15); identityCache.CacheSystemIdentity(user, systemIdentity, identExpiry); diff --git a/src/Tgstation.Server.Host/Security/ITokenFactory.cs b/src/Tgstation.Server.Host/Security/ITokenFactory.cs index 6df27f813fd..bf9a70fab15 100644 --- a/src/Tgstation.Server.Host/Security/ITokenFactory.cs +++ b/src/Tgstation.Server.Host/Security/ITokenFactory.cs @@ -1,7 +1,4 @@ -using System.Threading; -using System.Threading.Tasks; - -using Microsoft.IdentityModel.Tokens; +using Microsoft.IdentityModel.Tokens; using Tgstation.Server.Api.Models.Response; @@ -22,8 +19,7 @@ public interface ITokenFactory /// /// The to create the token for. Must have the field available. /// Whether or not this is an OAuth login. - /// The for the operation. - /// A resulting in a new . - ValueTask CreateToken(Models.User user, bool oAuth, CancellationToken cancellationToken); + /// A new . + TokenResponse CreateToken(Models.User user, bool oAuth); } } diff --git a/src/Tgstation.Server.Host/Security/TokenFactory.cs b/src/Tgstation.Server.Host/Security/TokenFactory.cs index 9ceac820cac..c45a282e2ae 100644 --- a/src/Tgstation.Server.Host/Security/TokenFactory.cs +++ b/src/Tgstation.Server.Host/Security/TokenFactory.cs @@ -1,9 +1,9 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.IdentityModel.Tokens.Jwt; +using System.Linq; using System.Security.Claims; -using System.Threading; -using System.Threading.Tasks; using Microsoft.Extensions.Options; using Microsoft.IdentityModel.Tokens; @@ -11,7 +11,6 @@ using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.System; -using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Security { @@ -26,16 +25,6 @@ sealed class TokenFactory : ITokenFactory /// readonly SecurityConfiguration securityConfiguration; - /// - /// The claim. - /// - readonly Claim issuerClaim; - - /// - /// The claim. - /// - readonly Claim audienceClaim; - /// /// The for generating tokens. /// @@ -46,26 +35,17 @@ sealed class TokenFactory : ITokenFactory /// readonly JwtSecurityTokenHandler tokenHandler; - /// - /// The for the . - /// - readonly IAsyncDelayer asyncDelayer; - /// /// Initializes a new instance of the class. /// - /// The value of . /// The used for generating the . /// The used to generate the issuer name. /// The containing the value of . public TokenFactory( - IAsyncDelayer asyncDelayer, ICryptographySuite cryptographySuite, IAssemblyInformationProvider assemblyInformationProvider, IOptions securityConfigurationOptions) { - this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); - ArgumentNullException.ThrowIfNull(cryptographySuite); ArgumentNullException.ThrowIfNull(assemblyInformationProvider); @@ -94,8 +74,6 @@ public TokenFactory( RequireExpirationTime = true, }; - issuerClaim = new Claim(JwtRegisteredClaimNames.Iss, ValidationParameters.ValidIssuer); - audienceClaim = new Claim(JwtRegisteredClaimNames.Aud, ValidationParameters.ValidAudience); tokenHeader = new JwtHeader( new SigningCredentials( ValidationParameters.IssuerSigningKey, @@ -104,7 +82,7 @@ public TokenFactory( } /// - public async ValueTask CreateToken(Models.User user, bool oAuth, CancellationToken cancellationToken) + public TokenResponse CreateToken(Models.User user, bool oAuth) { ArgumentNullException.ThrowIfNull(user); @@ -117,30 +95,37 @@ public async ValueTask CreateToken(Models.User user, bool oAuth, // this happens occasionally in unit tests // just delay a second so we can force a round up var userLastPassworUpdateUnix = user.LastPasswordUpdate?.ToUnixTimeSeconds(); + DateTimeOffset notBefore; if (nowUnix == userLastPassworUpdateUnix) - await asyncDelayer.Delay(TimeSpan.FromSeconds(1), cancellationToken); + notBefore = now.AddSeconds(1); + else + notBefore = now; var expiry = now.AddMinutes(oAuth ? securityConfiguration.OAuthTokenExpiryMinutes : securityConfiguration.TokenExpiryMinutes); - var claims = new Claim[] - { - new (JwtRegisteredClaimNames.Sub, user.Id.Value.ToString(CultureInfo.InvariantCulture)), - new (JwtRegisteredClaimNames.Exp, expiry.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture)), - new (JwtRegisteredClaimNames.Nbf, nowUnix.ToString(CultureInfo.InvariantCulture)), - issuerClaim, - audienceClaim, - }; var securityToken = new JwtSecurityToken( tokenHeader, - new JwtPayload(claims)); - + new JwtPayload( + ValidationParameters.ValidIssuer, + ValidationParameters.ValidAudience, + Enumerable.Empty(), + new Dictionary + { + { JwtRegisteredClaimNames.Sub, user.Id.Value.ToString(CultureInfo.InvariantCulture) }, + }, + notBefore.UtcDateTime, + expiry.UtcDateTime, + now.UtcDateTime)); + +#pragma warning disable CS0618 // Type or member is obsolete var tokenResponse = new TokenResponse { Bearer = tokenHandler.WriteToken(securityToken), ExpiresAt = expiry, }; +#pragma warning restore CS0618 // Type or member is obsolete return tokenResponse; } diff --git a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs index 1845ee9effa..dae4ff1a34f 100644 --- a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs +++ b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs @@ -200,11 +200,13 @@ static async Task TestServerInformation(IServerClientFactory clientFactory, ISer Assert.AreEqual(RuntimeInformation.IsOSPlatform(OSPlatform.Windows), serverInfo.WindowsHost); //check that modifying the token even slightly fucks up the auth +#pragma warning disable CS0618 // Type or member is obsolete var newToken = new TokenResponse { ExpiresAt = serverClient.Token.ExpiresAt, Bearer = serverClient.Token.Bearer + '0' }; +#pragma warning restore CS0618 // Type or member is obsolete var badClient = clientFactory.CreateFromToken(serverClient.Url, newToken); await ApiAssert.ThrowsException(() => badClient.Administration.Read(cancellationToken)); From e1d359d2f7fa161c0462bcc4ec22e9949980cf3c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 30 Oct 2023 00:33:19 -0400 Subject: [PATCH 086/138] Change instance references to be a `ulong` --- .../Components/IInstanceReference.cs | 2 +- src/Tgstation.Server.Host/Components/InstanceWrapper.cs | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/IInstanceReference.cs b/src/Tgstation.Server.Host/Components/IInstanceReference.cs index a6453747f8c..55f9379e80c 100644 --- a/src/Tgstation.Server.Host/Components/IInstanceReference.cs +++ b/src/Tgstation.Server.Host/Components/IInstanceReference.cs @@ -10,6 +10,6 @@ public interface IInstanceReference : IInstanceCore, IDisposable /// /// A unique ID for the . /// - public Guid Uid { get; } + public ulong Uid { get; } } } diff --git a/src/Tgstation.Server.Host/Components/InstanceWrapper.cs b/src/Tgstation.Server.Host/Components/InstanceWrapper.cs index c8524b50e22..8f44a93b8a0 100644 --- a/src/Tgstation.Server.Host/Components/InstanceWrapper.cs +++ b/src/Tgstation.Server.Host/Components/InstanceWrapper.cs @@ -18,8 +18,13 @@ namespace Tgstation.Server.Host.Components /// sealed class InstanceWrapper : ReferenceCounter, IInstanceReference { + /// + /// Static counter for . + /// + static ulong instanceWrapperInstances; + /// - public Guid Uid { get; } + public ulong Uid { get; } /// public IRepositoryManager RepositoryManager => Instance.RepositoryManager; @@ -44,7 +49,7 @@ sealed class InstanceWrapper : ReferenceCounter, IInstanceReference /// public InstanceWrapper() { - Uid = Guid.NewGuid(); + Uid = Interlocked.Increment(ref instanceWrapperInstances); } /// From f58ea64c96a1752342880a7dbae94d19ba397a79 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 30 Oct 2023 21:53:49 -0400 Subject: [PATCH 087/138] Remove spammy logging --- .../Utils/ApiHeadersProvider.cs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs b/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs index 868a7683c70..df19b90759e 100644 --- a/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs +++ b/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs @@ -1,7 +1,6 @@ -using System; +using System; using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; using Tgstation.Server.Api; @@ -23,11 +22,6 @@ sealed class ApiHeadersProvider : IApiHeadersProvider /// readonly IHttpContextAccessor httpContextAccessor; - /// - /// The for the . - /// - readonly ILogger logger; - /// /// Backing field for . /// @@ -42,11 +36,9 @@ sealed class ApiHeadersProvider : IApiHeadersProvider /// Initializes a new instance of the class. /// /// The value of . - /// The value of . - public ApiHeadersProvider(IHttpContextAccessor httpContextAccessor, ILogger logger) + public ApiHeadersProvider(IHttpContextAccessor httpContextAccessor) { this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException(nameof(httpContextAccessor)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// @@ -78,7 +70,6 @@ ApiHeaders CreateApiHeaders(bool includeAuthAndSetProperties) } catch (HeadersException ex) when (includeAuthAndSetProperties) { - logger.LogTrace(ex, "Failed to parse API headers!"); HeadersException = ex; return null; } From ba0ccdff53f0ffb4a63f37e7fa944e9ef1944515 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 31 Oct 2023 23:13:49 -0400 Subject: [PATCH 088/138] Set `sleep_offline` before `del(world)` --- src/DMAPI/tgs/core/datum.dm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DMAPI/tgs/core/datum.dm b/src/DMAPI/tgs/core/datum.dm index 935263cc955..8d402c9cfec 100644 --- a/src/DMAPI/tgs/core/datum.dm +++ b/src/DMAPI/tgs/core/datum.dm @@ -14,8 +14,8 @@ TGS_DEFINE_AND_SET_GLOBAL(tgs, null) /datum/tgs_api/proc/TerminateWorld() while(TRUE) TGS_DEBUG_LOG("About to terminate world. Tick: [world.time], sleep_offline: [world.sleep_offline]") - del(world) world.sleep_offline = FALSE // https://www.byond.com/forum/post/2894866 + del(world) sleep(1) TGS_DEBUG_LOG("BYOND DIDN'T TERMINATE THE WORLD!!! TICK IS: [world.time], sleep_offline: [world.sleep_offline]") From 242c74955d82f3e7ddae2cf4c43ccf973536f894 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 2 Nov 2023 22:58:56 -0400 Subject: [PATCH 089/138] Correct documentation for ApiHeaders.IsTokenAuthentication --- src/Tgstation.Server.Api/ApiHeaders.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Api/ApiHeaders.cs b/src/Tgstation.Server.Api/ApiHeaders.cs index 98f14b91dba..498d475eaa5 100644 --- a/src/Tgstation.Server.Api/ApiHeaders.cs +++ b/src/Tgstation.Server.Api/ApiHeaders.cs @@ -119,7 +119,7 @@ public sealed class ApiHeaders public OAuthProvider? OAuthProvider { get; } /// - /// If the header uses password or TGS JWT authentication. + /// If the header uses OAuth or TGS JWT authentication. /// public bool IsTokenAuthentication => Token != null && !OAuthProvider.HasValue; From 1b1a91361c7b6c2d42cb70a440b8cd4693c827aa Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 2 Nov 2023 22:59:36 -0400 Subject: [PATCH 090/138] Correct documentation for ApiContoller.NotFound() --- src/Tgstation.Server.Host/Controllers/ApiController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index d0b72134277..399b28a5b09 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -188,7 +188,7 @@ protected override async ValueTask HookExecuteAction(Func e /// /// Generic 404 response. /// - /// An with . + /// A with an appropriate . protected new NotFoundObjectResult NotFound() => NotFound(new ErrorMessageResponse(ErrorCode.ResourceNeverPresent)); /// From fb1aee74a1ecce0db985d423f4179d68b1b4bf58 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 2 Nov 2023 22:59:56 -0400 Subject: [PATCH 091/138] Fix using spacing --- src/Tgstation.Server.Host/Controllers/HomeController.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index 7a821316833..afa526ad925 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Net; using System.Threading; @@ -11,6 +11,7 @@ using Microsoft.Extensions.Options; using Microsoft.Extensions.Primitives; using Microsoft.Net.Http.Headers; + using Octokit; using Tgstation.Server.Api; From d851df82764b6031fee35473a4ff5ac3ffccee70 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 2 Nov 2023 23:00:46 -0400 Subject: [PATCH 092/138] GET / with an Authorization header should fail if it's not valid --- .../Controllers/ApiController.cs | 8 +++++++- .../Controllers/HomeController.cs | 11 ++++++++++- tests/Tgstation.Server.Tests/Live/RawRequestTests.cs | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index 399b28a5b09..deeaa55dc33 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -191,6 +191,12 @@ protected override async ValueTask HookExecuteAction(Func e /// A with an appropriate . protected new NotFoundObjectResult NotFound() => NotFound(new ErrorMessageResponse(ErrorCode.ResourceNeverPresent)); + /// + /// Generic 401 response. + /// + /// An with . + protected new ObjectResult Unauthorized() => this.StatusCode(HttpStatusCode.Unauthorized, null); + /// /// Generic 501 response. /// diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index afa526ad925..6b1c2ebdf85 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using System.Net; using System.Threading; @@ -164,6 +164,8 @@ public IActionResult Home() HeaderNames.Vary, new StringValues(ApiHeaders.ApiVersionHeader)); + // if they tried to authenticate in any form and failed, let them know immediately + bool failIfUnauthed; if (ApiHeaders == null) { if (controlPanelConfiguration.Enable && !Request.Headers.TryGetValue(ApiHeaders.ApiVersionHeader, out _)) @@ -190,7 +192,14 @@ public IActionResult Home() { return HeadersIssue(ex); } + + failIfUnauthed = Request.Headers.Authorization.Any(); } + else + failIfUnauthed = ApiHeaders.Token != null; + + if (failIfUnauthed && !AuthenticationContext.Valid) + return Unauthorized(); return Json(new ServerInformationResponse { diff --git a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs index dae4ff1a34f..7dc5d243f76 100644 --- a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs +++ b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs @@ -210,6 +210,7 @@ static async Task TestServerInformation(IServerClientFactory clientFactory, ISer var badClient = clientFactory.CreateFromToken(serverClient.Url, newToken); await ApiAssert.ThrowsException(() => badClient.Administration.Read(cancellationToken)); + await ApiAssert.ThrowsException(() => badClient.ServerInformation(cancellationToken)); } static async Task TestOAuthFails(IServerClient serverClient, CancellationToken cancellationToken) From be6fe8fa88a8452ec2d52cb3042519e664ec40cf Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 3 Nov 2023 00:34:37 -0400 Subject: [PATCH 093/138] Run systemd as the user `tgstation-server` - Fix .deb build only working in gh-actions tailored environment. - Fix indentation in `rules`. - Fix build script not working fully without a signing key. - Standardize to node 20 as a build dependency. - Fix my ADHD in the `README.md`. Closes #1658 --- .github/CONTRIBUTING.md | 2 +- README.md | 4 +++- build/package/deb/build_package.sh | 20 ++++++++++++++----- build/package/deb/debian/control | 2 +- build/package/deb/debian/postinst | 7 ++++++- build/package/deb/debian/rules | 32 ++++++++++++++++++++---------- build/package/deb/tgs-configure | 4 +--- build/tgstation-server.service | 1 + 8 files changed, 49 insertions(+), 23 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 941aee55759..b8709f641fd 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -34,7 +34,7 @@ You can of course, as always, ask for help at [#coderbus](irc://irc.rizon.net/co ### Development Environment -You need the .NET 8.0 SDK and npm>=v5.7 (in your PATH) to compile the server. +You need the .NET 8.0 SDK, node>=v20, and npm>=v5.7 (in your PATH) to compile the server. The recommended IDE is Visual Studio 2022 or VSCode. diff --git a/README.md b/README.md index a288af1e995..26290b06a11 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,8 @@ sudo dpkg --add-architecture i386 \ && sudo systemctl start tgstation-server ``` +The service will execute as the newly created user: `tgstation-server`. + ##### Debian The `aspnetcore-runtime-8.0` package isn't yet available on mainline Debian and must be [installed from Microsoft](https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian) first. Use the following one-liner to add their packages repository. @@ -143,7 +145,7 @@ The following dependencies are required. [Download the latest release .zip](https://github.com/tgstation/tgstation-server/releases/latest). Choose `ServerConsole`. -If you have SystemD installed, we recommend installing the service unit [here](./build/tgstation-server.service). It assumes TGS is installed into `/opt/tgstation-server` and you will be using the but feel free to adjust it to your needs. Note that the server will need to have it's configuration file setup before running with SystemD. +If you have SystemD installed, we recommend installing the service unit [here](./build/tgstation-server.service). It assumes TGS is installed into `/opt/tgstation-server`, it is executing as the user `tgstation-server`, and you will be using the console runner, but feel free to adjust it to your needs. Note that the server will need to have it's configuration file setup before running with SystemD. Alternatively, to launch the server in the current shell, run `./tgs.sh` in the root of the installation directory. The process will run in a blocking fashion. SIGQUIT will close the server, terminating all live game instances. diff --git a/build/package/deb/build_package.sh b/build/package/deb/build_package.sh index 14dc1a50eda..79f2e57244e 100755 --- a/build/package/deb/build_package.sh +++ b/build/package/deb/build_package.sh @@ -9,16 +9,27 @@ set -x dpkg --add-architecture i386 apt-get update # This package set needs cleanup probably, StackOverflow copypasta -apt-get install -y npm \ +apt-get install -y \ build-essential \ binutils \ lintian \ debhelper \ dh-make \ devscripts \ + ca-certificates \ + curl \ + gnupg \ xmlstarlet # dotnet-sdk-8.0 # Disabled while in preview +# https://github.com/nodesource/distributions +mkdir -p /etc/apt/keyrings +curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg +export NODE_MAJOR=20 +echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +apt-get update +apt-get install nodejs -y + CURRENT_COMMIT=$(git rev-parse HEAD) rm -rf packaging @@ -56,14 +67,13 @@ set +e if [[ -z "$PACKAGING_KEYGRIP" ]]; then dpkg-buildpackage --no-sign + EXIT_CODE=$? else dpkg-buildpackage --sign-key=$PACKAGING_KEYGRIP --sign-command="$SIGN_COMMAND" + cat /tmp/tgs_wrap_gpg_output.log + EXIT_CODE=$? fi -EXIT_CODE=$? - -cat /tmp/tgs_wrap_gpg_output.log - cd .. exit $EXIT_CODE diff --git a/build/package/deb/debian/control b/build/package/deb/debian/control index 486be1bcf27..a63d1d0fab5 100644 --- a/build/package/deb/debian/control +++ b/build/package/deb/debian/control @@ -5,7 +5,7 @@ Maintainer: Jordan Dominion Rules-Requires-Root: no Build-Depends: debhelper-compat (= 13), - npm, + nodejs, #dotnet-sdk-8.0, Disabled while in preview Standards-Version: 4.6.2 Homepage: https://tgstation.github.io/tgstation-server diff --git a/build/package/deb/debian/postinst b/build/package/deb/debian/postinst index 9f71a1931f2..9a1cb1b13c0 100755 --- a/build/package/deb/debian/postinst +++ b/build/package/deb/debian/postinst @@ -1,9 +1,14 @@ #!/bin/sh -e +adduser --system tgstation-server +mkdir -m 754 -p /var/log/tgstation-server +chown -R tgstation-server /etc/tgstation-server +chown -R tgstation-server /opt/tgstation-server +chown -R tgstation-server /var/log/tgstation-server + #DEBHELPER# if [ "$1" = "configure" ]; then - chmod 600 /etc/tgstation-server deb-systemd-helper stop 'tgstation-server.service' >/dev/null || true echo " _ _ _ _ " diff --git a/build/package/deb/debian/rules b/build/package/deb/debian/rules index a68a2cd7d78..7a13e96dd26 100755 --- a/build/package/deb/debian/rules +++ b/build/package/deb/debian/rules @@ -3,23 +3,23 @@ export DH_VERBOSE = 1 %: - dh $@ + dh $@ override_dh_auto_clean: - rm -rf artifacts - dotnet clean -c ReleaseNoWindows + rm -rf artifacts + dotnet clean -c ReleaseNoWindows override_dh_auto_build: - dotnet restore - cd src/Tgstation.Server.Host.Console && dotnet publish -c Release -o ../../artifacts - cd src/Tgstation.Server.Host && dotnet publish -c Release -o ../../artifacts/lib/Default - rm artifacts/lib/Default/appsettings.yml - build/RemoveUnsupportedRuntimes.sh artifacts/lib/Default - build/RemoveUnsupportedRuntimes.sh artifacts + dotnet restore + cd src/Tgstation.Server.Host.Console && dotnet publish -c Release -o ../../artifacts + cd src/Tgstation.Server.Host && dotnet publish -c Release -o ../../artifacts/lib/Default + rm artifacts/lib/Default/appsettings.yml + build/RemoveUnsupportedRuntimes.sh artifacts/lib/Default + build/RemoveUnsupportedRuntimes.sh artifacts override_dh_auto_install: - cp build/package/deb/MakeInstall ./Makefile - dh_auto_install -- + cp build/package/deb/MakeInstall ./Makefile + dh_auto_install -- override_dh_strip: @@ -27,3 +27,13 @@ override_dh_shlibdeps: override_dh_installsystemd: dh_installsystemd -v --restart-after-upgrade + +override_dh_fixperms: + dh_fixperms + find debian/tgstation-server/opt/tgstation-server -exec chmod 544 {} + + find debian/tgstation-server/opt/tgstation-server -type d -exec chmod 555 {} + + find debian/tgstation-server/opt/tgstation-server/lib -exec chmod 744 {} + + find debian/tgstation-server/opt/tgstation-server/lib -type d -exec chmod 755 {} + + find debian/tgstation-server/etc/tgstation-server -exec chmod 644 {} + + find debian/tgstation-server/etc/tgstation-server -type d -exec chmod 755 {} + + chmod 640 debian/tgstation-server/etc/tgstation-server/appsettings.Production.yml diff --git a/build/package/deb/tgs-configure b/build/package/deb/tgs-configure index 96186f68b07..a4bfd3e5f4c 100755 --- a/build/package/deb/tgs-configure +++ b/build/package/deb/tgs-configure @@ -1,5 +1,3 @@ #!/bin/sh -cd /opt/tgstation-server -export General__SetupWizardMode=Only -exec /usr/bin/dotnet /opt/tgstation-server/lib/Default/Tgstation.Server.Host.dll /tmp/tgs_temp_should_not_be_used --appsettings-base-path=/etc/tgstation-server +exec su -s /bin/sh -c "cd /opt/tgstation-server && export General__SetupWizardMode=Only && exec /usr/bin/dotnet /opt/tgstation-server/lib/Default/Tgstation.Server.Host.dll /tmp/tgs_temp_should_not_be_used --appsettings-base-path=/etc/tgstation-server" tgstation-server diff --git a/build/tgstation-server.service b/build/tgstation-server.service index 83e8e47793d..6f3c087584b 100644 --- a/build/tgstation-server.service +++ b/build/tgstation-server.service @@ -7,6 +7,7 @@ After=postgresql.service After=mssql-server.service [Service] +User=tgstation-server Type=notify-reload NotifyAccess=all WorkingDirectory=/opt/tgstation-server From 61cae1a02d54c256c8c94f5913c30097c499b232 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 17:40:54 -0400 Subject: [PATCH 094/138] Fix nested default Linux log directory --- .../Configuration/FileLoggingConfiguration.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs b/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs index 48b08b5b485..ba1bb3c8752 100644 --- a/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs @@ -67,16 +67,17 @@ public string GetFullLogDirectory( ArgumentNullException.ThrowIfNull(assemblyInformationProvider); ArgumentNullException.ThrowIfNull(platformIdentifier); - var directoryToUse = platformIdentifier.IsWindows - ? Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData) // C:/ProgramData - : "/var/log"; // :pain: + if (!String.IsNullOrEmpty(Directory)) + return Directory; - return !String.IsNullOrEmpty(Directory) - ? Directory - : ioManager.ConcatPath( - directoryToUse, + return platformIdentifier.IsWindows + ? ioManager.ConcatPath( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), assemblyInformationProvider.VersionPrefix, - "logs"); + "logs") + : ioManager.ConcatPath( + "/var/log", + assemblyInformationProvider.VersionPrefix); } } } From fda26e1f153e891c7f4d83c9708238446f9bd099 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 3 Nov 2023 02:07:11 -0400 Subject: [PATCH 095/138] Fix issue with DMAPI validation shutdown timeouts - Increase grace period to 30s to avoid false positives. --- src/DMAPI/tgs/core/datum.dm | 1 + .../Components/Session/SessionController.cs | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/DMAPI/tgs/core/datum.dm b/src/DMAPI/tgs/core/datum.dm index 8d402c9cfec..07ce3b68458 100644 --- a/src/DMAPI/tgs/core/datum.dm +++ b/src/DMAPI/tgs/core/datum.dm @@ -16,6 +16,7 @@ TGS_DEFINE_AND_SET_GLOBAL(tgs, null) TGS_DEBUG_LOG("About to terminate world. Tick: [world.time], sleep_offline: [world.sleep_offline]") world.sleep_offline = FALSE // https://www.byond.com/forum/post/2894866 del(world) + world.sleep_offline = FALSE // just in case, this is BYOND after all... sleep(1) TGS_DEBUG_LOG("BYOND DIDN'T TERMINATE THE WORLD!!! TICK IS: [world.time], sleep_offline: [world.sleep_offline]") diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 01ddcea2d41..bf0d22fac7b 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -689,10 +689,12 @@ async Task PostValidationShutdown(Task proceedTask) return; } - Logger.LogDebug("Server will terminated in 10s if it does not exit..."); - var delayTask = asyncDelayer.Delay(TimeSpan.FromSeconds(10), CancellationToken.None); // DCT: None available - var completedTask = await Task.WhenAny(process.Lifetime, delayTask); - if (completedTask == delayTask) + const int GracePeriodSeconds = 30; + Logger.LogDebug("Server will terminated in {gracePeriodSeconds}s if it does not exit...", GracePeriodSeconds); + var delayTask = asyncDelayer.Delay(TimeSpan.FromSeconds(GracePeriodSeconds), CancellationToken.None); // DCT: None available + await Task.WhenAny(process.Lifetime, delayTask); + + if (!process.Lifetime.IsCompleted) { Logger.LogWarning("DMAPI took too long to shutdown server after validation request!"); process.Terminate(); From d85e2b8d36ede54d8916d92c5911eb81e377173f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 31 Oct 2023 23:11:23 -0400 Subject: [PATCH 096/138] Add SignalR - Add one hub `/hubs/jobs`, strongly typed API included. - Adjust request pipeline to support SignalR. - Allow `Accept: text/event-stream` for SSE requests. - Add client library support for hubs. - Add integration tests. - Add IPermissionSetNotifyee to support dynamic changes based on perms. - Keep job state in `JobService` for pushing updates. --- src/Tgstation.Server.Api/ApiHeaders.cs | 30 +- .../Hubs/ConnectionAbortReason.cs | 18 ++ .../Hubs/IErrorHandlingHub.cs | 19 ++ src/Tgstation.Server.Api/Hubs/IJobsHub.cs | 21 ++ src/Tgstation.Server.Api/Routes.cs | 10 + src/Tgstation.Server.Client/ApiClient.cs | 217 ++++++++++--- .../Extensions/HubConnectionExtensions.cs | 104 +++++++ src/Tgstation.Server.Client/IApiClient.cs | 21 +- src/Tgstation.Server.Client/IServerClient.cs | 20 +- .../InfiniteThirtySecondMaxRetryPolicy.cs | 21 ++ src/Tgstation.Server.Client/ServerClient.cs | 14 +- .../ServerClientFactory.cs | 4 +- .../Tgstation.Server.Client.csproj | 7 + .../Components/InstanceWrapper.cs | 3 +- .../Controllers/ApiController.cs | 4 - .../Controllers/HomeController.cs | 8 +- .../Controllers/InstanceController.cs | 16 +- .../InstancePermissionSetController.cs | 15 +- .../Controllers/UserController.cs | 17 +- src/Tgstation.Server.Host/Core/Application.cs | 47 ++- .../ApplicationBuilderExtensions.cs | 29 ++ .../Extensions/ServiceCollectionExtensions.cs | 19 ++ src/Tgstation.Server.Host/Jobs/JobService.cs | 108 ++++++- src/Tgstation.Server.Host/Jobs/JobsHub.cs | 52 ++++ .../Jobs/JobsHubGroupMapper.cs | 172 ++++++++++ .../Security/AuthorizationContextHubFilter.cs | 89 ++++++ .../Security/IPermissionsUpdateNotifyee.cs | 37 +++ .../Tgstation.Server.Host.csproj | 2 + .../Utils/ApiHeadersProvider.cs | 24 +- .../Utils/SignalR/ComprehensiveHubContext.cs | 165 ++++++++++ .../Utils/SignalR/ConnectionMappingHub.cs | 60 ++++ .../SignalR/IConnectionMappedHubContext.cs | 43 +++ .../Utils/SignalR/IHubConnectionMapper.cs | 36 +++ .../TestApiHeaders.cs | 2 +- .../Tgstation.Server.Client.Tests.csproj | 4 + .../Live/Instance/JobsHubTests.cs | 293 ++++++++++++++++++ .../Live/RateLimitRetryingApiClient.cs | 14 +- .../Live/RateLimitRetryingApiClientFactory.cs | 7 +- .../Live/RawRequestTests.cs | 132 +++++++- .../Live/TestLiveServer.cs | 94 ++++-- .../Tgstation.Server.Tests.csproj | 4 + .../Tgstation.Server.Migrator.csproj | 3 +- 42 files changed, 1873 insertions(+), 132 deletions(-) create mode 100644 src/Tgstation.Server.Api/Hubs/ConnectionAbortReason.cs create mode 100644 src/Tgstation.Server.Api/Hubs/IErrorHandlingHub.cs create mode 100644 src/Tgstation.Server.Api/Hubs/IJobsHub.cs create mode 100644 src/Tgstation.Server.Client/Extensions/HubConnectionExtensions.cs create mode 100644 src/Tgstation.Server.Client/InfiniteThirtySecondMaxRetryPolicy.cs create mode 100644 src/Tgstation.Server.Host/Jobs/JobsHub.cs create mode 100644 src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs create mode 100644 src/Tgstation.Server.Host/Security/AuthorizationContextHubFilter.cs create mode 100644 src/Tgstation.Server.Host/Security/IPermissionsUpdateNotifyee.cs create mode 100644 src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs create mode 100644 src/Tgstation.Server.Host/Utils/SignalR/ConnectionMappingHub.cs create mode 100644 src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs create mode 100644 src/Tgstation.Server.Host/Utils/SignalR/IHubConnectionMapper.cs create mode 100644 tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs diff --git a/src/Tgstation.Server.Api/ApiHeaders.cs b/src/Tgstation.Server.Api/ApiHeaders.cs index 498d475eaa5..01e61cb373b 100644 --- a/src/Tgstation.Server.Api/ApiHeaders.cs +++ b/src/Tgstation.Server.Api/ApiHeaders.cs @@ -58,6 +58,11 @@ public sealed class ApiHeaders /// public const string ApplicationJsonMime = "application/json"; + /// + /// Added to in netstandard2.1. Can't use because of lack of .NET Framework support. + /// + const string TextEventStreamMime = "text/event-stream"; + /// /// Get the version of the the caller is using. /// @@ -184,9 +189,10 @@ public ApiHeaders(ProductHeaderValue userAgent, string username, string password /// /// The containing the serialized . /// If a missing should be ignored. + /// If is a valid accept. /// Thrown if the constitue invalid . #pragma warning disable CA1502 // TODO: Decomplexify - public ApiHeaders(RequestHeaders requestHeaders, bool ignoreMissingAuth) + public ApiHeaders(RequestHeaders requestHeaders, bool ignoreMissingAuth, bool allowEventStreamAccept) { if (requestHeaders == null) throw new ArgumentNullException(nameof(requestHeaders)); @@ -207,8 +213,12 @@ void AddError(HeaderErrorTypes headerType, string message) } var jsonAccept = new Microsoft.Net.Http.Headers.MediaTypeHeaderValue(ApplicationJsonMime); - if (!requestHeaders.Accept.Any(x => jsonAccept.IsSubsetOf(x))) - AddError(HeaderErrorTypes.Accept, $"Client does not accept {ApplicationJsonMime}!"); + var eventStreamAccept = new Microsoft.Net.Http.Headers.MediaTypeHeaderValue(TextEventStreamMime); + if (!requestHeaders.Accept.Any(jsonAccept.IsSubsetOf)) + if (!allowEventStreamAccept) + AddError(HeaderErrorTypes.Accept, $"Client does not accept {ApplicationJsonMime}!"); + else if (!requestHeaders.Accept.Any(eventStreamAccept.IsSubsetOf)) + AddError(HeaderErrorTypes.Accept, $"Client does not accept {ApplicationJsonMime} or {TextEventStreamMime}!"); if (!requestHeaders.Headers.TryGetValue(HeaderNames.UserAgent, out var userAgentValues) || userAgentValues.Count == 0) AddError(HeaderErrorTypes.UserAgent, $"Missing {HeaderNames.UserAgent} header!"); @@ -386,6 +396,20 @@ public void SetRequestHeaders(HttpRequestHeaders headers, long? instanceId = nul headers.Add(InstanceIdHeader, instanceId.Value.ToString(CultureInfo.InvariantCulture)); } + /// + /// Adds the necessary for a SignalR hub connection. + /// + /// The headers to write to. + public void SetHubConnectionHeaders(IDictionary headers) + { + if (headers == null) + throw new ArgumentNullException(nameof(headers)); + + headers.Add(HeaderNames.UserAgent, RawUserAgent ?? throw new InvalidOperationException("Missing UserAgent!")); + headers.Add(HeaderNames.Accept, ApplicationJsonMime); + headers.Add(ApiVersionHeader, CreateApiVersionHeader()); + } + /// /// Create the ified for of the . /// diff --git a/src/Tgstation.Server.Api/Hubs/ConnectionAbortReason.cs b/src/Tgstation.Server.Api/Hubs/ConnectionAbortReason.cs new file mode 100644 index 00000000000..05b1cac9b43 --- /dev/null +++ b/src/Tgstation.Server.Api/Hubs/ConnectionAbortReason.cs @@ -0,0 +1,18 @@ +namespace Tgstation.Server.Api.Hubs +{ + /// + /// The reason an aborts a connection. + /// + public enum ConnectionAbortReason + { + /// + /// The provided token is no longer authenticated or authorized to keep the connection. + /// + TokenInvalid, + + /// + /// The server is restarting. + /// + ServerRestart, + } +} diff --git a/src/Tgstation.Server.Api/Hubs/IErrorHandlingHub.cs b/src/Tgstation.Server.Api/Hubs/IErrorHandlingHub.cs new file mode 100644 index 00000000000..844c621d4f1 --- /dev/null +++ b/src/Tgstation.Server.Api/Hubs/IErrorHandlingHub.cs @@ -0,0 +1,19 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Tgstation.Server.Api.Hubs +{ + /// + /// Hub for handling communication errors. + /// + public interface IErrorHandlingHub + { + /// + /// Called if a hub connection or call is attempted with an invalid or unauthorized token. After calling this, the connection is aborted. + /// + /// The . + /// The for the operation. + /// A representing the running operation. + Task AbortingConnection(ConnectionAbortReason reason, CancellationToken cancellationToken); + } +} diff --git a/src/Tgstation.Server.Api/Hubs/IJobsHub.cs b/src/Tgstation.Server.Api/Hubs/IJobsHub.cs new file mode 100644 index 00000000000..01b4aec8bde --- /dev/null +++ b/src/Tgstation.Server.Api/Hubs/IJobsHub.cs @@ -0,0 +1,21 @@ +using System.Threading; +using System.Threading.Tasks; + +using Tgstation.Server.Api.Models.Response; + +namespace Tgstation.Server.Api.Hubs +{ + /// + /// SignalR client methods for receiving s. + /// + public interface IJobsHub : IErrorHandlingHub + { + /// + /// Push a update to the client. + /// + /// The to push. + /// The for the operation. + /// A representing the running operation. + Task ReceiveJobUpdate(JobResponse job, CancellationToken cancellationToken); + } +} diff --git a/src/Tgstation.Server.Api/Routes.cs b/src/Tgstation.Server.Api/Routes.cs index ab8bd05c23f..b62d7816043 100644 --- a/src/Tgstation.Server.Api/Routes.cs +++ b/src/Tgstation.Server.Api/Routes.cs @@ -12,6 +12,11 @@ public static class Routes /// public const string Root = "/"; + /// + /// The root route of all hubs. + /// + public const string HubsRoot = Root + "hubs"; + /// /// The server administration controller. /// @@ -102,6 +107,11 @@ public static class Routes /// public const string List = "List"; + /// + /// The root route of all hubs. + /// + public const string JobsHub = HubsRoot + "/jobs"; + /// /// Apply an postfix to a . /// diff --git a/src/Tgstation.Server.Client/ApiClient.cs b/src/Tgstation.Server.Client/ApiClient.cs index ccc5bb9e9bc..ed48bc7bc7d 100644 --- a/src/Tgstation.Server.Client/ApiClient.cs +++ b/src/Tgstation.Server.Client/ApiClient.cs @@ -11,6 +11,10 @@ using System.Threading.Tasks; using System.Web; +using Microsoft.AspNetCore.Http.Connections; +using Microsoft.AspNetCore.SignalR.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Net.Http.Headers; using Newtonsoft.Json; @@ -20,6 +24,7 @@ using Tgstation.Server.Api; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; +using Tgstation.Server.Client.Extensions; using Tgstation.Server.Common.Extensions; using Tgstation.Server.Common.Http; @@ -51,6 +56,18 @@ public TimeSpan Timeout set => httpClient.Timeout = value; } + /// + /// The to use. + /// + static readonly JsonSerializerSettings SerializerSettings = new () + { + ContractResolver = new CamelCasePropertyNamesContractResolver(), + Converters = new[] + { + new VersionConverter(), + }, + }; + /// /// The for the . /// @@ -61,6 +78,11 @@ public TimeSpan Timeout /// readonly List requestLoggers; + /// + /// List of s created by the . + /// + readonly List hubConnections; + /// /// Backing field for . /// @@ -82,14 +104,9 @@ public TimeSpan Timeout ApiHeaders headers; /// - /// Get the to use. + /// If the is disposed. /// - /// A new instance. - static JsonSerializerSettings GetSerializerSettings() => new () - { - ContractResolver = new CamelCasePropertyNamesContractResolver(), - Converters = new[] { new VersionConverter() }, - }; + bool disposed; /// /// Handle a bad HTTP . @@ -102,7 +119,7 @@ static void HandleBadResponse(HttpResponseMessage response, string json) try { // check if json serializes to an error message - errorMessage = JsonConvert.DeserializeObject(json, GetSerializerSettings()); + errorMessage = JsonConvert.DeserializeObject(json, SerializerSettings); } catch (JsonException) { @@ -149,7 +166,12 @@ static void HandleBadResponse(HttpResponseMessage response, string json) /// The value of . /// The value of . /// The value of . - public ApiClient(IHttpClient httpClient, Uri url, ApiHeaders apiHeaders, ApiHeaders? tokenRefreshHeaders, bool authless) + public ApiClient( + IHttpClient httpClient, + Uri url, + ApiHeaders apiHeaders, + ApiHeaders? tokenRefreshHeaders, + bool authless) { this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); Url = url ?? throw new ArgumentNullException(nameof(url)); @@ -158,12 +180,27 @@ public ApiClient(IHttpClient httpClient, Uri url, ApiHeaders apiHeaders, ApiHead this.authless = authless; requestLoggers = new List(); + hubConnections = new List(); semaphoreSlim = new SemaphoreSlim(1); } /// - public void Dispose() + public async ValueTask DisposeAsync() { + List localHubConnections; + lock (hubConnections) + { + if (disposed) + return; + + disposed = true; + + localHubConnections = hubConnections.ToList(); + hubConnections.Clear(); + } + + await ValueTaskExtensions.WhenAll(hubConnections.Select(connection => connection.DisposeAsync())); + httpClient.Dispose(); semaphoreSlim.Dispose(); } @@ -294,6 +331,128 @@ await RunRequest( } } + /// + /// Attempt to refresh the stored Bearer token in . + /// + /// The for the operation. + /// A resulting in if the refresh was successful, if a refresh is unable to be performed. + public async ValueTask RefreshToken(CancellationToken cancellationToken) + { + if (tokenRefreshHeaders == null) + return false; + + var startingToken = headers.Token; + await semaphoreSlim.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (startingToken != headers.Token) + return true; + + var token = await RunRequest(Routes.Root, new object(), HttpMethod.Post, null, true, cancellationToken).ConfigureAwait(false); + headers = new ApiHeaders(headers.UserAgent!, token); + } + finally + { + semaphoreSlim.Release(); + } + + return true; + } + + /// + public async ValueTask CreateHubConnection( + THubImplementation hubImplementation, + IRetryPolicy? retryPolicy, + Action? loggingConfigureAction, + CancellationToken cancellationToken) + where THubImplementation : class + { + if (hubImplementation == null) + throw new ArgumentNullException(nameof(hubImplementation)); + + retryPolicy ??= new InfiniteThirtySecondMaxRetryPolicy(); + + HubConnection? hubConnection = null; + var hubConnectionBuilder = new HubConnectionBuilder() + .AddNewtonsoftJsonProtocol(options => + { + options.PayloadSerializerSettings = SerializerSettings; + }) + .WithAutomaticReconnect(retryPolicy) + .WithUrl( + new Uri(Url, Routes.JobsHub), + HttpTransportType.ServerSentEvents, + options => + { + options.AccessTokenProvider = async () => + { + // DCT: None available. + if (Headers.Token == null + || (Headers.Token.ParseJwt().ValidTo <= DateTime.UtcNow + && !await RefreshToken(CancellationToken.None))) + { + _ = hubConnection!.StopAsync(); // DCT: None available. + return null; + } + + return Headers.Token.Bearer; + }; + + options.CloseTimeout = Timeout; + + Headers.SetHubConnectionHeaders(options.Headers); + }); + + if (loggingConfigureAction != null) + hubConnectionBuilder.ConfigureLogging(loggingConfigureAction); + + hubConnection = hubConnectionBuilder.Build(); + try + { + hubConnection.Closed += async (error) => + { + if (error is HttpRequestException httpRequestException) + { + // .StatusCode isn't in netstandard but fuck the police + var property = error.GetType().GetProperty("StatusCode"); + if (property != null) + { + var statusCode = (HttpStatusCode?)property.GetValue(error); + if (statusCode == HttpStatusCode.Unauthorized + && !await RefreshToken(CancellationToken.None)) + _ = hubConnection!.StopAsync(); + } + } + }; + + hubConnection.ProxyOn(hubImplementation); + + Task startTask; + lock (hubConnections) + { + if (disposed) + throw new ObjectDisposedException(nameof(ApiClient)); + + hubConnections.Add(hubConnection); + startTask = hubConnection.StartAsync(cancellationToken); + } + + await startTask; + + return hubConnection; + } + catch + { + bool needsDispose; + lock (hubConnections) + needsDispose = hubConnections.Remove(hubConnection); + + if (needsDispose) + await hubConnection.DisposeAsync(); + throw; + } + } + /// /// Main request method. /// @@ -305,6 +464,7 @@ await RunRequest( /// If this is a token refresh operation. /// The for the operation. /// A resulting in the response on success. +#pragma warning disable CA1506 // TODO: Decomplexify protected virtual async ValueTask RunRequest( string route, HttpContent? content, @@ -322,7 +482,7 @@ protected virtual async ValueTask RunRequest( HttpResponseMessage response; var fullUri = new Uri(Url, route); - var serializerSettings = GetSerializerSettings(); + var serializerSettings = SerializerSettings; var fileDownload = typeof(TResult) == typeof(Stream); using (var request = new HttpRequestMessage(method, fullUri)) { @@ -418,38 +578,7 @@ protected virtual async ValueTask RunRequest( } } } - - /// - /// Attempt to refresh the bearer token in the . - /// - /// The for the operation. - /// A resulting in if the refresh was successful, otherwise. - async ValueTask RefreshToken(CancellationToken cancellationToken) - { - if (tokenRefreshHeaders == null) - return false; - - var startingToken = headers.Token; - await semaphoreSlim.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - if (startingToken != headers.Token) - return true; - - var token = await RunRequest(Routes.Root, new object(), HttpMethod.Post, null, true, cancellationToken).ConfigureAwait(false); - headers = new ApiHeaders(headers.UserAgent!, token); - } - catch (ClientException) - { - return false; - } - finally - { - semaphoreSlim.Release(); - } - - return true; - } +#pragma warning restore CA1506 /// /// Main request method. @@ -475,7 +604,7 @@ async ValueTask RunRequest( HttpContent? content = null; if (body != null) content = new StringContent( - JsonConvert.SerializeObject(body, typeof(TBody), Formatting.None, GetSerializerSettings()), + JsonConvert.SerializeObject(body, typeof(TBody), Formatting.None, SerializerSettings), Encoding.UTF8, ApiHeaders.ApplicationJsonMime); diff --git a/src/Tgstation.Server.Client/Extensions/HubConnectionExtensions.cs b/src/Tgstation.Server.Client/Extensions/HubConnectionExtensions.cs new file mode 100644 index 00000000000..44351272f0c --- /dev/null +++ b/src/Tgstation.Server.Client/Extensions/HubConnectionExtensions.cs @@ -0,0 +1,104 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.SignalR.Client; + +namespace Tgstation.Server.Client.Extensions +{ + /// + /// Extension methods for the . + /// + static class HubConnectionExtensions + { + /// + /// Apply a given to a given . + /// + /// The strongly typed client proxy. + /// The to proxy on. + /// The to forward operations to. + public static void ProxyOn(this HubConnection hubConnection, TClientProxy proxy) + where TClientProxy : class + { + if (hubConnection == null) + throw new ArgumentNullException(nameof(hubConnection)); + + if (proxy == null) + throw new ArgumentNullException(nameof(proxy)); + + ProxyOn(hubConnection, typeof(TClientProxy), proxy); + } + + /// + /// Apply a given to a given . + /// + /// The to proxy on. + /// The of . + /// The to forward operations to. + static void ProxyOn(this HubConnection hubConnection, Type proxyType, object proxyObject) + { + var clientMethods = proxyType.GetMethods(); + var cancellationTokenType = typeof(CancellationToken); + foreach (var clientMethod in clientMethods) + { + var parametersList = clientMethod + .GetParameters() + .Select(parameterInfo => parameterInfo.ParameterType) + .ToList(); + + var cancellationTokenIndex = parametersList.IndexOf(cancellationTokenType); + if (cancellationTokenIndex != -1) + { + parametersList.RemoveAt(cancellationTokenIndex); +#if DEBUG + if (parametersList.IndexOf(cancellationTokenType) != -1) + throw new InvalidOperationException("Cannot ProxyOn a method with multiple CancellationToken parameters!"); +#endif + } + + var parameters = parametersList.ToArray(); + + object?[] AddCancellationTokenToParametersArray(object?[] parametersArray) + { + if (cancellationTokenIndex == -1) + return parametersArray; + + var newList = parametersArray.ToList(); + newList.Insert(cancellationTokenIndex, CancellationToken.None); + return newList.ToArray(); + } + + var returnType = clientMethod.ReturnType; + if (returnType != typeof(Task)) + { + if (returnType.BaseType != typeof(Task)) + throw new InvalidOperationException($"Return type {returnType} of {proxyType.FullName}.{clientMethod.Name} is not supported! Only Task and derivatives are supported."); + + var resultProperty = returnType.GetProperty(nameof(Task.Result)); + hubConnection.On( + clientMethod.Name, + parameters, + async (parameterArray, _) => + { + var task = (Task)clientMethod.Invoke(proxyObject, AddCancellationTokenToParametersArray(parameterArray)); + await task; + return resultProperty.GetValue(task); + }, + hubConnection); + } + else + hubConnection.On( + clientMethod.Name, + parameters, + (parameterArray) => + { + return (Task)clientMethod.Invoke(proxyObject, AddCancellationTokenToParametersArray(parameterArray)); + }); + } + + foreach (var inheritedInterface in proxyType.GetInterfaces()) + ProxyOn(hubConnection, inheritedInterface, proxyObject); + } + } +} diff --git a/src/Tgstation.Server.Client/IApiClient.cs b/src/Tgstation.Server.Client/IApiClient.cs index 836a9c71149..10d6477e3e5 100644 --- a/src/Tgstation.Server.Client/IApiClient.cs +++ b/src/Tgstation.Server.Client/IApiClient.cs @@ -3,6 +3,9 @@ using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.SignalR.Client; +using Microsoft.Extensions.Logging; + using Tgstation.Server.Api; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; @@ -12,7 +15,7 @@ namespace Tgstation.Server.Client /// /// Web interface for the API. /// - interface IApiClient : IDisposable + interface IApiClient : IAsyncDisposable { /// /// The the uses. @@ -35,6 +38,22 @@ interface IApiClient : IDisposable /// The to add. void AddRequestLogger(IRequestLogger requestLogger); + /// + /// Subscribe to all job updates available to the . + /// + /// The of the hub being implemented. + /// The to use for proxying the methods of the hub connection. + /// The optional to use for the backing connection. The default retry policy waits for 1, 2, 4, 8, and 16 seconds, then 30s repeatedly. + /// The optional used to configure a . + /// The for the operation. + /// An representing the lifetime of the subscription. + ValueTask CreateHubConnection( + THubImplementation hubImplementation, + IRetryPolicy? retryPolicy, + Action? loggingConfigureAction, + CancellationToken cancellationToken) + where THubImplementation : class; + /// /// Run an HTTP PUT request. /// diff --git a/src/Tgstation.Server.Client/IServerClient.cs b/src/Tgstation.Server.Client/IServerClient.cs index f5dc8213653..65014f66b1e 100644 --- a/src/Tgstation.Server.Client/IServerClient.cs +++ b/src/Tgstation.Server.Client/IServerClient.cs @@ -2,6 +2,10 @@ using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.SignalR.Client; +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Api.Hubs; using Tgstation.Server.Api.Models.Response; namespace Tgstation.Server.Client @@ -9,7 +13,7 @@ namespace Tgstation.Server.Client /// /// Main client for communicating with a server. /// - public interface IServerClient : IDisposable + public interface IServerClient : IAsyncDisposable { /// /// The connected server . @@ -53,6 +57,20 @@ public interface IServerClient : IDisposable /// A resulting in the of the target server. ValueTask ServerInformation(CancellationToken cancellationToken); + /// + /// Subscribe to all job updates available to the . + /// + /// The to use to subscribe to updates. + /// The optional to use for the backing connection. The default retry policy waits for 1, 2, 4, 8, and 16 seconds, then 30s repeatedly. + /// The optional used to configure a . + /// The for the operation. + /// An representing the lifetime of the subscription. + ValueTask SubscribeToJobUpdates( + IJobsHub jobsReceiver, + IRetryPolicy? retryPolicy = null, + Action? loggingConfigureAction = null, + CancellationToken cancellationToken = default); + /// /// Adds a to the request pipeline. /// diff --git a/src/Tgstation.Server.Client/InfiniteThirtySecondMaxRetryPolicy.cs b/src/Tgstation.Server.Client/InfiniteThirtySecondMaxRetryPolicy.cs new file mode 100644 index 00000000000..8452631306f --- /dev/null +++ b/src/Tgstation.Server.Client/InfiniteThirtySecondMaxRetryPolicy.cs @@ -0,0 +1,21 @@ +using System; + +using Microsoft.AspNetCore.SignalR.Client; + +namespace Tgstation.Server.Client +{ + /// + /// A that returns seconds in powers of 2, maxing out at 30s. + /// + sealed class InfiniteThirtySecondMaxRetryPolicy : IRetryPolicy + { + /// + public TimeSpan? NextRetryDelay(RetryContext retryContext) + { + if (retryContext == null) + throw new ArgumentNullException(nameof(retryContext)); + + return TimeSpan.FromSeconds(Math.Min(Math.Pow(2, retryContext.PreviousRetryCount), 30)); + } + } +} diff --git a/src/Tgstation.Server.Client/ServerClient.cs b/src/Tgstation.Server.Client/ServerClient.cs index 5826b76bdc6..186e32745b2 100644 --- a/src/Tgstation.Server.Client/ServerClient.cs +++ b/src/Tgstation.Server.Client/ServerClient.cs @@ -2,7 +2,11 @@ using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.SignalR.Client; +using Microsoft.Extensions.Logging; + using Tgstation.Server.Api; +using Tgstation.Server.Api.Hubs; using Tgstation.Server.Api.Models.Response; namespace Tgstation.Server.Client @@ -59,12 +63,20 @@ public ServerClient(IApiClient apiClient) } /// - public void Dispose() => apiClient.Dispose(); + public ValueTask DisposeAsync() => apiClient.DisposeAsync(); /// public ValueTask ServerInformation(CancellationToken cancellationToken) => apiClient.Read(Routes.Root, cancellationToken); /// public void AddRequestLogger(IRequestLogger requestLogger) => apiClient.AddRequestLogger(requestLogger); + + /// + public ValueTask SubscribeToJobUpdates( + IJobsHub jobsReceiver, + IRetryPolicy? retryPolicy, + Action? loggingConfigureAction, + CancellationToken cancellationToken) + => apiClient.CreateHubConnection(jobsReceiver, retryPolicy, loggingConfigureAction, cancellationToken); } } diff --git a/src/Tgstation.Server.Client/ServerClientFactory.cs b/src/Tgstation.Server.Client/ServerClientFactory.cs index 2d464ec0ecc..155198996a2 100644 --- a/src/Tgstation.Server.Client/ServerClientFactory.cs +++ b/src/Tgstation.Server.Client/ServerClientFactory.cs @@ -120,7 +120,7 @@ public async ValueTask GetServerInformation( TimeSpan? timeout = null, CancellationToken cancellationToken = default) { - using var api = ApiClientFactory.CreateApiClient( + await using var api = ApiClientFactory.CreateApiClient( host, new ApiHeaders( productHeaderValue, @@ -162,7 +162,7 @@ async ValueTask CreateWithNewToken( requestLoggers ??= Enumerable.Empty(); TokenResponse token; - using (var api = ApiClientFactory.CreateApiClient(host, loginHeaders, null, false)) + await using (var api = ApiClientFactory.CreateApiClient(host, loginHeaders, null, false)) { foreach (var requestLogger in requestLoggers) api.AddRequestLogger(requestLogger); diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 6868098fa69..e6c2aaaea74 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -9,6 +9,13 @@ $(TGS_NUGET_RELEASE_NOTES_CLIENT) + + + + + + + diff --git a/src/Tgstation.Server.Host/Components/InstanceWrapper.cs b/src/Tgstation.Server.Host/Components/InstanceWrapper.cs index 8f44a93b8a0..1e1359c3e9b 100644 --- a/src/Tgstation.Server.Host/Components/InstanceWrapper.cs +++ b/src/Tgstation.Server.Host/Components/InstanceWrapper.cs @@ -1,5 +1,4 @@ -using System; -using System.Threading; +using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.Components.Byond; diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index deeaa55dc33..b3136a115f0 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -116,10 +116,6 @@ protected override async ValueTask HookExecuteAction(Func e if (requireHeaders) return HeadersIssue(ApiHeadersProvider.HeadersException); } - else if (!ApiHeaders.Compatible()) - return this.StatusCode( - HttpStatusCode.UpgradeRequired, - new ErrorMessageResponse(ErrorCode.ApiMismatch)); var errorCase = await ValidateRequest(cancellationToken); if (errorCase != null) diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index 6b1c2ebdf85..b33c5bf77d9 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -1,6 +1,5 @@ using System; using System.Linq; -using System.Net; using System.Threading; using System.Threading.Tasks; @@ -21,7 +20,6 @@ using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Database; -using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.Security.OAuth; @@ -182,11 +180,7 @@ public IActionResult Home() try { // we only allow authorization header issues - var headers = ApiHeadersProvider.CreateAuthlessHeaders(); - if (!headers.Compatible()) - return this.StatusCode( - HttpStatusCode.UpgradeRequired, - new ErrorMessageResponse(ErrorCode.ApiMismatch)); + ApiHeadersProvider.CreateAuthlessHeaders(); } catch (HeadersException ex) { diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index d4ae7122817..7310f56f1c6 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -68,6 +68,11 @@ public sealed class InstanceController : ComponentInterfacingController /// readonly IPortAllocator portAllocator; + /// + /// The for the . + /// + readonly IPermissionsUpdateNotifyee permissionsUpdateNotifyee; + /// /// The for the . /// @@ -88,7 +93,8 @@ public sealed class InstanceController : ComponentInterfacingController /// The value of . /// The value of . /// The value of . - /// The value of . + /// The value of . + /// The value of . /// The containing the value of . /// The containing the value of . /// The for the . @@ -101,6 +107,7 @@ public InstanceController( IIOManager ioManager, IPortAllocator portAllocator, IPlatformIdentifier platformIdentifier, + IPermissionsUpdateNotifyee permissionsUpdateNotifyee, IOptions generalConfigurationOptions, IOptions swarmConfigurationOptions, IApiHeadersProvider apiHeaders) @@ -116,6 +123,8 @@ public InstanceController( this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); this.portAllocator = portAllocator ?? throw new ArgumentNullException(nameof(portAllocator)); + this.permissionsUpdateNotifyee = permissionsUpdateNotifyee ?? throw new ArgumentNullException(nameof(permissionsUpdateNotifyee)); + generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); swarmConfiguration = swarmConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(swarmConfigurationOptions)); } @@ -267,6 +276,10 @@ async ValueTask DirExistsAndIsNotEmpty() newInstance.Id, newInstance.Path); + await permissionsUpdateNotifyee.InstancePermissionSetCreated( + newInstance.InstancePermissionSets.First(), + cancellationToken); + var api = newInstance.ToApi(); api.Accessible = true; // instances are always accessible by their creator return attached ? Json(api) : Created(api); @@ -770,6 +783,7 @@ InstancePermissionSet InstanceAdminPermissionSet(InstancePermissionSet permissio { permissionSetToModify ??= new InstancePermissionSet() { + PermissionSet = AuthenticationContext.PermissionSet, PermissionSetId = AuthenticationContext.PermissionSet.Id.Value, }; permissionSetToModify.ByondRights = RightsHelper.AllRights(); diff --git a/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs b/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs index 5c1a91e6ba4..30a1827de3d 100644 --- a/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstancePermissionSetController.cs @@ -30,19 +30,26 @@ namespace Tgstation.Server.Host.Controllers [Route(Routes.InstancePermissionSet)] public sealed class InstancePermissionSetController : InstanceRequiredController { + /// + /// The for the . + /// + readonly IPermissionsUpdateNotifyee permissionsUpdateNotifyee; + /// /// Initializes a new instance of the class. /// /// The for the . - /// The for the . + /// The for the . /// The for the . /// The for the . + /// The value of . /// The for the . public InstancePermissionSetController( IDatabaseContext databaseContext, IAuthenticationContext authenticationContext, ILogger logger, IInstanceManager instanceManager, + IPermissionsUpdateNotifyee permissionsUpdateNotifyee, IApiHeadersProvider apiHeaders) : base( databaseContext, @@ -51,6 +58,7 @@ public InstancePermissionSetController( instanceManager, apiHeaders) { + this.permissionsUpdateNotifyee = permissionsUpdateNotifyee ?? throw new ArgumentNullException(nameof(permissionsUpdateNotifyee)); } /// @@ -76,6 +84,7 @@ public async ValueTask Create([FromBody] InstancePermissionSetReq .Where(x => x.Id == model.PermissionSetId) .Select(x => new Models.PermissionSet { + Id = x.Id, UserId = x.UserId, }) .FirstOrDefaultAsync(cancellationToken); @@ -112,6 +121,10 @@ public async ValueTask Create([FromBody] InstancePermissionSetReq DatabaseContext.InstancePermissionSets.Add(dbUser); await DatabaseContext.Save(cancellationToken); + + // needs to be set for next call + dbUser.PermissionSet = existingPermissionSet; + await permissionsUpdateNotifyee.InstancePermissionSetCreated(dbUser, cancellationToken); return Created(dbUser.ToApi()); } #pragma warning restore CA1506 diff --git a/src/Tgstation.Server.Host/Controllers/UserController.cs b/src/Tgstation.Server.Host/Controllers/UserController.cs index 38c75043021..08c61e05fe0 100644 --- a/src/Tgstation.Server.Host/Controllers/UserController.cs +++ b/src/Tgstation.Server.Host/Controllers/UserController.cs @@ -40,6 +40,11 @@ public sealed class UserController : ApiController /// readonly ICryptographySuite cryptographySuite; + /// + /// The for the . + /// + readonly IPermissionsUpdateNotifyee permissionsUpdateNotifyee; + /// /// The for the . /// @@ -52,6 +57,7 @@ public sealed class UserController : ApiController /// The for the . /// The value of . /// The value of . + /// The value of . /// The for the . /// The containing the value of . /// The for the . @@ -60,6 +66,7 @@ public UserController( IAuthenticationContext authenticationContext, ISystemIdentityFactory systemIdentityFactory, ICryptographySuite cryptographySuite, + IPermissionsUpdateNotifyee permissionsUpdateNotifyee, ILogger logger, IOptions generalConfigurationOptions, IApiHeadersProvider apiHeaders) @@ -72,6 +79,7 @@ public UserController( { this.systemIdentityFactory = systemIdentityFactory ?? throw new ArgumentNullException(nameof(systemIdentityFactory)); this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); + this.permissionsUpdateNotifyee = permissionsUpdateNotifyee ?? throw new ArgumentNullException(nameof(permissionsUpdateNotifyee)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); } @@ -243,13 +251,17 @@ public async ValueTask Update([FromBody] UserUpdateRequest model, if (model.Name != null && Models.User.CanonicalizeName(model.Name) != originalUser.CanonicalName) return BadRequest(new ErrorMessageResponse(ErrorCode.UserNameChange)); + bool userWasDisabled; if (model.Enabled.HasValue) { - if (originalUser.Enabled.Value && !model.Enabled.Value) + userWasDisabled = originalUser.Enabled.Value && !model.Enabled.Value; + if (userWasDisabled) originalUser.LastPasswordUpdate = DateTimeOffset.UtcNow; originalUser.Enabled = model.Enabled.Value; } + else + userWasDisabled = false; if (model.OAuthConnections != null && (model.OAuthConnections.Count != originalUser.OAuthConnections.Count @@ -315,6 +327,9 @@ public async ValueTask Update([FromBody] UserUpdateRequest model, Logger.LogInformation("Updated user {userName} ({userId})", originalUser.Name, originalUser.Id); + if (userWasDisabled) + await permissionsUpdateNotifyee.UserDisabled(originalUser, cancellationToken); + // return id only if not a self update and cannot read users var canReadBack = AuthenticationContext.User.Id == originalUser.Id || callerAdministrationRights.HasFlag(AdministrationRights.ReadUsers); diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 7253e90d687..b97a21600f0 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -14,9 +14,12 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Cors.Infrastructure; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Connections; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.AspNetCore.Mvc.Infrastructure; +using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -31,6 +34,7 @@ using Serilog.Sinks.Elasticsearch; using Tgstation.Server.Api; +using Tgstation.Server.Api.Hubs; using Tgstation.Server.Common.Http; using Tgstation.Server.Host.Components; using Tgstation.Server.Host.Components.Byond; @@ -250,6 +254,18 @@ void ConfigureNewtonsoftJsonSerializerSettingsForApi(JsonSerializerSettings sett ConfigureNewtonsoftJsonSerializerSettingsForApi(options.SerializerSettings); }); + services.AddSignalR( + options => + { + options.AddFilter(); + }) + .AddNewtonsoftJsonProtocol(options => + { + ConfigureNewtonsoftJsonSerializerSettingsForApi(options.PayloadSerializerSettings); + }); + + services.AddHub(); + if (postSetupServices.GeneralConfiguration.HostApiDocumentation) { string GetDocumentationFilePath(string assemblyLocation) => ioManager.ConcatPath(ioManager.GetDirectoryName(assemblyLocation), String.Concat(ioManager.GetFileNameWithoutExtension(assemblyLocation), ".xml")); @@ -386,6 +402,7 @@ void AddTypedContext() // configure root services services.AddSingleton(); services.AddSingleton(x => x.GetRequiredService()); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(x => x.GetRequiredService()); @@ -501,6 +518,9 @@ public void Configure( logger.LogTrace("Web control panel disabled!"); #endif + // validate the API version + applicationBuilder.UseApiCompatibility(); + // authenticate JWT tokens using our security pipeline if present, returns 401 if bad applicationBuilder.UseAuthentication(); @@ -513,6 +533,17 @@ public void Configure( // setup endpoints applicationBuilder.UseEndpoints(endpoints => { + // access to the signalR jobs hub + endpoints.MapHub( + Routes.JobsHub, + options => + { + options.Transports = HttpTransportType.ServerSentEvents; + options.CloseOnAuthenticationExpiration = true; + }) + .RequireAuthorization() + .RequireCors(corsBuilder); + // majority of handling is done in the controllers endpoints.MapControllers(); }); @@ -541,10 +572,18 @@ void ConfigureAuthenticationPipeline(IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(provider => provider.GetRequiredService()); - services.AddScoped(provider => - { - return provider.GetRequiredService().CurrentAuthenticationContext; - }); + + // what if you + // wanted to just do this: + // return provider.GetRequiredService().CurrentAuthenticationContext + // But M$ said + // https://stackoverflow.com/questions/56792917/scoped-services-in-asp-net-core-with-signalr-hubs + services.AddScoped(provider => provider + .GetRequiredService() + .HttpContext + .RequestServices + .GetRequiredService() + .CurrentAuthenticationContext); services.AddScoped(); services.AddScoped(); diff --git a/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs b/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs index ae8eb65ae61..55086228dc6 100644 --- a/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs @@ -119,6 +119,35 @@ public static void UseServerErrorHandling(this IApplicationBuilder applicationBu }); } + /// + /// Check that the API version is the current major version if it's present in the headers. + /// + /// The to configure. + public static void UseApiCompatibility(this IApplicationBuilder applicationBuilder) + { + ArgumentNullException.ThrowIfNull(applicationBuilder); + + applicationBuilder.Use(async (context, next) => + { + var apiHeadersProvider = context.RequestServices.GetRequiredService(); + if (apiHeadersProvider.ApiHeaders?.Compatible() == false) + { + await new JsonResult( + new ErrorMessageResponse(ErrorCode.ApiMismatch)) + { + StatusCode = (int)HttpStatusCode.UpgradeRequired, + } + .ExecuteResultAsync(new ActionContext + { + HttpContext = context, + }); + return; + } + + await next(); + }); + } + /// /// Add the X-Powered-By response header. /// diff --git a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs index c4d7e029b51..6693ca670ef 100644 --- a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs @@ -11,11 +11,13 @@ using Serilog.Configuration; using Serilog.Sinks.Elasticsearch; +using Tgstation.Server.Api.Hubs; using Tgstation.Server.Host.Components.Chat.Providers; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Utils; using Tgstation.Server.Host.Utils.GitHub; +using Tgstation.Server.Host.Utils.SignalR; namespace Tgstation.Server.Host.Extensions { @@ -220,6 +222,23 @@ public static IServiceCollection SetupLogging( }); } + /// + /// Attempt to add the given to services. + /// + /// The of the being added. + /// The implementation of the . + /// The to add the to. + public static void AddHub(this IServiceCollection services) + where THub : ConnectionMappingHub + where THubMethods : class, IErrorHandlingHub + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(typeof(ComprehensiveHubContext<,>)); + services.AddSingleton>(provider => provider.GetRequiredService>()); + services.AddSingleton>(provider => provider.GetRequiredService>()); + } + /// /// Set the modifiable services to their default types. /// diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index b5029e2fbe8..92d50965429 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -1,12 +1,17 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; + using Serilog.Context; + +using Tgstation.Server.Api.Hubs; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Common.Extensions; using Tgstation.Server.Host.Components; @@ -14,12 +19,23 @@ using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.SignalR; namespace Tgstation.Server.Host.Jobs { /// sealed class JobService : IJobService, IDisposable { + /// + /// The maximum rate at which hub clients can receive updates. + /// + const int MaxHubUpdatesPerSecond = 4; + + /// + /// The for the . + /// + readonly IConnectionMappedHubContext hub; + /// /// The for the . /// @@ -63,17 +79,21 @@ sealed class JobService : IJobService, IDisposable /// /// Initializes a new instance of the class. /// + /// The value of . /// The value of . /// The value of . /// The value of . public JobService( + IConnectionMappedHubContext hub, IDatabaseContextFactory databaseContextFactory, ILoggerFactory loggerFactory, ILogger logger) { + this.hub = hub ?? throw new ArgumentNullException(nameof(hub)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + jobs = new Dictionary(); activationTcs = new TaskCompletionSource(); synchronizationLock = new object(); @@ -269,13 +289,12 @@ public void SetJobProgress(JobResponse apiResponse) if (noMoreJobsShouldStart && !handler.Started) await Extensions.TaskExtensions.InfiniteTask.WaitAsync(cancellationToken); - ValueTask? cancelTask = null; + var cancelTask = ValueTask.FromResult(null); bool result; using (jobCancellationToken.Register(() => cancelTask = CancelJob(job, canceller, true, cancellationToken))) result = await handler.Wait(cancellationToken); - if (cancelTask.HasValue) - await cancelTask.Value; + await cancelTask; return result; } @@ -292,23 +311,60 @@ public void Activate(IInstanceCoreProvider instanceCoreProvider) /// /// Runner for s. /// - /// The being run. + /// The being run. Must be fully populated. /// The for the . /// The for the operation. /// A representing the running operation. +#pragma warning disable CA1506 // TODO: Decomplexify async Task RunJob(Job job, JobEntrypoint operation, CancellationToken cancellationToken) +#pragma warning restore CA1506 { using (LogContext.PushProperty(SerilogContextHelper.JobIdContextProperty, job.Id)) try { void LogException(Exception ex) => logger.LogDebug(ex, "Job {jobId} exited with error!", job.Id); + var hubUpdatesTask = Task.CompletedTask; var result = false; - try + + Stopwatch stopwatch = null; + void QueueHubUpdate(JobResponse update) { - var oldJob = job; - job = new Job { Id = oldJob.Id }; + var currentUpdatesTask = hubUpdatesTask; + async Task ChainHubUpdate() + { + await currentUpdatesTask; + // DCT: Cancellation token is for job, operation should always run + await hub + .Clients + .Group(JobsHub.HubGroupName(job)) + .ReceiveJobUpdate(update, CancellationToken.None); + } + + Stopwatch enteredLock = null; + try + { + if (stopwatch != null) + { + Monitor.Enter(stopwatch); + enteredLock = stopwatch; + if (stopwatch.ElapsedMilliseconds * MaxHubUpdatesPerSecond < 1) + return; // don't spam client + } + + hubUpdatesTask = ChainHubUpdate(); + stopwatch = Stopwatch.StartNew(); + } + finally + { + if (enteredLock != null) + Monitor.Exit(enteredLock); + } + } + + try + { void UpdateProgress(string stage, double? progress) { if (progress.HasValue @@ -319,19 +375,26 @@ void UpdateProgress(string stage, double? progress) return; } + int? newProgress = progress.HasValue ? (int)Math.Floor(progress.Value * 100) : null; lock (synchronizationLock) - if (jobs.TryGetValue(oldJob.Id.Value, out var handler)) + if (jobs.TryGetValue(job.Id.Value, out var handler)) { handler.Stage = stage; - handler.Progress = progress.HasValue ? (int)Math.Floor(progress.Value * 100) : null; + handler.Progress = newProgress; + + var updatedJob = job.ToApi(); + updatedJob.Stage = stage; + updatedJob.Progress = newProgress; + QueueHubUpdate(updatedJob); } } var instanceCoreProvider = await activationTcs.Task.WaitAsync(cancellationToken); + QueueHubUpdate(job.ToApi()); logger.LogTrace("Starting job..."); await operation( - instanceCoreProvider.GetInstance(oldJob.Instance), + instanceCoreProvider.GetInstance(job.Instance), databaseContextFactory, job, new JobProgressReporter( @@ -377,6 +440,31 @@ await databaseContextFactory.UseContext(async databaseContext => await databaseContext.Save(CancellationToken.None); }); + // Resetting the context here because I CBA to worry if the cache is being used + await databaseContextFactory.UseContext(async databaseContext => + { + // Cancellation might be set in another async context, forced to reload here for the final hub update + // DCT: Cancellation token is for job, operation should always run + var finalJob = await databaseContext + .Jobs + .AsQueryable() + .Include(x => x.Instance) + .Include(x => x.StartedBy) + .Include(x => x.CancelledBy) + .Where(dbJob => dbJob.Id == job.Id.Value) + .FirstAsync(CancellationToken.None); + QueueHubUpdate(finalJob.ToApi()); + }); + + try + { + await hubUpdatesTask; + } + catch (Exception ex) + { + logger.LogError(ex, "Error in hub updates chain task!"); + } + return result; } finally diff --git a/src/Tgstation.Server.Host/Jobs/JobsHub.cs b/src/Tgstation.Server.Host/Jobs/JobsHub.cs new file mode 100644 index 00000000000..b74e19d22b0 --- /dev/null +++ b/src/Tgstation.Server.Host/Jobs/JobsHub.cs @@ -0,0 +1,52 @@ +using System; + +using Microsoft.AspNetCore.SignalR; + +using Tgstation.Server.Api.Hubs; +using Tgstation.Server.Host.Models; +using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Utils.SignalR; + +namespace Tgstation.Server.Host.Jobs +{ + /// + /// A SignalR for pushing job updates. + /// + sealed class JobsHub : ConnectionMappingHub + { + /// + /// Get the group name for a given . + /// + /// The . + /// The name of the group for the . + public static string HubGroupName(long instanceId) + => $"instance-{instanceId}"; + + /// + /// Get the group name for a given . + /// + /// The . + /// The name of the group for the . + public static string HubGroupName(Job job) + { + ArgumentNullException.ThrowIfNull(job); + + if (job.Instance == null) + throw new InvalidOperationException("job.Instance was null!"); + + return HubGroupName(job.Instance.Id.Value); + } + + /// + /// Initializes a new instance of the class. + /// + /// The for the . + /// The for the . + public JobsHub( + IHubConnectionMapper connectionMapper, + IAuthenticationContext authenticationContext) + : base(connectionMapper, authenticationContext) + { + } + } +} diff --git a/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs b/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs new file mode 100644 index 00000000000..a05d3283b58 --- /dev/null +++ b/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.SignalR; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Api.Hubs; +using Tgstation.Server.Host.Database; +using Tgstation.Server.Host.Models; +using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Utils.SignalR; + +namespace Tgstation.Server.Host.Jobs +{ + /// + /// Handles mapping groups for the . + /// + sealed class JobsHubGroupMapper : IPermissionsUpdateNotifyee + { + /// + /// The for the . + /// + readonly IConnectionMappedHubContext hub; + + /// + /// The for the . + /// + readonly IDatabaseContextFactory databaseContextFactory; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + public JobsHubGroupMapper(IConnectionMappedHubContext hub, IDatabaseContextFactory databaseContextFactory, ILogger logger) + { + this.hub = hub ?? throw new ArgumentNullException(nameof(hub)); + this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + hub.OnConnectionMapGroups += MapConnectionGroups; + } + + /// + public ValueTask InstancePermissionSetCreated(InstancePermissionSet instancePermissionSet, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(instancePermissionSet); + var permissionSetId = instancePermissionSet.PermissionSet.Id ?? instancePermissionSet.PermissionSetId; + + logger.LogTrace("InstancePermissionSetCreated"); + return RefreshHubGroups( + permissionSetId, + cancellationToken); + } + + /// + public ValueTask UserDisabled(User user, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(user); + if (!user.Id.HasValue) + throw new InvalidOperationException("user.Id was null!"); + + logger.LogTrace("UserDisabled"); + return hub.NotifyAndAbortUnauthedConnections(user, cancellationToken); + } + + /// + public ValueTask InstancePermissionSetDeleted(PermissionSet permissionSet, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(permissionSet); + logger.LogTrace("InstancePermissionSetDeleted"); + return RefreshHubGroups( + permissionSet.Id ?? throw new InvalidOperationException("permissionSet?.Id was null!"), + cancellationToken); + } + + /// + /// Implementation of . + /// + /// The to map the groups for. + /// The for the operation. + /// A resulting in an of the group names the user belongs in. + async ValueTask> MapConnectionGroups(IAuthenticationContext authenticationContext, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(authenticationContext); + + List permedInstanceIds = null; + await databaseContextFactory.UseContext( + async databaseContext => + permedInstanceIds = await databaseContext + .InstancePermissionSets + .AsQueryable() + .Where(ips => ips.PermissionSetId == authenticationContext.PermissionSet.Id.Value) + .Select(ips => ips.Id) + .ToListAsync(cancellationToken)); + + return permedInstanceIds.Select(JobsHub.HubGroupName); + } + + /// + /// Refresh the for clients associated with a given . + /// + /// The of the who's users need updating. + /// The for the operation. + /// A representing the running operation. + ValueTask RefreshHubGroups(long permissionSetId, CancellationToken cancellationToken) + => databaseContextFactory.UseContext( + async databaseContext => + { + logger.LogTrace("RefreshHubGroups"); + var permissionSetUsers = await databaseContext + .Users + .Where(x => x.PermissionSet.Id == permissionSetId) + .ToListAsync(cancellationToken); + var allInstanceIds = await databaseContext + .Instances + .Select( + instance => instance.Id.Value) + .ToListAsync(cancellationToken); + var permissionSetAccessibleInstanceIds = await databaseContext + .InstancePermissionSets + .AsQueryable() + .Where(ips => ips.PermissionSetId == permissionSetId) + .Select(ips => ips.InstanceId) + .ToListAsync(cancellationToken); + + var groupsToRemove = allInstanceIds + .Except(permissionSetAccessibleInstanceIds) + .Select(JobsHub.HubGroupName); + + var groupsToAdd = permissionSetAccessibleInstanceIds + .Select(JobsHub.HubGroupName); + + var connectionIds = permissionSetUsers + .SelectMany(user => hub.UserConnectionIds(user)) + .ToList(); + + logger.LogTrace( + "Updating groups for the {connectionCount} hub connections of permission set {permissionSetId}. They may access {allowed}/{total} instances.", + connectionIds.Count, + permissionSetId, + permissionSetAccessibleInstanceIds.Count, + allInstanceIds.Count); + + var removeTasks = connectionIds + .SelectMany(connectionId => groupsToRemove + .Select(groupName => hub + .Groups + .RemoveFromGroupAsync(connectionId, groupName, cancellationToken))); + + var addTasks = connectionIds + .SelectMany(connectionId => groupsToAdd + .Select(groupName => hub + .Groups + .AddToGroupAsync(connectionId, groupName, cancellationToken))); + + // Checked internally, the default implementations for these tasks complete synchronously + // https://github.com/dotnet/aspnetcore/blob/ce330d9d12f7676ff35c2223bd8a3b1e252a4e86/src/SignalR/server/Core/src/DefaultHubLifetimeManager.cs#L34-L70 + await Task.WhenAll(removeTasks.Concat(addTasks)); + }); + } +} diff --git a/src/Tgstation.Server.Host/Security/AuthorizationContextHubFilter.cs b/src/Tgstation.Server.Host/Security/AuthorizationContextHubFilter.cs new file mode 100644 index 00000000000..7820d66494c --- /dev/null +++ b/src/Tgstation.Server.Host/Security/AuthorizationContextHubFilter.cs @@ -0,0 +1,89 @@ +using System; +using System.Linq; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.SignalR; +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Api.Hubs; + +namespace Tgstation.Server.Host.Security +{ + /// + /// An that denies method calls and connections if the is not valid for an authorized user. + /// + sealed class AuthorizationContextHubFilter : IHubFilter + { + /// + /// The for the . + /// + readonly IAuthenticationContext authenticationContext; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + public AuthorizationContextHubFilter( + IAuthenticationContext authenticationContext, + ILogger logger) + { + this.authenticationContext = authenticationContext ?? throw new ArgumentNullException(nameof(authenticationContext)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public async Task OnConnectedAsync(HubLifetimeContext context, Func next) + { + ArgumentNullException.ThrowIfNull(context); + if (await ValidateAuthenticationContext(context.Hub)) + await next(context); + } + + /// + public async ValueTask InvokeMethodAsync(HubInvocationContext invocationContext, Func> next) + { + ArgumentNullException.ThrowIfNull(invocationContext); + if (await ValidateAuthenticationContext(invocationContext.Hub)) + return await next(invocationContext); + + return null; + } + + /// + /// Validates the for the hub event. + /// + /// The current . + /// if the hub call should continue, if it shouldn't and has been aborted. + async ValueTask ValidateAuthenticationContext(Hub hub) + { + if (!authenticationContext.Valid) + logger.LogTrace("The token for connection {connectionId} is no longer authenticated! Aborting...", hub.Context.ConnectionId); + else if (!authenticationContext.User.Enabled.Value) + logger.LogTrace("The token for connection {connectionId} is no longer authorized! Aborting...", hub.Context.ConnectionId); + else + return true; + + var hubType = hub.GetType(); + var allHubProperties = hubType.GetProperties(); + var typedClientsProperty = allHubProperties.Single( + prop => prop.PropertyType.IsConstructedGenericType + && prop.Name == nameof(hub.Clients)); + var clients = typedClientsProperty.GetValue(hub); + var callerProperty = clients.GetType().GetProperty(nameof(hub.Clients.Caller)); + var caller = callerProperty.GetValue(clients); + + if (caller is not IErrorHandlingHub specifiedHub) + throw new InvalidOperationException("This filter only supports IErrorHandlingHubs"); + + await specifiedHub.AbortingConnection(ConnectionAbortReason.TokenInvalid, hub.Context.ConnectionAborted); + hub.Context.Abort(); + return false; + } + } +} diff --git a/src/Tgstation.Server.Host/Security/IPermissionsUpdateNotifyee.cs b/src/Tgstation.Server.Host/Security/IPermissionsUpdateNotifyee.cs new file mode 100644 index 00000000000..7a95f7f0760 --- /dev/null +++ b/src/Tgstation.Server.Host/Security/IPermissionsUpdateNotifyee.cs @@ -0,0 +1,37 @@ +using System.Threading; +using System.Threading.Tasks; + +using Tgstation.Server.Host.Models; + +namespace Tgstation.Server.Host.Security +{ + /// + /// Receives notifications about permissions updates. + /// + public interface IPermissionsUpdateNotifyee + { + /// + /// Called when a given is successfully created. + /// + /// The . must be populated. + /// The for the operation. + /// A representing the running operation. + ValueTask InstancePermissionSetCreated(InstancePermissionSet instancePermissionSet, CancellationToken cancellationToken); + + /// + /// Called when an is successfully deleted. + /// + /// The of the deleted . + /// The for the operation. + /// A representing the running operation. + ValueTask InstancePermissionSetDeleted(PermissionSet permissionSet, CancellationToken cancellationToken); + + /// + /// Called when a given is successfully disabled. + /// + /// The that was disabled. + /// The for the operation. + /// A representing the running operation. + ValueTask UserDisabled(User user, CancellationToken cancellationToken); + } +} diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index e156db8d188..f1f8d02ca78 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -78,6 +78,8 @@ + + diff --git a/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs b/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs index df19b90759e..56aa2886195 100644 --- a/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs +++ b/src/Tgstation.Server.Host/Utils/ApiHeadersProvider.cs @@ -1,4 +1,4 @@ -using System; +using System; using Microsoft.AspNetCore.Http; @@ -12,7 +12,7 @@ sealed class ApiHeadersProvider : IApiHeadersProvider /// public ApiHeaders ApiHeaders => attemptedApiHeadersCreation ? apiHeaders - : CreateApiHeaders(true); + : CreateApiHeaders(false); /// public HeadersException HeadersException { get; private set; } @@ -42,33 +42,31 @@ public ApiHeadersProvider(IHttpContextAccessor httpContextAccessor) } /// - public ApiHeaders CreateAuthlessHeaders() => CreateApiHeaders(false); + public ApiHeaders CreateAuthlessHeaders() => CreateApiHeaders(true); /// /// Attempt to parse from the , optionally populating the properties. /// - /// If the error should be ignored and / should be populated. - /// A newly parsed or if was set and the parse failed. - ApiHeaders CreateApiHeaders(bool includeAuthAndSetProperties) + /// If the error should be ignored and / should not be populated. + /// A newly parsed or if was set and the parse failed. + ApiHeaders CreateApiHeaders(bool authless) { if (httpContextAccessor.HttpContext == null) throw new InvalidOperationException("httpContextAccessor has no HttpContext!"); - var request = httpContextAccessor.HttpContext.Request; - var ignoreMissingAuth = !includeAuthAndSetProperties; - - if (includeAuthAndSetProperties) + var typedHeaders = httpContextAccessor.HttpContext.Request.GetTypedHeaders(); + if (!authless) attemptedApiHeadersCreation = true; try { - var headers = new ApiHeaders(request.GetTypedHeaders(), ignoreMissingAuth); - if (includeAuthAndSetProperties) + var headers = new ApiHeaders(typedHeaders, authless, !authless); + if (!authless) apiHeaders = headers; return headers; } - catch (HeadersException ex) when (includeAuthAndSetProperties) + catch (HeadersException ex) when (!authless) { HeadersException = ex; return null; diff --git a/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs b/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs new file mode 100644 index 00000000000..411c424f088 --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs @@ -0,0 +1,165 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.SignalR; +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Api.Hubs; +using Tgstation.Server.Common.Extensions; +using Tgstation.Server.Host.Core; +using Tgstation.Server.Host.Models; +using Tgstation.Server.Host.Security; + +namespace Tgstation.Server.Host.Utils.SignalR +{ + /// + /// An implementation of with connection ID mapping. + /// + /// The the is for. + /// The interface for implementing methods. + sealed class ComprehensiveHubContext : IConnectionMappedHubContext, IHubConnectionMapper, IRestartHandler + where THub : ConnectionMappingHub + where THubMethods : class, IErrorHandlingHub + { + /// + public IHubClients Clients => wrappedHubContext.Clients; + + /// + public IGroupManager Groups => wrappedHubContext.Groups; + + /// + /// The being wrapped. + /// + readonly IHubContext wrappedHubContext; + + /// + /// The for the . + /// + readonly ILogger> logger; + + /// + /// Map of s to their associated s. + /// + readonly ConcurrentDictionary> userConnections; + + /// + public event Func>> OnConnectionMapGroups; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The to with. + /// The value of . + public ComprehensiveHubContext( + IHubContext wrappedHubContext, + IServerControl serverControl, + ILogger> logger) + { + this.wrappedHubContext = wrappedHubContext ?? throw new ArgumentNullException(nameof(wrappedHubContext)); + ArgumentNullException.ThrowIfNull(serverControl); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + userConnections = new ConcurrentDictionary>(); + + serverControl.RegisterForRestart(this); + } + + /// + public List UserConnectionIds(User user) + { + ArgumentNullException.ThrowIfNull(user); + var connectionIds = userConnections.GetOrAdd(user.Id.Value, _ => new Dictionary()); + lock (connectionIds) + return connectionIds.Keys.ToList(); + } + + /// + public async ValueTask UserConnected(IAuthenticationContext authenticationContext, THub hub, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(authenticationContext); + ArgumentNullException.ThrowIfNull(hub); + + var userId = authenticationContext.User.Id.Value; + var context = hub.Context; + logger.LogTrace( + "Mapping user {userId} to hub connection ID: {connectionId}", + userId, + context.ConnectionId); + + var mappedGroupsTask = OnConnectionMapGroups(authenticationContext, cancellationToken); + userConnections.AddOrUpdate( + userId, + _ => new Dictionary + { + { context.ConnectionId, context }, + }, + (_, old) => + { + lock (old) + old[context.ConnectionId] = context; + + return old; + }); + + var mappedGroups = await mappedGroupsTask; + await Task.WhenAll( + mappedGroups.Select( + group => hub.Groups.AddToGroupAsync(context.ConnectionId, group, cancellationToken))); + } + + /// + public void UserDisconnected(string connectionId) + { + ArgumentNullException.ThrowIfNull(connectionId); + foreach (var kvp in userConnections) + lock (kvp.Value) + if (kvp.Value.Remove(connectionId)) + logger.LogTrace("User {userId} disconnected connection ID: {connectionId}", kvp.Key, connectionId); + } + + /// + public ValueTask NotifyAndAbortUnauthedConnections(User user, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(user); + logger.LogTrace("NotifyAndAbortUnauthedConnections. UID {userId}", user.Id.Value); + + List connections = null; + userConnections.AddOrUpdate( + user.Id.Value, + _ => new Dictionary(), + (_, old) => + { + lock (old) + { + connections = old.Values.ToList(); + old.Clear(); + } + + return old; + }); + + async ValueTask NotifyAndAbortConnection(HubCallerContext context) + { + await Clients + .Client(context.ConnectionId) + .AbortingConnection(ConnectionAbortReason.TokenInvalid, cancellationToken); + context.Abort(); + } + + return ValueTaskExtensions.WhenAll(connections.Select(NotifyAndAbortConnection)); + } + + /// + public async ValueTask HandleRestart(Version updateVersion, bool handlerMayDelayShutdownWithExtremelyLongRunningTasks, CancellationToken cancellationToken) + { + logger.LogTrace("HandleRestart. {connectionCount} active connections", userConnections.Count); + await Clients.All.AbortingConnection(ConnectionAbortReason.ServerRestart, cancellationToken); + userConnections.Clear(); + } + } +} diff --git a/src/Tgstation.Server.Host/Utils/SignalR/ConnectionMappingHub.cs b/src/Tgstation.Server.Host/Utils/SignalR/ConnectionMappingHub.cs new file mode 100644 index 00000000000..5f81010e8b2 --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/SignalR/ConnectionMappingHub.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.SignalR; + +using Tgstation.Server.Api.Hubs; +using Tgstation.Server.Host.Security; + +namespace Tgstation.Server.Host.Utils.SignalR +{ + /// + /// Base for s that want to map their connection IDs to s. + /// + /// The child inheriting from the . + /// The interface for implementing methods. + [TgsAuthorize] + abstract class ConnectionMappingHub : Hub + where TChildHub : ConnectionMappingHub + where THubMethods : class, IErrorHandlingHub + { + /// + /// The used to map connections. + /// + readonly IHubConnectionMapper connectionMapper; + + /// + /// The for the . + /// + readonly IAuthenticationContext authenticationContext; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + protected ConnectionMappingHub( + IHubConnectionMapper connectionMapper, + IAuthenticationContext authenticationContext) + { + this.connectionMapper = connectionMapper ?? throw new ArgumentNullException(nameof(connectionMapper)); + this.authenticationContext = authenticationContext ?? throw new ArgumentNullException(nameof(authenticationContext)); + } + + /// + public override async Task OnConnectedAsync() + { + await connectionMapper.UserConnected(authenticationContext, (TChildHub)this, Context.ConnectionAborted); + await base.OnConnectedAsync(); + } + + /// + [AllowAnonymous] + public override Task OnDisconnectedAsync(Exception exception) + { + connectionMapper.UserDisconnected(Context.ConnectionId); + return base.OnDisconnectedAsync(exception); + } + } +} diff --git a/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs b/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs new file mode 100644 index 00000000000..d44ad96108d --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.SignalR; + +using Tgstation.Server.Api.Hubs; +using Tgstation.Server.Host.Models; +using Tgstation.Server.Host.Security; + +namespace Tgstation.Server.Host.Utils.SignalR +{ + /// + /// A that maps s to their connection IDs. + /// + /// The the is for. + /// The interface for implementing methods. + interface IConnectionMappedHubContext : IHubContext + where THub : Hub + where THubMethods : class, IErrorHandlingHub + { + /// + /// Called when a user connects. Should return an of hub group names the given belongs in. + /// + event Func>> OnConnectionMapGroups; + + /// + /// Gets a of current connection IDs for a given . + /// + /// The to get connection IDs for. + /// A representing the active connection IDs of the . + List UserConnectionIds(User user); + + /// + /// Calls with on and aborts the connections associated with the given . + /// + /// The to abort the connections of. + /// The for the operation. + /// A representing the running operation. + ValueTask NotifyAndAbortUnauthedConnections(User user, CancellationToken cancellationToken); + } +} diff --git a/src/Tgstation.Server.Host/Utils/SignalR/IHubConnectionMapper.cs b/src/Tgstation.Server.Host/Utils/SignalR/IHubConnectionMapper.cs new file mode 100644 index 00000000000..f941ac28fe0 --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/SignalR/IHubConnectionMapper.cs @@ -0,0 +1,36 @@ +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.SignalR; + +using Tgstation.Server.Api.Hubs; +using Tgstation.Server.Host.Models; +using Tgstation.Server.Host.Security; + +namespace Tgstation.Server.Host.Utils.SignalR +{ + /// + /// Handles mapping connection IDs to s for a given . + /// + /// The whose connections are being mapped. + /// The interface for implementing methods. + interface IHubConnectionMapper + where THub : ConnectionMappingHub + where THubMethods : class, IErrorHandlingHub + { + /// + /// To be called when a hub connection is made. + /// + /// The associated with the connection. + /// The . + /// The for the operation. + /// A representing the running operation. + ValueTask UserConnected(IAuthenticationContext authenticationContext, THub hub, CancellationToken cancellationToken); + + /// + /// To be called when a hub connection is terminated. + /// + /// The connection ID. + void UserDisconnected(string connectionId); + } +} diff --git a/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs b/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs index 27329a1af4b..8d6808f1b2e 100644 --- a/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs +++ b/tests/Tgstation.Server.Api.Tests/TestApiHeaders.cs @@ -43,7 +43,7 @@ static ApiHeaders TestHeader(string userAgent) { "User-Agent", userAgent } }; - return new ApiHeaders(new RequestHeaders(headers), false); + return new ApiHeaders(new RequestHeaders(headers), false, false); }; var header = TestHeader(BrowserHeader); diff --git a/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj b/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj index b4041bd259f..98434096f70 100644 --- a/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj +++ b/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj @@ -5,6 +5,10 @@ $(TgsFrameworkVersion) + + + + diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs new file mode 100644 index 00000000000..f78352772d0 --- /dev/null +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -0,0 +1,293 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.AspNetCore.SignalR.Client; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using Tgstation.Server.Api.Hubs; +using Tgstation.Server.Api.Models.Request; +using Tgstation.Server.Api.Models.Response; +using Tgstation.Server.Client; +using Tgstation.Server.Common.Extensions; + +namespace Tgstation.Server.Tests.Live.Instance +{ + sealed class JobsHubTests : IJobsHub + { + const int ActiveConnections = 2; + + readonly IServerClient permedUser; + readonly IServerClient permlessUser; + + readonly TaskCompletionSource finishTcs; + + readonly ConcurrentDictionary seenJobs; + + readonly HashSet permlessSeenJobs; + + HubConnection conn1, conn2; + int expectedReboots; + bool permlessIsPermed; + + long? permlessPsId; + + public JobsHubTests(IServerClient permedUser, IServerClient permlessUser) + { + this.permedUser = permedUser; + this.permlessUser = permlessUser; + + Assert.AreNotSame(permedUser, permlessUser); + + finishTcs = new TaskCompletionSource(); + + seenJobs = new ConcurrentDictionary(); + permlessSeenJobs = new HashSet(); + } + + public Task ReceiveJobUpdate(JobResponse job, CancellationToken cancellationToken) + { + try + { + Assert.IsTrue(job.InstanceId.HasValue); + Assert.IsNotNull(job.StartedBy); + Assert.IsTrue(job.StartedBy.Id.HasValue); + Assert.IsTrue(job.StartedAt.HasValue); + Assert.IsNotNull(job.Description); + + seenJobs.AddOrUpdate(job.Id.Value, job, (_, old) => + { + Assert.IsFalse(old.StoppedAt.HasValue, $"Received update for job {job.Id} after it had completed!"); + + return job; + }); + } + catch(Exception ex) + { + finishTcs.SetException(ex); + } + + return Task.CompletedTask; + } + + + class ShouldNeverReceiveUpdates : IJobsHub + { + public Action Callback { get; set; } + public Func Error { get; set; } + + public Task AbortingConnection(ConnectionAbortReason reason, CancellationToken cancellationToken) + => Error(reason, cancellationToken); + + public Task ReceiveJobUpdate(JobResponse job, CancellationToken cancellationToken) + { + Callback(job); + return Task.CompletedTask; + } + } + + public async Task Run(CancellationToken cancellationToken) + { + var neverReceiverTcs = new TaskCompletionSource(); + var neverReceiver = new ShouldNeverReceiveUpdates() + { + Callback = job => + { + if (!permlessIsPermed) + neverReceiverTcs.TrySetException(new Exception($"ShouldNeverReceiveUpdates received an update for job {job.Id}!")); + else + lock (permlessSeenJobs) + permlessSeenJobs.Add(job.Id.Value); + }, + Error = AbortingConnection, + }; + + await using (conn1 = (HubConnection)await permedUser.SubscribeToJobUpdates( + this, + null, + null, + cancellationToken)) + await using (conn2 = (HubConnection)await permlessUser.SubscribeToJobUpdates( + neverReceiver, + null, + null, + cancellationToken)) + { + Console.WriteLine($"Initial conn1: {conn1.ConnectionId}"); + Console.WriteLine($"Initial conn2: {conn2.ConnectionId}"); + + conn1.Reconnected += (newId) => + { + Console.WriteLine($"conn1 reconnected: {newId}"); + return Task.CompletedTask; + }; + conn2.Reconnected += (newId) => + { + Console.WriteLine($"conn1 reconnected: {newId}"); + return Task.CompletedTask; + }; + + var completedTask = await Task.WhenAny(finishTcs.Task, neverReceiverTcs.Task); + await completedTask; + } + + neverReceiverTcs.TrySetResult(); + await neverReceiverTcs.Task; + + var allInstances = await permedUser.Instances.List(null, cancellationToken); + + async ValueTask> CheckInstance(InstanceResponse instance) + { + var wasOffline = !instance.Online.Value; + if (wasOffline) + await permedUser.Instances.Update(new InstanceUpdateRequest + { + Id = instance.Id, + Online = true, + }, cancellationToken); + + var jobs = await permedUser.Instances.CreateClient(instance).Jobs.List(null, cancellationToken); + if (wasOffline) + await permedUser.Instances.Update(new InstanceUpdateRequest + { + Id = instance.Id, + Online = false, + }, cancellationToken); + + return jobs; + } + + var allJobsTask = allInstances + .Select(CheckInstance); + + var allJobs = (await ValueTaskExtensions.WhenAll(allJobsTask, allInstances.Count)).SelectMany(x => x).ToList(); + var missableMissedJobs = 0; + foreach (var job in allJobs) + { + var seenThisJob = seenJobs.TryGetValue(job.Id.Value, out var hubJob); + if (seenThisJob) + { + Assert.AreEqual(job.StoppedAt, hubJob.StoppedAt); + Assert.AreEqual(job.InstanceId, hubJob.InstanceId); + Assert.AreEqual(job.ExceptionDetails, hubJob.ExceptionDetails); + Assert.AreEqual(job.Stage, hubJob.Stage); + Assert.AreEqual(job.CancelledBy?.Id, hubJob.CancelledBy?.Id); + Assert.AreEqual(job.Cancelled, hubJob.Cancelled); + Assert.AreEqual(job.StartedBy?.Id, hubJob.StartedBy?.Id); + Assert.AreEqual(job.CancelRight, hubJob.CancelRight); + Assert.AreEqual(job.CancelRightsType, hubJob.CancelRightsType); + Assert.AreEqual(job.Progress, hubJob.Progress); + Assert.AreEqual(job.Description, hubJob.Description); + Assert.AreEqual(job.ErrorCode, hubJob.ErrorCode); + Assert.AreEqual(job.StartedAt, hubJob.StartedAt); + } + else + { + var wasMissableJob = job.Description.StartsWith("Reconnect chat bot") + || job.Description.StartsWith("Instance startup watchdog reattach") + || job.Description.StartsWith("Instance startup watchdog launch"); + Assert.IsTrue(wasMissableJob); + ++missableMissedJobs; + } + } + + // some instances may be detached, but our cache remains + var accountedJobs = allJobs.Count - missableMissedJobs; + var accountedSeenJobs = seenJobs.Where(x => allInstances.Any(i => i.Id.Value == x.Value.InstanceId)).Count(); + Assert.AreEqual(accountedJobs, accountedSeenJobs); + Assert.IsTrue(accountedJobs <= seenJobs.Count); + Assert.AreNotEqual(0, permlessSeenJobs.Count); + Assert.IsTrue(permlessSeenJobs.Count < seenJobs.Count); + Assert.IsTrue(permlessSeenJobs.All(id => seenJobs.ContainsKey(id))); + + await using var conn3 = (HubConnection)await permedUser.SubscribeToJobUpdates( + this, + null, + null, + cancellationToken); + + Assert.AreEqual(HubConnectionState.Connected, conn3.State); + await permlessUser.DisposeAsync(); + await permedUser.DisposeAsync(); + Assert.AreEqual(0, expectedReboots); + } + + public void ExpectShutdown() + { + Assert.AreEqual(0, Interlocked.Exchange(ref expectedReboots, ActiveConnections)); + Assert.AreEqual(HubConnectionState.Connected, conn1.State); + Assert.AreEqual(HubConnectionState.Connected, conn2.State); + } + + public async ValueTask WaitForReconnect(CancellationToken cancellationToken) + { + Assert.AreEqual(0, expectedReboots); + await Task.WhenAll(conn1.StopAsync(cancellationToken), conn2.StopAsync(cancellationToken)); + + Assert.AreEqual(HubConnectionState.Disconnected, conn1.State); + Assert.AreEqual(HubConnectionState.Disconnected, conn2.State); + + // force token refreshs + await Task.WhenAll(permedUser.Administration.Read(cancellationToken).AsTask(), permlessUser.Instances.List(null, cancellationToken).AsTask()); + + await Task.WhenAll(conn1.StartAsync(cancellationToken), conn2.StartAsync(cancellationToken)); + + Assert.AreEqual(HubConnectionState.Connected, conn1.State); + Assert.AreEqual(HubConnectionState.Connected, conn2.State); + Console.WriteLine($"New conn1: {conn1.ConnectionId}"); + Console.WriteLine($"New conn2: {conn2.ConnectionId}"); + + if (!permlessPsId.HasValue) + { + var permlessUserId = long.Parse(permlessUser.Token.ParseJwt().Subject); + permlessPsId = (await permedUser.Users.GetId(new Api.Models.EntityId + { + Id = permlessUserId + }, cancellationToken)).PermissionSet.Id; + } + + var instancesTask = permedUser.Instances.List(null, cancellationToken); + + permlessIsPermed = !permlessIsPermed; + + var instances = await instancesTask; + await ValueTaskExtensions.WhenAll( + instances + .Where(instance => instance.Online.Value) + .Select(async instance => + { + var ic = permedUser.Instances.CreateClient(instance); + if (permlessIsPermed) + await ic.PermissionSets.Create(new InstancePermissionSetRequest + { + PermissionSetId = permlessPsId.Value, + }, cancellationToken); + else + await ic.PermissionSets.Delete(new InstancePermissionSetRequest + { + PermissionSetId = permlessPsId.Value + }, cancellationToken); + })); + } + + public void CompleteNow() => finishTcs.TrySetResult(); + + public Task AbortingConnection(ConnectionAbortReason reason, CancellationToken cancellationToken) + { + try + { + Assert.AreEqual(ConnectionAbortReason.ServerRestart, reason); + var remaining = Interlocked.Decrement(ref expectedReboots); + Assert.IsTrue(remaining >= 0); + } + catch (Exception ex) + { + finishTcs.TrySetException(ex); + } + return Task.CompletedTask; + } + } +} diff --git a/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClient.cs b/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClient.cs index 515180c0494..a487a919917 100644 --- a/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClient.cs +++ b/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClient.cs @@ -13,8 +13,18 @@ namespace Tgstation.Server.Tests.Live { sealed class RateLimitRetryingApiClient : ApiClient { - public RateLimitRetryingApiClient(IHttpClient httpClient, Uri url, ApiHeaders apiHeaders, ApiHeaders tokenRefreshHeaders, bool authless) - : base(httpClient, url, apiHeaders, tokenRefreshHeaders, authless) + public RateLimitRetryingApiClient( + IHttpClient httpClient, + Uri url, + ApiHeaders apiHeaders, + ApiHeaders tokenRefreshHeaders, + bool authless) + : base( + httpClient, + url, + apiHeaders, + tokenRefreshHeaders, + authless) { } diff --git a/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClientFactory.cs b/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClientFactory.cs index 7edca577975..f9ba45e66e1 100644 --- a/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClientFactory.cs +++ b/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClientFactory.cs @@ -1,6 +1,7 @@ using System; using Tgstation.Server.Api; +using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Client; using Tgstation.Server.Common.Http; @@ -8,7 +9,11 @@ namespace Tgstation.Server.Tests.Live { sealed class RateLimitRetryingApiClientFactory : IApiClientFactory { - public IApiClient CreateApiClient(Uri url, ApiHeaders apiHeaders, ApiHeaders tokenRefreshHeaders, bool authless) + public IApiClient CreateApiClient( + Uri url, + ApiHeaders apiHeaders, + ApiHeaders tokenRefreshHeaders, + bool authless) => new RateLimitRetryingApiClient( new HttpClient(), url, diff --git a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs index 7dc5d243f76..bb028397b48 100644 --- a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs +++ b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs @@ -5,15 +5,25 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Net.Mime; +using System.Reflection; using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.Http.Connections; +using Microsoft.AspNetCore.SignalR.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; + using Tgstation.Server.Api; +using Tgstation.Server.Api.Hubs; using Tgstation.Server.Api.Models; +using Tgstation.Server.Api.Models.Request; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Client; +using Tgstation.Server.Client.Extensions; using Tgstation.Server.Common.Extensions; using Tgstation.Server.Host; @@ -346,12 +356,132 @@ static async Task RegressionTestForLeakedPasswordHashesBug(IServerClient serverC } } + class FuncProxiedJobsHub : IJobsHub + { + public Func ProxyFunc { get; set; } + public Func ErrorFunc { get; set; } + + public Task AbortingConnection(ConnectionAbortReason reason, CancellationToken cancellationToken) + => ErrorFunc(reason); + + public Task ReceiveJobUpdate(JobResponse job, CancellationToken cancellationToken) + => ProxyFunc(job, cancellationToken); + } + + static async Task TestSignalRUsage(IServerClientFactory serverClientFactory, IServerClient serverClient, CancellationToken cancellationToken) + { + // test regular creation works without error + var hubConnectionBuilder = new HubConnectionBuilder(); + + var tokenRetrivalFunc = () => Task.FromResult("FakeToken"); + + hubConnectionBuilder.WithUrl( + new Uri(serverClient.Url, Routes.JobsHub), + HttpTransportType.ServerSentEvents, + options => + { + options.AccessTokenProvider = () => tokenRetrivalFunc(); + ((IApiClient)typeof(ServerClient) + .GetField( + "apiClient", + BindingFlags.NonPublic | BindingFlags.Instance) + .GetValue(serverClient)) + .Headers + .SetHubConnectionHeaders(options.Headers); + }); + + hubConnectionBuilder.ConfigureLogging( + loggingBuilder => + { + loggingBuilder.SetMinimumLevel(LogLevel.Trace); + loggingBuilder.AddConsole(); + loggingBuilder + .Services + .TryAddEnumerable( + ServiceDescriptor.Singleton()); + }); + + var proxy = new FuncProxiedJobsHub(); + var errorTcs = new TaskCompletionSource(); + proxy.ErrorFunc = reason => + { + errorTcs.SetException(new Exception($"Aborted: {reason}")); + return Task.CompletedTask; + }; + + HubConnection hubConnection; + HardFailLoggerProvider.BlockFails = true; + try + { + await using (hubConnection = hubConnectionBuilder.Build()) + { + Assert.AreEqual(HubConnectionState.Disconnected, hubConnection.State); + hubConnection.ProxyOn(proxy); + + var exception = await Assert.ThrowsExceptionAsync(() => hubConnection.StartAsync(cancellationToken)); + + Assert.AreEqual(HttpStatusCode.Unauthorized, exception.StatusCode); + Assert.AreEqual(HubConnectionState.Disconnected, hubConnection.State); + + tokenRetrivalFunc = () => Task.FromResult(serverClient.Token.Bearer); + await hubConnection.StartAsync(cancellationToken); + + Assert.AreEqual(HubConnectionState.Connected, hubConnection.State); + } + + Assert.AreEqual(HubConnectionState.Disconnected, hubConnection.State); + + Assert.IsFalse(errorTcs.Task.IsCompleted); + + var createRequest = new UserCreateRequest + { + Enabled = true, + Name = "SignalRTestUser", + Password = "asdfasdfasdfasdfasdf" + }; + + var testUser = await serverClient.Users.Create(createRequest, cancellationToken); + await using (var testUserClient = await serverClientFactory.CreateFromLogin(serverClient.Url, createRequest.Name, createRequest.Password, cancellationToken: cancellationToken)) + { + errorTcs = new TaskCompletionSource(); + await using var testUserConn1 = await testUserClient.SubscribeToJobUpdates(proxy, cancellationToken: cancellationToken); + + Assert.IsFalse(errorTcs.Task.IsCompleted); + + await serverClient.Users.Update(new UserUpdateRequest + { + Id = testUser.Id, + Enabled = false, + }, cancellationToken); + + // need a second here + for (var i = 0; i < 10 && !errorTcs.Task.IsCompleted; ++i) + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + + Assert.IsTrue(errorTcs.Task.IsCompleted); + + errorTcs = new TaskCompletionSource(); + await using var testUserConn2 = await testUserClient.SubscribeToJobUpdates(proxy, cancellationToken: cancellationToken); + for (var i = 0; i < 10 && !errorTcs.Task.IsCompleted; ++i) + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + } + + Assert.IsTrue(errorTcs.Task.IsCompleted); + await Assert.ThrowsExceptionAsync(() => errorTcs.Task); + } + finally + { + HardFailLoggerProvider.BlockFails = false; + } + } + public static Task Run(IServerClientFactory clientFactory, IServerClient serverClient, CancellationToken cancellationToken) => Task.WhenAll( TestRequestValidation(serverClient, cancellationToken), TestOAuthFails(serverClient, cancellationToken), TestServerInformation(clientFactory, serverClient, cancellationToken), TestInvalidTransfers(serverClient, cancellationToken), - RegressionTestForLeakedPasswordHashesBug(serverClient, cancellationToken)); + RegressionTestForLeakedPasswordHashesBug(serverClient, cancellationToken), + TestSignalRUsage(clientFactory, serverClient, cancellationToken)); } } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index b7a430b8105..1f51de5f599 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -255,7 +255,7 @@ async ValueTask TestWithoutAndWithPermission(Func( () => adminClient.Administration.Update( new ServerUpdateRequest @@ -442,7 +442,7 @@ public async Task TestOneServerSwarmUpdate() try { - using var controllerClient = await CreateAdminClient(controller.Url, cancellationToken); + await using var controllerClient = await CreateAdminClient(controller.Url, cancellationToken); var controllerInfo = await controllerClient.ServerInformation(cancellationToken); @@ -544,9 +544,9 @@ public async Task TestSwarmSynchronizationAndUpdates() try { - using var controllerClient = await CreateAdminClient(controller.Url, cancellationToken); - using var node1Client = await CreateAdminClient(node1.Url, cancellationToken); - using var node2Client = await CreateAdminClient(node2.Url, cancellationToken); + await using var controllerClient = await CreateAdminClient(controller.Url, cancellationToken); + await using var node1Client = await CreateAdminClient(node1.Url, cancellationToken); + await using var node2Client = await CreateAdminClient(node2.Url, cancellationToken); var controllerInfo = await controllerClient.ServerInformation(cancellationToken); @@ -610,12 +610,12 @@ await Task.WhenAny( Assert.AreEqual(newUser.Name, node1User.Name); Assert.AreEqual(newUser.Enabled, node1User.Enabled); - using var controllerUserClient = await clientFactory.CreateFromLogin( + await using var controllerUserClient = await clientFactory.CreateFromLogin( controllerAddress, newUser.Name, "asdfasdfasdfasdf"); - using var node1BadClient = clientFactory.CreateFromToken(node1.Url, controllerUserClient.Token); + await using var node1BadClient = clientFactory.CreateFromToken(node1.Url, controllerUserClient.Token); await ApiAssert.ThrowsException(() => node1BadClient.Administration.Read(cancellationToken)); // check instance info is not shared @@ -685,8 +685,8 @@ void CheckServerUpdated(LiveTestingServer server) controller.Run(cancellationToken).AsTask(), node1.Run(cancellationToken).AsTask()); - using var controllerClient2 = await CreateAdminClient(controller.Url, cancellationToken); - using var node1Client2 = await CreateAdminClient(node1.Url, cancellationToken); + await using var controllerClient2 = await CreateAdminClient(controller.Url, cancellationToken); + await using var node1Client2 = await CreateAdminClient(node1.Url, cancellationToken); await ApiAssert.ThrowsException(() => controllerClient2.Administration.Update( new ServerUpdateRequest @@ -701,7 +701,7 @@ await ApiAssert.ThrowsException(() = serverTask, node2.Run(cancellationToken).AsTask()); - using var node2Client2 = await CreateAdminClient(node2.Url, cancellationToken); + await using var node2Client2 = await CreateAdminClient(node2.Url, cancellationToken); async Task WaitForSwarmServerUpdate2() { @@ -815,9 +815,9 @@ public async Task TestSwarmReconnection() try { - using var controllerClient = await CreateAdminClient(controller.Url, cancellationToken); - using var node1Client = await CreateAdminClient(node1.Url, cancellationToken); - using var node2Client = await CreateAdminClient(node2.Url, cancellationToken); + await using var controllerClient = await CreateAdminClient(controller.Url, cancellationToken); + await using var node1Client = await CreateAdminClient(node1.Url, cancellationToken); + await using var node2Client = await CreateAdminClient(node2.Url, cancellationToken); var controllerInfo = await controllerClient.ServerInformation(cancellationToken); @@ -897,7 +897,7 @@ await Task.WhenAny( Assert.IsTrue(controllerTask.IsCompleted); controllerTask = controller.Run(cancellationToken).AsTask(); - using var controllerClient2 = await CreateAdminClient(controller.Url, cancellationToken); + await using var controllerClient2 = await CreateAdminClient(controller.Url, cancellationToken); // node 2 should reconnect once it's health check triggers await Task.WhenAny( @@ -934,7 +934,7 @@ await ApiAssert.ThrowsException( ErrorCode.SwarmIntegrityCheckFailed); node2Task = node2.Run(cancellationToken).AsTask(); - using var node2Client2 = await CreateAdminClient(node2.Url, cancellationToken); + await using var node2Client2 = await CreateAdminClient(node2.Url, cancellationToken); // should re-register await Task.WhenAny( @@ -991,7 +991,7 @@ async ValueTask TestTgstation(bool interactive) var serverTask = server.Run(cancellationToken); try { - using var adminClient = await CreateAdminClient(server.Url, cancellationToken); + await using var adminClient = await CreateAdminClient(server.Url, cancellationToken); var instanceManagerTest = new InstanceManagerTest(adminClient, server.Directory); var instance = await instanceManagerTest.CreateTestInstance("TgTestInstance", cancellationToken); @@ -1273,7 +1273,29 @@ async Task TestTgsInternal(CancellationToken hardCancellationToken) { Api.Models.Instance instance; long initialStaged, initialActive; - using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) + await using var firstAdminClient = await CreateAdminClient(server.Url, cancellationToken); + + async ValueTask CreateUserWithNoInstancePerms() + { + var createRequest = new UserCreateRequest() + { + Name = "SomePermlessChum", + Password = "alidfjuwh84322r4yrkajhfdqh38hrfiouw4", + Enabled = true, + PermissionSet = new PermissionSet + { + InstanceManagerRights = InstanceManagerRights.Read, + } + }; + + var user = await firstAdminClient.Users.Create(createRequest, cancellationToken); + Assert.IsTrue(user.Enabled); + + return await clientFactory.CreateFromLogin(server.Url, createRequest.Name, createRequest.Password, cancellationToken: cancellationToken); + } + + var jobsHubTest = new JobsHubTests(firstAdminClient, await CreateUserWithNoInstancePerms()); + Task jobsHubTestTask; { if (server.DumpOpenApiSpecpath) { @@ -1302,21 +1324,23 @@ async Task FailFast(Task task) } } - var rootTest = FailFast(RawRequestTests.Run(clientFactory, adminClient, cancellationToken)); - var adminTest = FailFast(new AdministrationTest(adminClient.Administration).Run(cancellationToken)); - var usersTest = FailFast(new UsersTest(adminClient).Run(cancellationToken)); - var instanceManagerTest = new InstanceManagerTest(adminClient, server.Directory); + var rootTest = FailFast(RawRequestTests.Run(clientFactory, firstAdminClient, cancellationToken)); + var adminTest = FailFast(new AdministrationTest(firstAdminClient.Administration).Run(cancellationToken)); + var usersTest = FailFast(new UsersTest(firstAdminClient).Run(cancellationToken)); + + jobsHubTestTask = FailFast(jobsHubTest.Run(cancellationToken)); + var instanceManagerTest = new InstanceManagerTest(firstAdminClient, server.Directory); var compatInstanceTask = instanceManagerTest.CreateTestInstance("CompatTestsInstance", cancellationToken); instance = await instanceManagerTest.CreateTestInstance("LiveTestsInstance", cancellationToken); var compatInstance = await compatInstanceTask; var instancesTest = FailFast(instanceManagerTest.RunPreTest(cancellationToken)); Assert.IsTrue(Directory.Exists(instance.Path)); - var instanceClient = adminClient.Instances.CreateClient(instance); + var instanceClient = firstAdminClient.Instances.CreateClient(instance); Assert.IsTrue(Directory.Exists(instanceClient.Metadata.Path)); var instanceTest = new InstanceTest( - adminClient.Instances, + firstAdminClient.Instances, fileDownloader, GetInstanceManager(), (ushort)server.Url.Port); @@ -1329,7 +1353,7 @@ async Task RunInstanceTests() new PlatformIdentifier().IsWindows ? new Version(510, 1346) : new Version(512, 1451), // http://www.byond.com/forum/?forum=5&command=search&scope=local&text=resolved%3a512.1451 - adminClient.Instances.CreateClient(compatInstance), + firstAdminClient.Instances.CreateClient(compatInstance), compatDMPort, compatDDPort, server.HighPriorityDreamDaemon, @@ -1364,7 +1388,8 @@ await FailFast( initialActive = dd.ActiveCompileJob.Id.Value; initialStaged = dd.StagedCompileJob.Id.Value; - await adminClient.Administration.Restart(cancellationToken); + jobsHubTest.ExpectShutdown(); + await firstAdminClient.Administration.Restart(cancellationToken); } await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); @@ -1412,8 +1437,9 @@ await FailFast( // chat bot start and DD reattach test serverTask = server.Run(cancellationToken).AsTask(); - using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) + await using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) { + await jobsHubTest.WaitForReconnect(cancellationToken); var instanceClient = adminClient.Instances.CreateClient(instance); var jobs = await instanceClient.Jobs.ListActive(null, cancellationToken); @@ -1478,6 +1504,7 @@ await FailFast( Assert.AreEqual(WatchdogStatus.Offline, dd.Status); + jobsHubTest.ExpectShutdown(); await adminClient.Administration.Restart(cancellationToken); } @@ -1496,7 +1523,6 @@ async Task WaitForInitialJobs(IInstanceClient instanceClient) .Select(e => instanceClient.Jobs.GetId(e, cancellationToken)) .ToList(); - jobs = (await ValueTaskExtensions.WhenAll(getTasks)) .Where(x => x.StartedAt.Value > preStartupTime) .ToList(); @@ -1515,10 +1541,11 @@ async Task WaitForInitialJobs(IInstanceClient instanceClient) serverTask = server.Run(cancellationToken).AsTask(); long expectedCompileJobId, expectedStaged; var edgeByond = await ByondTest.GetEdgeVersion(fileDownloader, cancellationToken); - using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) + await using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) { var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); + await jobsHubTest.WaitForReconnect(cancellationToken); var dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1554,6 +1581,7 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); expectedStaged = compileJob.Id.Value; + jobsHubTest.ExpectShutdown(); await adminClient.Administration.Restart(cancellationToken); } @@ -1562,10 +1590,11 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest // post/entity deletion tests serverTask = server.Run(cancellationToken).AsTask(); - using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) + await using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) { var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); + await jobsHubTest.WaitForReconnect(cancellationToken); var currentDD = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(expectedCompileJobId, currentDD.ActiveCompileJob.Id.Value); @@ -1581,6 +1610,9 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance).RunPostTest(cancellationToken); await repoTest; + jobsHubTest.CompleteNow(); + await jobsHubTestTask; + await new InstanceManagerTest(adminClient, server.Directory).RunPostTest(instance, cancellationToken); } } diff --git a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj index 82c98aa9f01..2ed1cd139b1 100644 --- a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj +++ b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj @@ -5,6 +5,10 @@ $(TgsFrameworkVersion) + + + + diff --git a/tools/Tgstation.Server.Migrator/Tgstation.Server.Migrator.csproj b/tools/Tgstation.Server.Migrator/Tgstation.Server.Migrator.csproj index a74f718d7c1..4d4dc7f61dc 100644 --- a/tools/Tgstation.Server.Migrator/Tgstation.Server.Migrator.csproj +++ b/tools/Tgstation.Server.Migrator/Tgstation.Server.Migrator.csproj @@ -1,4 +1,4 @@ - + @@ -13,6 +13,7 @@ + From 5b1e123e64f73e666ec3f2327816af222a541d5a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 19:22:57 -0400 Subject: [PATCH 097/138] Add `DeploymentActivation` event --- .../Components/Events/EventType.cs | 6 ++++++ .../Components/Watchdog/AdvancedWatchdog.cs | 11 +++-------- .../Components/Watchdog/BasicWatchdog.cs | 5 ++++- .../Components/Watchdog/WatchdogBase.cs | 19 +++++++++++++++++++ .../Components/Watchdog/WatchdogFactory.cs | 1 + 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index 0a54518a248..2b9575c8561 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -161,5 +161,11 @@ public enum EventType /// [EventScript("DeploymentCleanup")] DeploymentCleanup, + + /// + /// Whenever a deployment is about to be used by the game server. May fire multiple times per deployment. + /// + [EventScript("DeploymentActivation")] + DeploymentActivation, } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 35d3e508e7e..7878854458e 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -29,11 +29,6 @@ abstract class AdvancedWatchdog : BasicWatchdog /// protected SwappableDmbProvider ActiveSwappable { get; private set; } - /// - /// The for the pointing to the Game directory. - /// - protected IIOManager GameIOManager { get; } - /// /// The for the . /// @@ -64,10 +59,10 @@ abstract class AdvancedWatchdog : BasicWatchdog /// The for the . /// The for the . /// The for the . - /// The for the . + /// The 'Diagnostics' for the . /// The for the . /// The for the . - /// The value of . + /// The 'Game' for the . /// The value of . /// The for the . /// The for the . @@ -101,6 +96,7 @@ public AdvancedWatchdog( diagnosticsIOManager, eventConsumer, remoteDeploymentManagerFactory, + gameIOManager, logger, initialLaunchParameters, instance, @@ -108,7 +104,6 @@ public AdvancedWatchdog( { try { - GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager)); LinkFactory = linkFactory ?? throw new ArgumentNullException(nameof(linkFactory)); deploymentCleanupTasks = new List(); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index 815f87fa9ba..d915a8b9356 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -50,9 +50,10 @@ class BasicWatchdog : WatchdogBase /// The for the . /// The for the . /// The for the . - /// The for the . + /// The 'Diagnostics' for the . /// The for the . /// The for the . + /// The 'Game' for the . /// The for the . /// The for the . /// The for the . @@ -68,6 +69,7 @@ public BasicWatchdog( IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IIOManager gameIOManager, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, @@ -83,6 +85,7 @@ public BasicWatchdog( diagnosticsIOManager, eventConsumer, remoteDeploymentManagerFactory, + gameIOManager, logger, initialLaunchParameters, instance, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 3847e26077c..5b379d0043f 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -96,6 +96,11 @@ public WatchdogStatus Status /// protected IAsyncDelayer AsyncDelayer { get; } + /// + /// The for the pointing to the Game directory. + /// + protected IIOManager GameIOManager { get; } + /// /// The for the . /// @@ -184,6 +189,7 @@ public WatchdogStatus Status /// The value of . /// The value of . /// The value of . + /// The value of . /// The value of . /// The initial value of . May be modified. /// The value of . @@ -199,6 +205,7 @@ protected WatchdogBase( IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IIOManager gameIOManager, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance metadata, @@ -213,6 +220,7 @@ protected WatchdogBase( this.diagnosticsIOManager = diagnosticsIOManager ?? throw new ArgumentNullException(nameof(diagnosticsIOManager)); this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory)); + GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager)); Logger = logger ?? throw new ArgumentNullException(nameof(logger)); ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters)); this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); @@ -679,6 +687,15 @@ protected async ValueTask BeforeApplyDmb(Models.CompileJob newCompileJob, Cancel metadata, newCompileJob); + var eventTask = eventConsumer.HandleEvent( + EventType.DeploymentActivation, + new List + { + GameIOManager.ResolvePath(newCompileJob.DirectoryName.ToString()), + }, + false, + cancellationToken); + try { await remoteDeploymentManager.ApplyDeployment(newCompileJob, cancellationToken); @@ -687,6 +704,8 @@ protected async ValueTask BeforeApplyDmb(Models.CompileJob newCompileJob, Cancel { Logger.LogWarning(ex, "Failed to apply remote deployment!"); } + + await eventTask; } /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs index 2fb71827557..e9a37ecb2b5 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs @@ -90,6 +90,7 @@ public virtual IWatchdog CreateWatchdog( diagnosticsIOManager, eventConsumer, remoteDeploymentManagerfactory, + gameIOManager, LoggerFactory.CreateLogger(), settings, instance, From 6fb9ba6f372f09604a512c30e9bafaec055e1cbc Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 19:43:33 -0400 Subject: [PATCH 098/138] Check registered jobs have instance and user IDs --- src/Tgstation.Server.Host/Jobs/JobService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index 92d50965429..2212bba37dc 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -120,7 +120,7 @@ public ValueTask RegisterOperation(Job job, JobEntrypoint operation, Cancellatio job.Instance = new Models.Instance { - Id = job.Instance.Id, + Id = job.Instance.Id ?? throw new InvalidOperationException("Instance associated with job does not have an Id!"), }; databaseContext.Instances.Attach(job.Instance); @@ -131,7 +131,7 @@ public ValueTask RegisterOperation(Job job, JobEntrypoint operation, Cancellatio else job.StartedBy = new User { - Id = job.StartedBy.Id, + Id = job.StartedBy.Id ?? throw new InvalidOperationException("StartedBy User associated with job does not have an Id!"), }; databaseContext.Users.Attach(job.StartedBy); From 385771f477eaf1933f349947b06eec40ad5289cb Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 20:38:47 -0400 Subject: [PATCH 099/138] Change this `.OfType()` to `.Cast()` --- .../Components/StaticFiles/Configuration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 380e0e02ea2..0a9a356e771 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -73,7 +73,7 @@ sealed class Configuration : IConfiguration /// static readonly IReadOnlyDictionary EventTypeScriptFileNameMap = new Dictionary( Enum.GetValues(typeof(EventType)) - .OfType() + .Cast() .Select( eventType => new KeyValuePair( eventType, From 189d594b4cb7729a7aab297b68a0cc922e6eb27f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 20:47:14 -0400 Subject: [PATCH 100/138] Update nuget packages --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index f1f8d02ca78..4d35ef12803 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -73,7 +73,7 @@ - + @@ -97,7 +97,7 @@ - + From 85e1c0d5f1e69abfbecfece78125b402e314d5e2 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 20:47:41 -0400 Subject: [PATCH 101/138] Fix dotnet-ef version being out of sync --- src/Tgstation.Server.Host/.config/dotnet-tools.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/.config/dotnet-tools.json b/src/Tgstation.Server.Host/.config/dotnet-tools.json index c418dc80b94..37aa7721e35 100644 --- a/src/Tgstation.Server.Host/.config/dotnet-tools.json +++ b/src/Tgstation.Server.Host/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-ef": { - "version": "7.0.11", + "version": "7.0.13", "commands": [ "dotnet-ef" ] From ebd09d7e86882b181710375748262331b62d4010 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 21:03:55 -0400 Subject: [PATCH 102/138] Add `JobCode`s to strongly type jobs - Default `0` for legacy jobs. - Standardize job creation prior to registration. - Add `RightsHelper.TypeToRight()`. --- .../Models/Internal/Job.cs | 7 + src/Tgstation.Server.Api/Models/JobCode.cs | 112 ++ .../Rights/RightsHelper.cs | 10 + .../Components/Chat/Providers/Provider.cs | 9 +- .../Components/Deployment/DreamMaker.cs | 11 +- .../Components/Instance.cs | 21 +- .../Components/InstanceManager.cs | 5 +- .../Components/Watchdog/WatchdogBase.cs | 17 +- .../Controllers/ByondController.cs | 26 +- .../Controllers/DreamDaemonController.cs | 27 +- .../Controllers/DreamMakerController.cs | 9 +- .../Controllers/InstanceController.cs | 16 +- .../Controllers/RepositoryController.cs | 30 +- .../Database/DatabaseContext.cs | 17 +- .../20231105004801_MSAddJobCodes.Designer.cs | 1075 ++++++++++++++++ .../20231105004801_MSAddJobCodes.cs | 33 + .../20231105004808_MYAddJobCodes.Designer.cs | 1109 +++++++++++++++++ .../20231105004808_MYAddJobCodes.cs | 33 + .../20231105004814_PGAddJobCodes.Designer.cs | 1069 ++++++++++++++++ .../20231105004814_PGAddJobCodes.cs | 33 + .../20231105004820_SLAddJobCodes.Designer.cs | 1041 ++++++++++++++++ .../20231105004820_SLAddJobCodes.cs | 33 + .../MySqlDatabaseContextModelSnapshot.cs | 6 +- ...PostgresSqlDatabaseContextModelSnapshot.cs | 6 +- .../SqlServerDatabaseContextModelSnapshot.cs | 6 +- .../SqliteDatabaseContextModelSnapshot.cs | 6 +- src/Tgstation.Server.Host/Jobs/JobService.cs | 18 +- src/Tgstation.Server.Host/Models/Job.cs | 85 +- .../Models/TestJobCode.cs | 27 + .../Live/Instance/JobsRequiredTest.cs | 10 + 30 files changed, 4752 insertions(+), 155 deletions(-) create mode 100644 src/Tgstation.Server.Api/Models/JobCode.cs create mode 100644 src/Tgstation.Server.Host/Database/Migrations/20231105004801_MSAddJobCodes.Designer.cs create mode 100644 src/Tgstation.Server.Host/Database/Migrations/20231105004801_MSAddJobCodes.cs create mode 100644 src/Tgstation.Server.Host/Database/Migrations/20231105004808_MYAddJobCodes.Designer.cs create mode 100644 src/Tgstation.Server.Host/Database/Migrations/20231105004808_MYAddJobCodes.cs create mode 100644 src/Tgstation.Server.Host/Database/Migrations/20231105004814_PGAddJobCodes.Designer.cs create mode 100644 src/Tgstation.Server.Host/Database/Migrations/20231105004814_PGAddJobCodes.cs create mode 100644 src/Tgstation.Server.Host/Database/Migrations/20231105004820_SLAddJobCodes.Designer.cs create mode 100644 src/Tgstation.Server.Host/Database/Migrations/20231105004820_SLAddJobCodes.cs create mode 100644 tests/Tgstation.Server.Api.Tests/Models/TestJobCode.cs diff --git a/src/Tgstation.Server.Api/Models/Internal/Job.cs b/src/Tgstation.Server.Api/Models/Internal/Job.cs index 54661a2eacf..8b3b210c602 100644 --- a/src/Tgstation.Server.Api/Models/Internal/Job.cs +++ b/src/Tgstation.Server.Api/Models/Internal/Job.cs @@ -10,9 +10,16 @@ namespace Tgstation.Server.Api.Models.Internal /// public class Job : EntityId { + /// + /// The . + /// + [Required] + public JobCode? JobCode { get; set; } + /// /// English description of the . /// + /// May not match the listed on the . [Required] public string? Description { get; set; } diff --git a/src/Tgstation.Server.Api/Models/JobCode.cs b/src/Tgstation.Server.Api/Models/JobCode.cs new file mode 100644 index 00000000000..072c71eda85 --- /dev/null +++ b/src/Tgstation.Server.Api/Models/JobCode.cs @@ -0,0 +1,112 @@ +using System.ComponentModel; + +namespace Tgstation.Server.Api.Models +{ + /// + /// The different types of . + /// + public enum JobCode : byte + { + /// + /// This catch-all code is applied to jobs that were created on a tgstation-server before v5.17.0. + /// + [Description("Legacy job")] + Unknown, + + /// + /// When the instance is being moved. + /// + [Description("Instance move")] + Move, + + /// + /// When the repository is cloning. + /// + [Description("Clone repository")] + RepositoryClone, + + /// + /// When the repository is being manually updated. + /// + [Description("Update repository")] + RepositoryUpdate, + + /// + /// When the repository is being automatically updated. + /// + [Description("Scheduled repository update")] + RepositoryAutoUpdate, + + /// + /// When the repository is being deleted. + /// + [Description("Delete repository")] + RepositoryDelete, + + /// + /// When a new BYOND version is being installed. + /// + [Description("Install BYOND version")] + ByondOfficialInstall, + + /// + /// When a new BYOND version is being installed. + /// + [Description("Install custom BYOND version")] + ByondCustomInstall, + + /// + /// When an installed BYOND version is being deleted. + /// + [Description("Delete installed BYOND version")] + ByondDelete, + + /// + /// When a deployment is manually triggered. + /// + [Description("Compile active repository code")] + Deployment, + + /// + /// When a deployment is automatically triggered. + /// + [Description("Scheduled code deployment")] + AutomaticDeployment, + + /// + /// When the watchdog is started manually. + /// + [Description("Launch DreamDaemon")] + WatchdogLaunch, + + /// + /// When the watchdog is restarted manually. + /// + [Description("Restart Watchdog")] + WatchdogRestart, + + /// + /// When a the watchdog is dumping the game server process. + /// + [Description("Create DreamDaemon Process Dump")] + WatchdogDump, + + /// + /// When the watchdog starts due to an instance being onlined. + /// + [Description("Instance startup watchdog launch")] + StartupWatchdogLaunch, + + /// + /// When the watchdog reattaches due to an instance being onlined. + /// + [Description("Instance startup watchdog reattach")] + StartupWatchdogReattach, + + /// + /// When a chat bot connects/reconnects. + /// + [Description("Reconnect chat bot")] + ReconnectChatBot, + } +} diff --git a/src/Tgstation.Server.Api/Rights/RightsHelper.cs b/src/Tgstation.Server.Api/Rights/RightsHelper.cs index eb33954a48a..73e948bdbb0 100644 --- a/src/Tgstation.Server.Api/Rights/RightsHelper.cs +++ b/src/Tgstation.Server.Api/Rights/RightsHelper.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.Linq; namespace Tgstation.Server.Api.Rights { @@ -32,6 +33,15 @@ public static class RightsHelper /// The of the given . public static Type RightToType(RightsType rightsType) => TypeMap[rightsType]; + /// + /// Map a given to its respective . + /// + /// The of the right. + /// The of . + public static RightsType TypeToRight() + where TRight : Enum + => TypeMap.First(kvp => kvp.Value == typeof(TRight)).Key; + /// /// Gets the role claim name used for a given . /// diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs index 88cd5b2e2c6..a6496c1cdad 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs @@ -273,13 +273,8 @@ async Task ReconnectionLoop(uint reconnectInterval, bool connectNow, Cancellatio connectNow = false; if (!Connected) { - var job = new Job - { - Description = $"Reconnect chat bot: {ChatBot.Name}", - CancelRight = (ulong)ChatBotRights.WriteEnabled, - CancelRightsType = RightsType.ChatBots, - Instance = ChatBot.Instance, - }; + var job = Job.Create(Api.Models.JobCode.ReconnectChatBot, null, ChatBot.Instance, ChatBotRights.WriteEnabled); + job.Description += $": {ChatBot.Name}"; await jobManager.RegisterOperation( job, diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index c561e259396..8792e2f304c 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -337,10 +337,7 @@ await databaseContextFactory.UseContext( async databaseContext => { var fullJob = compileJob.Job; - compileJob.Job = new Models.Job - { - Id = job.Id, - }; + compileJob.Job = new Models.Job(job.Id.Value); var fullRevInfo = compileJob.RevisionInformation; compileJob.RevisionInformation = new Models.RevisionInformation { @@ -431,10 +428,10 @@ await databaseContextFactory.UseContext( .Where(x => x.Job.Instance.Id == metadata.Id) .OrderByDescending(x => x.Job.StoppedAt) .Take(10) - .Select(x => new Models.Job + .Select(x => new { - StoppedAt = x.Job.StoppedAt, - StartedAt = x.Job.StartedAt, + x.Job.StoppedAt, + x.Job.StartedAt, }) .ToListAsync(cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 9843e5c0b87..de7e70d7966 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -501,17 +501,7 @@ async Task TimerLoop(uint minutes, CancellationToken cancellationToken) await eventConsumer.HandleEvent(EventType.InstanceAutoUpdateStart, Enumerable.Empty(), true, cancellationToken); try { - var repositoryUpdateJob = new Job - { - Instance = new Models.Instance - { - Id = metadata.Id, - }, - Description = "Scheduled repository update", - CancelRightsType = RightsType.Repository, - CancelRight = (ulong)RepositoryRights.CancelPendingChanges, - }; - + var repositoryUpdateJob = Job.Create(Api.Models.JobCode.RepositoryAutoUpdate, null, metadata, RepositoryRights.CancelPendingChanges); await jobManager.RegisterOperation( repositoryUpdateJob, RepositoryAutoUpdateJob, @@ -541,14 +531,7 @@ await jobManager.RegisterOperation( } // finally set up the job - compileProcessJob = new Job - { - Instance = repositoryUpdateJob.Instance, - Description = "Scheduled code deployment", - CancelRightsType = RightsType.DreamMaker, - CancelRight = (ulong)DreamMakerRights.CancelCompile, - }; - + compileProcessJob = Job.Create(Api.Models.JobCode.AutomaticDeployment, null, metadata, DreamMakerRights.CancelCompile); await jobManager.RegisterOperation( compileProcessJob, (core, databaseContextFactory, job, progressReporter, jobCancellationToken) => diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index ec66af63b1d..86dbad53320 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -351,10 +351,7 @@ await databaseContextFactory.UseContext( .Jobs .AsQueryable() .Where(x => x.Instance.Id == metadata.Id && !x.StoppedAt.HasValue) - .Select(x => new Models.Job - { - Id = x.Id, - }) + .Select(x => new Models.Job(x.Id.Value)) .ToListAsync(cancellationToken); foreach (var job in jobs) tasks.Add(jobService.CancelJob(job, user, true, cancellationToken)); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 5b379d0043f..025f3f27a86 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -377,16 +377,13 @@ public async Task StartAsync(CancellationToken cancellationToken) if (!autoStart && !reattaching) return; - var job = new Models.Job - { - Instance = new Models.Instance - { - Id = metadata.Id, - }, - Description = $"Instance startup watchdog {(reattaching ? "reattach" : "launch")}", - CancelRight = (ulong)DreamDaemonRights.Shutdown, - CancelRightsType = RightsType.DreamDaemon, - }; + var job = Models.Job.Create( + reattaching + ? JobCode.StartupWatchdogReattach + : JobCode.StartupWatchdogLaunch, + null, + metadata, + DreamDaemonRights.Shutdown); await jobManager.RegisterOperation( job, async (core, databaseContextFactory, paramJob, progressFunction, ct) => diff --git a/src/Tgstation.Server.Host/Controllers/ByondController.cs b/src/Tgstation.Server.Host/Controllers/ByondController.cs index 41cccc1584d..10a02eee868 100644 --- a/src/Tgstation.Server.Host/Controllers/ByondController.cs +++ b/src/Tgstation.Server.Host/Controllers/ByondController.cs @@ -195,14 +195,14 @@ public async ValueTask Update([FromBody] ByondVersionRequest mode Instance.Id); // run the install through the job manager - var job = new Job - { - Description = $"Install {(!uploadingZip ? String.Empty : "custom ")}BYOND version {version}", - StartedBy = AuthenticationContext.User, - CancelRightsType = RightsType.Byond, - CancelRight = (ulong)ByondRights.CancelInstall, - Instance = Instance, - }; + var job = Job.Create( + uploadingZip + ? JobCode.ByondCustomInstall + : JobCode.ByondOfficialInstall, + AuthenticationContext.User, + Instance, + ByondRights.CancelInstall); + job.Description += $" {version}"; IFileUploadTicket fileUploadTicket = null; if (uploadingZip) @@ -304,14 +304,8 @@ public async ValueTask Delete([FromBody] ByondVersionDeleteReques var isCustomVersion = version.Build != -1; // run the install through the job manager - var job = new Job - { - Description = $"Delete installed BYOND version {version}", - StartedBy = AuthenticationContext.User, - CancelRightsType = RightsType.Byond, - CancelRight = (ulong)(isCustomVersion ? ByondRights.InstallOfficialOrChangeActiveVersion : ByondRights.InstallCustomVersion), - Instance = Instance, - }; + var job = Job.Create(JobCode.ByondDelete, AuthenticationContext.User, Instance, isCustomVersion ? ByondRights.InstallOfficialOrChangeActiveVersion : ByondRights.InstallCustomVersion); + job.Description += $" {version}"; await jobManager.RegisterOperation( job, diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index 5bceb479d5c..097e0377d37 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -87,14 +87,7 @@ public ValueTask Create(CancellationToken cancellationToken) if (instance.Watchdog.Status != WatchdogStatus.Offline) return Conflict(new ErrorMessageResponse(ErrorCode.WatchdogRunning)); - var job = new Job - { - Description = "Launch DreamDaemon", - CancelRight = (ulong)DreamDaemonRights.Shutdown, - CancelRightsType = RightsType.DreamDaemon, - Instance = Instance, - StartedBy = AuthenticationContext.User, - }; + var job = Job.Create(JobCode.WatchdogLaunch, AuthenticationContext.User, Instance, DreamDaemonRights.Shutdown); await jobManager.RegisterOperation( job, (core, databaseContextFactory, paramJob, progressHandler, innerCt) => core.Watchdog.Launch(innerCt), @@ -270,14 +263,7 @@ bool CheckModified(Expression Restart(CancellationToken cancellationToken) => WithComponentInstance(async instance => { - var job = new Job - { - Instance = Instance, - CancelRightsType = RightsType.DreamDaemon, - CancelRight = (ulong)DreamDaemonRights.Shutdown, - StartedBy = AuthenticationContext.User, - Description = "Restart Watchdog", - }; + var job = Job.Create(JobCode.WatchdogRestart, AuthenticationContext.User, Instance, DreamDaemonRights.Shutdown); var watchdog = instance.Watchdog; @@ -303,14 +289,7 @@ await jobManager.RegisterOperation( public ValueTask CreateDump(CancellationToken cancellationToken) => WithComponentInstance(async instance => { - var job = new Job - { - Instance = Instance, - CancelRightsType = RightsType.DreamDaemon, - CancelRight = (ulong)DreamDaemonRights.CreateDump, - StartedBy = AuthenticationContext.User, - Description = "Create DreamDaemon Process Dump", - }; + var job = Job.Create(JobCode.WatchdogDump, AuthenticationContext.User, Instance, DreamDaemonRights.CreateDump); var watchdog = instance.Watchdog; diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 105a64d8b97..714713763f9 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -143,14 +143,7 @@ public ValueTask List([FromQuery] int? page, [FromQuery] int? pag [ProducesResponseType(typeof(JobResponse), 202)] public async ValueTask Create(CancellationToken cancellationToken) { - var job = new Job - { - Description = "Compile active repository code", - StartedBy = AuthenticationContext.User, - CancelRightsType = RightsType.DreamMaker, - CancelRight = (ulong)DreamMakerRights.CancelCompile, - Instance = Instance, - }; + var job = Job.Create(JobCode.Deployment, AuthenticationContext.User, Instance, DreamMakerRights.CancelCompile); await jobManager.RegisterOperation( job, diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index 7310f56f1c6..342d9193b6c 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -355,10 +355,7 @@ public async ValueTask Update([FromBody] InstanceUpdateRequest mo var moveJob = await InstanceQuery() .SelectMany(x => x.Jobs). Where(x => !x.StoppedAt.HasValue && x.Description.StartsWith(MoveInstanceJobPrefix)) - .Select(x => new Job - { - Id = x.Id, - }).FirstOrDefaultAsync(cancellationToken); + .Select(x => new Job(x.Id.Value)).FirstOrDefaultAsync(cancellationToken); if (moveJob != default) { @@ -490,14 +487,9 @@ await WithComponentInstance( var moving = originalModelPath != null; if (moving) { - var job = new Job - { - Description = $"{MoveInstanceJobPrefix}{originalModel.Id} from {originalModelPath} to {rawPath}", - Instance = originalModel, - CancelRightsType = RightsType.InstanceManager, - CancelRight = (ulong)InstanceManagerRights.Relocate, - StartedBy = AuthenticationContext.User, - }; + var description = $"{MoveInstanceJobPrefix}{originalModel.Id} from {originalModelPath} to {rawPath}"; + var job = Job.Create(JobCode.Move, AuthenticationContext.User, originalModel, InstanceManagerRights.Relocate); + job.Description = description; await jobManager.RegisterOperation( job, diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index d701142bfd6..37766ce0fd1 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -139,20 +139,15 @@ public async ValueTask Create([FromBody] RepositoryCreateRequest if (repo != null) return Conflict(new ErrorMessageResponse(ErrorCode.RepoExists)); - var job = new Job - { - Description = String.Format( + var description = String.Format( CultureInfo.InvariantCulture, "Clone{1} repository {0}", origin, cloneBranch != null ? $"\"{cloneBranch}\" branch of" - : String.Empty), - StartedBy = AuthenticationContext.User, - CancelRightsType = RightsType.Repository, - CancelRight = (ulong)RepositoryRights.CancelClone, - Instance = Instance, - }; + : String.Empty); + var job = Job.Create(JobCode.RepositoryClone, AuthenticationContext.User, Instance, RepositoryRights.CancelClone); + job.Description = description; var api = currentModel.ToApi(); await DatabaseContext.Save(cancellationToken); @@ -222,12 +217,7 @@ public async ValueTask Delete(CancellationToken cancellationToken Logger.LogInformation("Instance {instanceId} repository delete initiated by user {userId}", Instance.Id, AuthenticationContext.User.Id.Value); - var job = new Job - { - Description = "Delete repository", - StartedBy = AuthenticationContext.User, - Instance = Instance, - }; + var job = Job.Create(JobCode.RepositoryDelete, AuthenticationContext.User, Instance); var api = currentModel.ToApi(); await jobManager.RegisterOperation( job, @@ -454,14 +444,8 @@ bool CheckModified(Expression if (description == null) return Json(api); // no git changes - var job = new Job - { - Description = description, - StartedBy = AuthenticationContext.User, - Instance = Instance, - CancelRightsType = RightsType.Repository, - CancelRight = (ulong)RepositoryRights.CancelPendingChanges, - }; + var job = Job.Create(JobCode.RepositoryUpdate, AuthenticationContext.User, Instance, RepositoryRights.CancelPendingChanges); + job.Description = description; var repositoryUpdater = new RepositoryUpdateService( model, diff --git a/src/Tgstation.Server.Host/Database/DatabaseContext.cs b/src/Tgstation.Server.Host/Database/DatabaseContext.cs index fbd7d0a38c1..51e10a4e4b0 100644 --- a/src/Tgstation.Server.Host/Database/DatabaseContext.cs +++ b/src/Tgstation.Server.Host/Database/DatabaseContext.cs @@ -378,22 +378,22 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) /// /// Used by unit tests to remind us to setup the correct MSSQL migration downgrades. /// - internal static readonly Type MSLatestMigration = typeof(MSAddMapThreads); + internal static readonly Type MSLatestMigration = typeof(MSAddJobCodes); /// /// Used by unit tests to remind us to setup the correct MYSQL migration downgrades. /// - internal static readonly Type MYLatestMigration = typeof(MYAddMapThreads); + internal static readonly Type MYLatestMigration = typeof(MYAddJobCodes); /// /// Used by unit tests to remind us to setup the correct PostgresSQL migration downgrades. /// - internal static readonly Type PGLatestMigration = typeof(PGAddMapThreads); + internal static readonly Type PGLatestMigration = typeof(PGAddJobCodes); /// /// Used by unit tests to remind us to setup the correct SQLite migration downgrades. /// - internal static readonly Type SLLatestMigration = typeof(SLAddMapThreads); + internal static readonly Type SLLatestMigration = typeof(SLAddJobCodes); /// #pragma warning disable CA1502 // Cyclomatic complexity @@ -422,6 +422,15 @@ public async ValueTask SchemaDowngradeForServerVersion( string BadDatabaseType() => throw new ArgumentException($"Invalid DatabaseType: {currentDatabaseType}", nameof(currentDatabaseType)); + if (targetVersion < new Version(5, 17, 0)) + targetMigration = currentDatabaseType switch + { + DatabaseType.MySql => nameof(MYAddMapThreads), + DatabaseType.PostgresSql => nameof(PGAddMapThreads), + DatabaseType.SqlServer => nameof(MSAddMapThreads), + DatabaseType.Sqlite => nameof(SLAddMapThreads), + _ => BadDatabaseType(), + }; if (targetVersion < new Version(5, 13, 0)) targetMigration = currentDatabaseType switch { diff --git a/src/Tgstation.Server.Host/Database/Migrations/20231105004801_MSAddJobCodes.Designer.cs b/src/Tgstation.Server.Host/Database/Migrations/20231105004801_MSAddJobCodes.Designer.cs new file mode 100644 index 00000000000..556f6d1009d --- /dev/null +++ b/src/Tgstation.Server.Host/Database/Migrations/20231105004801_MSAddJobCodes.Designer.cs @@ -0,0 +1,1075 @@ +// +using System; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Tgstation.Server.Host.Database.Migrations +{ + [DbContext(typeof(SqlServerDatabaseContext))] + [Migration("20231105004801_MSAddJobCodes")] + partial class MSAddJobCodes + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "7.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ChannelLimit") + .HasColumnType("int"); + + b.Property("ConnectionString") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Provider") + .HasColumnType("int"); + + b.Property("ReconnectionInterval") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId", "Name") + .IsUnique(); + + b.ToTable("ChatBots"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatChannel", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ChatSettingsId") + .HasColumnType("bigint"); + + b.Property("DiscordChannelId") + .HasColumnType("decimal(20,0)"); + + b.Property("IrcChannel") + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("IsAdminChannel") + .IsRequired() + .HasColumnType("bit"); + + b.Property("IsSystemChannel") + .IsRequired() + .HasColumnType("bit"); + + b.Property("IsUpdatesChannel") + .IsRequired() + .HasColumnType("bit"); + + b.Property("IsWatchdogChannel") + .IsRequired() + .HasColumnType("bit"); + + b.Property("Tag") + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("ChatSettingsId", "DiscordChannelId") + .IsUnique() + .HasFilter("[DiscordChannelId] IS NOT NULL"); + + b.HasIndex("ChatSettingsId", "IrcChannel") + .IsUnique() + .HasFilter("[IrcChannel] IS NOT NULL"); + + b.ToTable("ChatChannels"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.CompileJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ByondVersion") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DMApiMajorVersion") + .HasColumnType("int"); + + b.Property("DMApiMinorVersion") + .HasColumnType("int"); + + b.Property("DMApiPatchVersion") + .HasColumnType("int"); + + b.Property("DirectoryName") + .IsRequired() + .HasColumnType("uniqueidentifier"); + + b.Property("DmeName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("GitHubDeploymentId") + .HasColumnType("int"); + + b.Property("GitHubRepoId") + .HasColumnType("bigint"); + + b.Property("JobId") + .HasColumnType("bigint"); + + b.Property("MinimumSecurityLevel") + .HasColumnType("int"); + + b.Property("Output") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("RepositoryOrigin") + .HasColumnType("nvarchar(max)"); + + b.Property("RevisionInformationId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("DirectoryName"); + + b.HasIndex("JobId") + .IsUnique(); + + b.HasIndex("RevisionInformationId"); + + b.ToTable("CompileJobs"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamDaemonSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AdditionalParameters") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.Property("AllowWebClient") + .IsRequired() + .HasColumnType("bit"); + + b.Property("AutoStart") + .IsRequired() + .HasColumnType("bit"); + + b.Property("DumpOnHealthCheckRestart") + .IsRequired() + .HasColumnType("bit"); + + b.Property("HealthCheckSeconds") + .HasColumnType("bigint"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("LogOutput") + .IsRequired() + .HasColumnType("bit"); + + b.Property("MapThreads") + .HasColumnType("bigint"); + + b.Property("Port") + .HasColumnType("int"); + + b.Property("SecurityLevel") + .HasColumnType("int"); + + b.Property("StartProfiler") + .IsRequired() + .HasColumnType("bit"); + + b.Property("StartupTimeout") + .HasColumnType("bigint"); + + b.Property("TopicRequestTimeout") + .HasColumnType("bigint"); + + b.Property("Visibility") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DreamDaemonSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamMakerSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ApiValidationPort") + .HasColumnType("int"); + + b.Property("ApiValidationSecurityLevel") + .HasColumnType("int"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("ProjectName") + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.Property("RequireDMApiValidation") + .IsRequired() + .HasColumnType("bit"); + + b.Property("Timeout") + .IsRequired() + .HasColumnType("time"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DreamMakerSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Instance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AutoUpdateInterval") + .HasColumnType("bigint"); + + b.Property("ChatBotLimit") + .HasColumnType("int"); + + b.Property("ConfigurationType") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Online") + .IsRequired() + .HasColumnType("bit"); + + b.Property("Path") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.Property("SwarmIdentifer") + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.HasIndex("Path", "SwarmIdentifer") + .IsUnique() + .HasFilter("[SwarmIdentifer] IS NOT NULL"); + + b.ToTable("Instances"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.InstancePermissionSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ByondRights") + .HasColumnType("decimal(20,0)"); + + b.Property("ChatBotRights") + .HasColumnType("decimal(20,0)"); + + b.Property("ConfigurationRights") + .HasColumnType("decimal(20,0)"); + + b.Property("DreamDaemonRights") + .HasColumnType("decimal(20,0)"); + + b.Property("DreamMakerRights") + .HasColumnType("decimal(20,0)"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("InstancePermissionSetRights") + .HasColumnType("decimal(20,0)"); + + b.Property("PermissionSetId") + .HasColumnType("bigint"); + + b.Property("RepositoryRights") + .HasColumnType("decimal(20,0)"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId"); + + b.HasIndex("PermissionSetId", "InstanceId") + .IsUnique(); + + b.ToTable("InstancePermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CancelRight") + .HasColumnType("decimal(20,0)"); + + b.Property("CancelRightsType") + .HasColumnType("decimal(20,0)"); + + b.Property("Cancelled") + .IsRequired() + .HasColumnType("bit"); + + b.Property("CancelledById") + .HasColumnType("bigint"); + + b.Property("Description") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ErrorCode") + .HasColumnType("bigint"); + + b.Property("ExceptionDetails") + .HasColumnType("nvarchar(max)"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("JobCode") + .HasColumnType("tinyint"); + + b.Property("StartedAt") + .IsRequired() + .HasColumnType("datetimeoffset"); + + b.Property("StartedById") + .HasColumnType("bigint"); + + b.Property("StoppedAt") + .HasColumnType("datetimeoffset"); + + b.HasKey("Id"); + + b.HasIndex("CancelledById"); + + b.HasIndex("InstanceId"); + + b.HasIndex("StartedById"); + + b.ToTable("Jobs"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.OAuthConnection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ExternalUserId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Provider") + .HasColumnType("int"); + + b.Property("UserId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.HasIndex("Provider", "ExternalUserId") + .IsUnique(); + + b.ToTable("OAuthConnections"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AdministrationRights") + .HasColumnType("decimal(20,0)"); + + b.Property("GroupId") + .HasColumnType("bigint"); + + b.Property("InstanceManagerRights") + .HasColumnType("decimal(20,0)"); + + b.Property("UserId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("GroupId") + .IsUnique() + .HasFilter("[GroupId] IS NOT NULL"); + + b.HasIndex("UserId") + .IsUnique() + .HasFilter("[UserId] IS NOT NULL"); + + b.ToTable("PermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ReattachInformation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AccessIdentifier") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("CompileJobId") + .HasColumnType("bigint"); + + b.Property("InitialCompileJobId") + .HasColumnType("bigint"); + + b.Property("LaunchSecurityLevel") + .HasColumnType("int"); + + b.Property("LaunchVisibility") + .HasColumnType("int"); + + b.Property("Port") + .HasColumnType("int"); + + b.Property("ProcessId") + .HasColumnType("int"); + + b.Property("RebootState") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CompileJobId"); + + b.HasIndex("InitialCompileJobId"); + + b.ToTable("ReattachInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AccessToken") + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.Property("AccessUser") + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.Property("AutoUpdatesKeepTestMerges") + .IsRequired() + .HasColumnType("bit"); + + b.Property("AutoUpdatesSynchronize") + .IsRequired() + .HasColumnType("bit"); + + b.Property("CommitterEmail") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.Property("CommitterName") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.Property("CreateGitHubDeployments") + .IsRequired() + .HasColumnType("bit"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("PostTestMergeComment") + .IsRequired() + .HasColumnType("bit"); + + b.Property("PushTestMergeCommits") + .IsRequired() + .HasColumnType("bit"); + + b.Property("ShowTestMergeCommitters") + .IsRequired() + .HasColumnType("bit"); + + b.Property("UpdateSubmodules") + .IsRequired() + .HasColumnType("bit"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("RepositorySettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevInfoTestMerge", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("RevisionInformationId") + .HasColumnType("bigint"); + + b.Property("TestMergeId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("RevisionInformationId"); + + b.HasIndex("TestMergeId"); + + b.ToTable("RevInfoTestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("nvarchar(40)"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("OriginCommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("nvarchar(40)"); + + b.Property("Timestamp") + .HasColumnType("datetimeoffset"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId", "CommitSha") + .IsUnique(); + + b.ToTable("RevisionInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Author") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("BodyAtMerge") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Comment") + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.Property("MergedAt") + .HasColumnType("datetimeoffset"); + + b.Property("MergedById") + .HasColumnType("bigint"); + + b.Property("Number") + .HasColumnType("int"); + + b.Property("PrimaryRevisionInformationId") + .IsRequired() + .HasColumnType("bigint"); + + b.Property("TargetCommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("nvarchar(40)"); + + b.Property("TitleAtMerge") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Url") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("MergedById"); + + b.HasIndex("PrimaryRevisionInformationId") + .IsUnique(); + + b.ToTable("TestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CanonicalName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("CreatedAt") + .IsRequired() + .HasColumnType("datetimeoffset"); + + b.Property("CreatedById") + .HasColumnType("bigint"); + + b.Property("Enabled") + .IsRequired() + .HasColumnType("bit"); + + b.Property("GroupId") + .HasColumnType("bigint"); + + b.Property("LastPasswordUpdate") + .HasColumnType("datetimeoffset"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("PasswordHash") + .HasColumnType("nvarchar(max)"); + + b.Property("SystemIdentifier") + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.HasKey("Id"); + + b.HasIndex("CanonicalName") + .IsUnique(); + + b.HasIndex("CreatedById"); + + b.HasIndex("GroupId"); + + b.HasIndex("SystemIdentifier") + .IsUnique() + .HasFilter("[SystemIdentifier] IS NOT NULL"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.UserGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Groups"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("ChatSettings") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatChannel", b => + { + b.HasOne("Tgstation.Server.Host.Models.ChatBot", "ChatSettings") + .WithMany("Channels") + .HasForeignKey("ChatSettingsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ChatSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.CompileJob", b => + { + b.HasOne("Tgstation.Server.Host.Models.Job", "Job") + .WithOne() + .HasForeignKey("Tgstation.Server.Host.Models.CompileJob", "JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "RevisionInformation") + .WithMany("CompileJobs") + .HasForeignKey("RevisionInformationId") + .OnDelete(DeleteBehavior.ClientNoAction) + .IsRequired(); + + b.Navigation("Job"); + + b.Navigation("RevisionInformation"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamDaemonSettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("DreamDaemonSettings") + .HasForeignKey("Tgstation.Server.Host.Models.DreamDaemonSettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamMakerSettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("DreamMakerSettings") + .HasForeignKey("Tgstation.Server.Host.Models.DreamMakerSettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.InstancePermissionSet", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("InstancePermissionSets") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.PermissionSet", "PermissionSet") + .WithMany("InstancePermissionSets") + .HasForeignKey("PermissionSetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + + b.Navigation("PermissionSet"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Job", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "CancelledBy") + .WithMany() + .HasForeignKey("CancelledById"); + + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("Jobs") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.User", "StartedBy") + .WithMany() + .HasForeignKey("StartedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CancelledBy"); + + b.Navigation("Instance"); + + b.Navigation("StartedBy"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.OAuthConnection", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "User") + .WithMany("OAuthConnections") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.HasOne("Tgstation.Server.Host.Models.UserGroup", "Group") + .WithOne("PermissionSet") + .HasForeignKey("Tgstation.Server.Host.Models.PermissionSet", "GroupId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("Tgstation.Server.Host.Models.User", "User") + .WithOne("PermissionSet") + .HasForeignKey("Tgstation.Server.Host.Models.PermissionSet", "UserId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("Group"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ReattachInformation", b => + { + b.HasOne("Tgstation.Server.Host.Models.CompileJob", "CompileJob") + .WithMany() + .HasForeignKey("CompileJobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob") + .WithMany() + .HasForeignKey("InitialCompileJobId"); + + b.Navigation("CompileJob"); + + b.Navigation("InitialCompileJob"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("RepositorySettings") + .HasForeignKey("Tgstation.Server.Host.Models.RepositorySettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevInfoTestMerge", b => + { + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "RevisionInformation") + .WithMany("ActiveTestMerges") + .HasForeignKey("RevisionInformationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.TestMerge", "TestMerge") + .WithMany("RevisonInformations") + .HasForeignKey("TestMergeId") + .OnDelete(DeleteBehavior.ClientNoAction) + .IsRequired(); + + b.Navigation("RevisionInformation"); + + b.Navigation("TestMerge"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("RevisionInformations") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "MergedBy") + .WithMany("TestMerges") + .HasForeignKey("MergedById") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "PrimaryRevisionInformation") + .WithOne("PrimaryTestMerge") + .HasForeignKey("Tgstation.Server.Host.Models.TestMerge", "PrimaryRevisionInformationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MergedBy"); + + b.Navigation("PrimaryRevisionInformation"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "CreatedBy") + .WithMany("CreatedUsers") + .HasForeignKey("CreatedById"); + + b.HasOne("Tgstation.Server.Host.Models.UserGroup", "Group") + .WithMany("Users") + .HasForeignKey("GroupId"); + + b.Navigation("CreatedBy"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.Navigation("Channels"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Instance", b => + { + b.Navigation("ChatSettings"); + + b.Navigation("DreamDaemonSettings"); + + b.Navigation("DreamMakerSettings"); + + b.Navigation("InstancePermissionSets"); + + b.Navigation("Jobs"); + + b.Navigation("RepositorySettings"); + + b.Navigation("RevisionInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.Navigation("InstancePermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.Navigation("ActiveTestMerges"); + + b.Navigation("CompileJobs"); + + b.Navigation("PrimaryTestMerge"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.Navigation("RevisonInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.Navigation("CreatedUsers"); + + b.Navigation("OAuthConnections"); + + b.Navigation("PermissionSet"); + + b.Navigation("TestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.UserGroup", b => + { + b.Navigation("PermissionSet") + .IsRequired(); + + b.Navigation("Users"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Tgstation.Server.Host/Database/Migrations/20231105004801_MSAddJobCodes.cs b/src/Tgstation.Server.Host/Database/Migrations/20231105004801_MSAddJobCodes.cs new file mode 100644 index 00000000000..87636a060f9 --- /dev/null +++ b/src/Tgstation.Server.Host/Database/Migrations/20231105004801_MSAddJobCodes.cs @@ -0,0 +1,33 @@ +using System; + +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Tgstation.Server.Host.Database.Migrations +{ + /// + public partial class MSAddJobCodes : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + ArgumentNullException.ThrowIfNull(migrationBuilder); + migrationBuilder.AddColumn( + name: "JobCode", + table: "Jobs", + type: "tinyint", + nullable: false, + defaultValue: (byte)0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + ArgumentNullException.ThrowIfNull(migrationBuilder); + migrationBuilder.DropColumn( + name: "JobCode", + table: "Jobs"); + } + } +} diff --git a/src/Tgstation.Server.Host/Database/Migrations/20231105004808_MYAddJobCodes.Designer.cs b/src/Tgstation.Server.Host/Database/Migrations/20231105004808_MYAddJobCodes.Designer.cs new file mode 100644 index 00000000000..c5f5fb94dcc --- /dev/null +++ b/src/Tgstation.Server.Host/Database/Migrations/20231105004808_MYAddJobCodes.Designer.cs @@ -0,0 +1,1109 @@ +// +using System; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Tgstation.Server.Host.Database.Migrations +{ + [DbContext(typeof(MySqlDatabaseContext))] + [Migration("20231105004808_MYAddJobCodes")] + partial class MYAddJobCodes + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "7.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 64); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("ChannelLimit") + .IsRequired() + .HasColumnType("smallint unsigned"); + + b.Property("ConnectionString") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("ConnectionString"), "utf8mb4"); + + b.Property("Enabled") + .HasColumnType("tinyint(1)"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + b.Property("Provider") + .HasColumnType("int"); + + b.Property("ReconnectionInterval") + .IsRequired() + .HasColumnType("int unsigned"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId", "Name") + .IsUnique(); + + b.ToTable("ChatBots"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatChannel", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("ChatSettingsId") + .HasColumnType("bigint"); + + b.Property("DiscordChannelId") + .HasColumnType("bigint unsigned"); + + b.Property("IrcChannel") + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("IrcChannel"), "utf8mb4"); + + b.Property("IsAdminChannel") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("IsSystemChannel") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("IsUpdatesChannel") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("IsWatchdogChannel") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("Tag") + .HasMaxLength(10000) + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("Tag"), "utf8mb4"); + + b.HasKey("Id"); + + b.HasIndex("ChatSettingsId", "DiscordChannelId") + .IsUnique(); + + b.HasIndex("ChatSettingsId", "IrcChannel") + .IsUnique(); + + b.ToTable("ChatChannels"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.CompileJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("ByondVersion") + .IsRequired() + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("ByondVersion"), "utf8mb4"); + + b.Property("DMApiMajorVersion") + .HasColumnType("int"); + + b.Property("DMApiMinorVersion") + .HasColumnType("int"); + + b.Property("DMApiPatchVersion") + .HasColumnType("int"); + + b.Property("DirectoryName") + .IsRequired() + .HasColumnType("char(36)"); + + b.Property("DmeName") + .IsRequired() + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("DmeName"), "utf8mb4"); + + b.Property("GitHubDeploymentId") + .HasColumnType("int"); + + b.Property("GitHubRepoId") + .HasColumnType("bigint"); + + b.Property("JobId") + .HasColumnType("bigint"); + + b.Property("MinimumSecurityLevel") + .HasColumnType("int"); + + b.Property("Output") + .IsRequired() + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("Output"), "utf8mb4"); + + b.Property("RepositoryOrigin") + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("RepositoryOrigin"), "utf8mb4"); + + b.Property("RevisionInformationId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("DirectoryName"); + + b.HasIndex("JobId") + .IsUnique(); + + b.HasIndex("RevisionInformationId"); + + b.ToTable("CompileJobs"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamDaemonSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("AdditionalParameters") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("AdditionalParameters"), "utf8mb4"); + + b.Property("AllowWebClient") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("AutoStart") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("DumpOnHealthCheckRestart") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("HealthCheckSeconds") + .IsRequired() + .HasColumnType("int unsigned"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("LogOutput") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("MapThreads") + .IsRequired() + .HasColumnType("int unsigned"); + + b.Property("Port") + .IsRequired() + .HasColumnType("smallint unsigned"); + + b.Property("SecurityLevel") + .HasColumnType("int"); + + b.Property("StartProfiler") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("StartupTimeout") + .IsRequired() + .HasColumnType("int unsigned"); + + b.Property("TopicRequestTimeout") + .IsRequired() + .HasColumnType("int unsigned"); + + b.Property("Visibility") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DreamDaemonSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamMakerSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("ApiValidationPort") + .IsRequired() + .HasColumnType("smallint unsigned"); + + b.Property("ApiValidationSecurityLevel") + .HasColumnType("int"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("ProjectName") + .HasMaxLength(10000) + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("ProjectName"), "utf8mb4"); + + b.Property("RequireDMApiValidation") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("Timeout") + .IsRequired() + .HasColumnType("time(6)"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DreamMakerSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Instance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("AutoUpdateInterval") + .IsRequired() + .HasColumnType("int unsigned"); + + b.Property("ChatBotLimit") + .IsRequired() + .HasColumnType("smallint unsigned"); + + b.Property("ConfigurationType") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("Name"), "utf8mb4"); + + b.Property("Online") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("Path") + .IsRequired() + .HasColumnType("varchar(255)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("Path"), "utf8mb4"); + + b.Property("SwarmIdentifer") + .HasColumnType("varchar(255)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("SwarmIdentifer"), "utf8mb4"); + + b.HasKey("Id"); + + b.HasIndex("Path", "SwarmIdentifer") + .IsUnique(); + + b.ToTable("Instances"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.InstancePermissionSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("ByondRights") + .HasColumnType("bigint unsigned"); + + b.Property("ChatBotRights") + .HasColumnType("bigint unsigned"); + + b.Property("ConfigurationRights") + .HasColumnType("bigint unsigned"); + + b.Property("DreamDaemonRights") + .HasColumnType("bigint unsigned"); + + b.Property("DreamMakerRights") + .HasColumnType("bigint unsigned"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("InstancePermissionSetRights") + .HasColumnType("bigint unsigned"); + + b.Property("PermissionSetId") + .HasColumnType("bigint"); + + b.Property("RepositoryRights") + .HasColumnType("bigint unsigned"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId"); + + b.HasIndex("PermissionSetId", "InstanceId") + .IsUnique(); + + b.ToTable("InstancePermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("CancelRight") + .HasColumnType("bigint unsigned"); + + b.Property("CancelRightsType") + .HasColumnType("bigint unsigned"); + + b.Property("Cancelled") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("CancelledById") + .HasColumnType("bigint"); + + b.Property("Description") + .IsRequired() + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("Description"), "utf8mb4"); + + b.Property("ErrorCode") + .HasColumnType("int unsigned"); + + b.Property("ExceptionDetails") + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("ExceptionDetails"), "utf8mb4"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("JobCode") + .HasColumnType("tinyint unsigned"); + + b.Property("StartedAt") + .IsRequired() + .HasColumnType("datetime(6)"); + + b.Property("StartedById") + .HasColumnType("bigint"); + + b.Property("StoppedAt") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("CancelledById"); + + b.HasIndex("InstanceId"); + + b.HasIndex("StartedById"); + + b.ToTable("Jobs"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.OAuthConnection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("ExternalUserId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("ExternalUserId"), "utf8mb4"); + + b.Property("Provider") + .HasColumnType("int"); + + b.Property("UserId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.HasIndex("Provider", "ExternalUserId") + .IsUnique(); + + b.ToTable("OAuthConnections"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("AdministrationRights") + .HasColumnType("bigint unsigned"); + + b.Property("GroupId") + .HasColumnType("bigint"); + + b.Property("InstanceManagerRights") + .HasColumnType("bigint unsigned"); + + b.Property("UserId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("GroupId") + .IsUnique(); + + b.HasIndex("UserId") + .IsUnique(); + + b.ToTable("PermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ReattachInformation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("AccessIdentifier") + .IsRequired() + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("AccessIdentifier"), "utf8mb4"); + + b.Property("CompileJobId") + .HasColumnType("bigint"); + + b.Property("InitialCompileJobId") + .HasColumnType("bigint"); + + b.Property("LaunchSecurityLevel") + .HasColumnType("int"); + + b.Property("LaunchVisibility") + .HasColumnType("int"); + + b.Property("Port") + .HasColumnType("smallint unsigned"); + + b.Property("ProcessId") + .HasColumnType("int"); + + b.Property("RebootState") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CompileJobId"); + + b.HasIndex("InitialCompileJobId"); + + b.ToTable("ReattachInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("AccessToken") + .HasMaxLength(10000) + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("AccessToken"), "utf8mb4"); + + b.Property("AccessUser") + .HasMaxLength(10000) + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("AccessUser"), "utf8mb4"); + + b.Property("AutoUpdatesKeepTestMerges") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("AutoUpdatesSynchronize") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("CommitterEmail") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("CommitterEmail"), "utf8mb4"); + + b.Property("CommitterName") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("CommitterName"), "utf8mb4"); + + b.Property("CreateGitHubDeployments") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("PostTestMergeComment") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("PushTestMergeCommits") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("ShowTestMergeCommitters") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("UpdateSubmodules") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("RepositorySettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevInfoTestMerge", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("RevisionInformationId") + .HasColumnType("bigint"); + + b.Property("TestMergeId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("RevisionInformationId"); + + b.HasIndex("TestMergeId"); + + b.ToTable("RevInfoTestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("CommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("varchar(40)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("CommitSha"), "utf8mb4"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("OriginCommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("varchar(40)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("OriginCommitSha"), "utf8mb4"); + + b.Property("Timestamp") + .HasColumnType("datetime(6)"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId", "CommitSha") + .IsUnique(); + + b.ToTable("RevisionInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("Author") + .IsRequired() + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("Author"), "utf8mb4"); + + b.Property("BodyAtMerge") + .IsRequired() + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("BodyAtMerge"), "utf8mb4"); + + b.Property("Comment") + .HasMaxLength(10000) + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("Comment"), "utf8mb4"); + + b.Property("MergedAt") + .HasColumnType("datetime(6)"); + + b.Property("MergedById") + .HasColumnType("bigint"); + + b.Property("Number") + .HasColumnType("int"); + + b.Property("PrimaryRevisionInformationId") + .IsRequired() + .HasColumnType("bigint"); + + b.Property("TargetCommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("varchar(40)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("TargetCommitSha"), "utf8mb4"); + + b.Property("TitleAtMerge") + .IsRequired() + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("TitleAtMerge"), "utf8mb4"); + + b.Property("Url") + .IsRequired() + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("Url"), "utf8mb4"); + + b.HasKey("Id"); + + b.HasIndex("MergedById"); + + b.HasIndex("PrimaryRevisionInformationId") + .IsUnique(); + + b.ToTable("TestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("CanonicalName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("CanonicalName"), "utf8mb4"); + + b.Property("CreatedAt") + .IsRequired() + .HasColumnType("datetime(6)"); + + b.Property("CreatedById") + .HasColumnType("bigint"); + + b.Property("Enabled") + .IsRequired() + .HasColumnType("tinyint(1)"); + + b.Property("GroupId") + .HasColumnType("bigint"); + + b.Property("LastPasswordUpdate") + .HasColumnType("datetime(6)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("Name"), "utf8mb4"); + + b.Property("PasswordHash") + .HasColumnType("longtext"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("PasswordHash"), "utf8mb4"); + + b.Property("SystemIdentifier") + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("SystemIdentifier"), "utf8mb4"); + + b.HasKey("Id"); + + b.HasIndex("CanonicalName") + .IsUnique(); + + b.HasIndex("CreatedById"); + + b.HasIndex("GroupId"); + + b.HasIndex("SystemIdentifier") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.UserGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("varchar(100)"); + + MySqlPropertyBuilderExtensions.HasCharSet(b.Property("Name"), "utf8mb4"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Groups"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("ChatSettings") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatChannel", b => + { + b.HasOne("Tgstation.Server.Host.Models.ChatBot", "ChatSettings") + .WithMany("Channels") + .HasForeignKey("ChatSettingsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ChatSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.CompileJob", b => + { + b.HasOne("Tgstation.Server.Host.Models.Job", "Job") + .WithOne() + .HasForeignKey("Tgstation.Server.Host.Models.CompileJob", "JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "RevisionInformation") + .WithMany("CompileJobs") + .HasForeignKey("RevisionInformationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Job"); + + b.Navigation("RevisionInformation"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamDaemonSettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("DreamDaemonSettings") + .HasForeignKey("Tgstation.Server.Host.Models.DreamDaemonSettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamMakerSettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("DreamMakerSettings") + .HasForeignKey("Tgstation.Server.Host.Models.DreamMakerSettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.InstancePermissionSet", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("InstancePermissionSets") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.PermissionSet", "PermissionSet") + .WithMany("InstancePermissionSets") + .HasForeignKey("PermissionSetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + + b.Navigation("PermissionSet"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Job", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "CancelledBy") + .WithMany() + .HasForeignKey("CancelledById"); + + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("Jobs") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.User", "StartedBy") + .WithMany() + .HasForeignKey("StartedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CancelledBy"); + + b.Navigation("Instance"); + + b.Navigation("StartedBy"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.OAuthConnection", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "User") + .WithMany("OAuthConnections") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.HasOne("Tgstation.Server.Host.Models.UserGroup", "Group") + .WithOne("PermissionSet") + .HasForeignKey("Tgstation.Server.Host.Models.PermissionSet", "GroupId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("Tgstation.Server.Host.Models.User", "User") + .WithOne("PermissionSet") + .HasForeignKey("Tgstation.Server.Host.Models.PermissionSet", "UserId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("Group"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ReattachInformation", b => + { + b.HasOne("Tgstation.Server.Host.Models.CompileJob", "CompileJob") + .WithMany() + .HasForeignKey("CompileJobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob") + .WithMany() + .HasForeignKey("InitialCompileJobId"); + + b.Navigation("CompileJob"); + + b.Navigation("InitialCompileJob"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("RepositorySettings") + .HasForeignKey("Tgstation.Server.Host.Models.RepositorySettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevInfoTestMerge", b => + { + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "RevisionInformation") + .WithMany("ActiveTestMerges") + .HasForeignKey("RevisionInformationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.TestMerge", "TestMerge") + .WithMany("RevisonInformations") + .HasForeignKey("TestMergeId") + .OnDelete(DeleteBehavior.ClientNoAction) + .IsRequired(); + + b.Navigation("RevisionInformation"); + + b.Navigation("TestMerge"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("RevisionInformations") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "MergedBy") + .WithMany("TestMerges") + .HasForeignKey("MergedById") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "PrimaryRevisionInformation") + .WithOne("PrimaryTestMerge") + .HasForeignKey("Tgstation.Server.Host.Models.TestMerge", "PrimaryRevisionInformationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MergedBy"); + + b.Navigation("PrimaryRevisionInformation"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "CreatedBy") + .WithMany("CreatedUsers") + .HasForeignKey("CreatedById"); + + b.HasOne("Tgstation.Server.Host.Models.UserGroup", "Group") + .WithMany("Users") + .HasForeignKey("GroupId"); + + b.Navigation("CreatedBy"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.Navigation("Channels"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Instance", b => + { + b.Navigation("ChatSettings"); + + b.Navigation("DreamDaemonSettings"); + + b.Navigation("DreamMakerSettings"); + + b.Navigation("InstancePermissionSets"); + + b.Navigation("Jobs"); + + b.Navigation("RepositorySettings"); + + b.Navigation("RevisionInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.Navigation("InstancePermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.Navigation("ActiveTestMerges"); + + b.Navigation("CompileJobs"); + + b.Navigation("PrimaryTestMerge"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.Navigation("RevisonInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.Navigation("CreatedUsers"); + + b.Navigation("OAuthConnections"); + + b.Navigation("PermissionSet"); + + b.Navigation("TestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.UserGroup", b => + { + b.Navigation("PermissionSet") + .IsRequired(); + + b.Navigation("Users"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Tgstation.Server.Host/Database/Migrations/20231105004808_MYAddJobCodes.cs b/src/Tgstation.Server.Host/Database/Migrations/20231105004808_MYAddJobCodes.cs new file mode 100644 index 00000000000..61eb53b6a5d --- /dev/null +++ b/src/Tgstation.Server.Host/Database/Migrations/20231105004808_MYAddJobCodes.cs @@ -0,0 +1,33 @@ +using System; + +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Tgstation.Server.Host.Database.Migrations +{ + /// + public partial class MYAddJobCodes : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + ArgumentNullException.ThrowIfNull(migrationBuilder); + migrationBuilder.AddColumn( + name: "JobCode", + table: "Jobs", + type: "tinyint unsigned", + nullable: false, + defaultValue: (byte)0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + ArgumentNullException.ThrowIfNull(migrationBuilder); + migrationBuilder.DropColumn( + name: "JobCode", + table: "Jobs"); + } + } +} diff --git a/src/Tgstation.Server.Host/Database/Migrations/20231105004814_PGAddJobCodes.Designer.cs b/src/Tgstation.Server.Host/Database/Migrations/20231105004814_PGAddJobCodes.Designer.cs new file mode 100644 index 00000000000..6ad0149338d --- /dev/null +++ b/src/Tgstation.Server.Host/Database/Migrations/20231105004814_PGAddJobCodes.Designer.cs @@ -0,0 +1,1069 @@ +// +using System; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Tgstation.Server.Host.Database.Migrations +{ + [DbContext(typeof(PostgresSqlDatabaseContext))] + [Migration("20231105004814_PGAddJobCodes")] + partial class PGAddJobCodes + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "7.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ChannelLimit") + .HasColumnType("integer"); + + b.Property("ConnectionString") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("character varying(10000)"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Provider") + .HasColumnType("integer"); + + b.Property("ReconnectionInterval") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId", "Name") + .IsUnique(); + + b.ToTable("ChatBots"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatChannel", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ChatSettingsId") + .HasColumnType("bigint"); + + b.Property("DiscordChannelId") + .HasColumnType("numeric(20,0)"); + + b.Property("IrcChannel") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("IsAdminChannel") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("IsSystemChannel") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("IsUpdatesChannel") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("IsWatchdogChannel") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("Tag") + .HasMaxLength(10000) + .HasColumnType("character varying(10000)"); + + b.HasKey("Id"); + + b.HasIndex("ChatSettingsId", "DiscordChannelId") + .IsUnique(); + + b.HasIndex("ChatSettingsId", "IrcChannel") + .IsUnique(); + + b.ToTable("ChatChannels"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.CompileJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ByondVersion") + .IsRequired() + .HasColumnType("text"); + + b.Property("DMApiMajorVersion") + .HasColumnType("integer"); + + b.Property("DMApiMinorVersion") + .HasColumnType("integer"); + + b.Property("DMApiPatchVersion") + .HasColumnType("integer"); + + b.Property("DirectoryName") + .IsRequired() + .HasColumnType("uuid"); + + b.Property("DmeName") + .IsRequired() + .HasColumnType("text"); + + b.Property("GitHubDeploymentId") + .HasColumnType("integer"); + + b.Property("GitHubRepoId") + .HasColumnType("bigint"); + + b.Property("JobId") + .HasColumnType("bigint"); + + b.Property("MinimumSecurityLevel") + .HasColumnType("integer"); + + b.Property("Output") + .IsRequired() + .HasColumnType("text"); + + b.Property("RepositoryOrigin") + .HasColumnType("text"); + + b.Property("RevisionInformationId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("DirectoryName"); + + b.HasIndex("JobId") + .IsUnique(); + + b.HasIndex("RevisionInformationId"); + + b.ToTable("CompileJobs"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamDaemonSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AdditionalParameters") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("character varying(10000)"); + + b.Property("AllowWebClient") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("AutoStart") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("DumpOnHealthCheckRestart") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("HealthCheckSeconds") + .HasColumnType("bigint"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("LogOutput") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("MapThreads") + .HasColumnType("bigint"); + + b.Property("Port") + .HasColumnType("integer"); + + b.Property("SecurityLevel") + .HasColumnType("integer"); + + b.Property("StartProfiler") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("StartupTimeout") + .HasColumnType("bigint"); + + b.Property("TopicRequestTimeout") + .HasColumnType("bigint"); + + b.Property("Visibility") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DreamDaemonSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamMakerSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ApiValidationPort") + .HasColumnType("integer"); + + b.Property("ApiValidationSecurityLevel") + .HasColumnType("integer"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("ProjectName") + .HasMaxLength(10000) + .HasColumnType("character varying(10000)"); + + b.Property("RequireDMApiValidation") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("Timeout") + .IsRequired() + .HasColumnType("interval"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DreamMakerSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Instance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AutoUpdateInterval") + .HasColumnType("bigint"); + + b.Property("ChatBotLimit") + .HasColumnType("integer"); + + b.Property("ConfigurationType") + .HasColumnType("integer"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Online") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.Property("SwarmIdentifer") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Path", "SwarmIdentifer") + .IsUnique(); + + b.ToTable("Instances"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.InstancePermissionSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ByondRights") + .HasColumnType("numeric(20,0)"); + + b.Property("ChatBotRights") + .HasColumnType("numeric(20,0)"); + + b.Property("ConfigurationRights") + .HasColumnType("numeric(20,0)"); + + b.Property("DreamDaemonRights") + .HasColumnType("numeric(20,0)"); + + b.Property("DreamMakerRights") + .HasColumnType("numeric(20,0)"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("InstancePermissionSetRights") + .HasColumnType("numeric(20,0)"); + + b.Property("PermissionSetId") + .HasColumnType("bigint"); + + b.Property("RepositoryRights") + .HasColumnType("numeric(20,0)"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId"); + + b.HasIndex("PermissionSetId", "InstanceId") + .IsUnique(); + + b.ToTable("InstancePermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CancelRight") + .HasColumnType("numeric(20,0)"); + + b.Property("CancelRightsType") + .HasColumnType("numeric(20,0)"); + + b.Property("Cancelled") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("CancelledById") + .HasColumnType("bigint"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text"); + + b.Property("ErrorCode") + .HasColumnType("bigint"); + + b.Property("ExceptionDetails") + .HasColumnType("text"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("JobCode") + .HasColumnType("smallint"); + + b.Property("StartedAt") + .IsRequired() + .HasColumnType("timestamp with time zone"); + + b.Property("StartedById") + .HasColumnType("bigint"); + + b.Property("StoppedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("CancelledById"); + + b.HasIndex("InstanceId"); + + b.HasIndex("StartedById"); + + b.ToTable("Jobs"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.OAuthConnection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ExternalUserId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Provider") + .HasColumnType("integer"); + + b.Property("UserId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.HasIndex("Provider", "ExternalUserId") + .IsUnique(); + + b.ToTable("OAuthConnections"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AdministrationRights") + .HasColumnType("numeric(20,0)"); + + b.Property("GroupId") + .HasColumnType("bigint"); + + b.Property("InstanceManagerRights") + .HasColumnType("numeric(20,0)"); + + b.Property("UserId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("GroupId") + .IsUnique(); + + b.HasIndex("UserId") + .IsUnique(); + + b.ToTable("PermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ReattachInformation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AccessIdentifier") + .IsRequired() + .HasColumnType("text"); + + b.Property("CompileJobId") + .HasColumnType("bigint"); + + b.Property("InitialCompileJobId") + .HasColumnType("bigint"); + + b.Property("LaunchSecurityLevel") + .HasColumnType("integer"); + + b.Property("LaunchVisibility") + .HasColumnType("integer"); + + b.Property("Port") + .HasColumnType("integer"); + + b.Property("ProcessId") + .HasColumnType("integer"); + + b.Property("RebootState") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("CompileJobId"); + + b.HasIndex("InitialCompileJobId"); + + b.ToTable("ReattachInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AccessToken") + .HasMaxLength(10000) + .HasColumnType("character varying(10000)"); + + b.Property("AccessUser") + .HasMaxLength(10000) + .HasColumnType("character varying(10000)"); + + b.Property("AutoUpdatesKeepTestMerges") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("AutoUpdatesSynchronize") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("CommitterEmail") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("character varying(10000)"); + + b.Property("CommitterName") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("character varying(10000)"); + + b.Property("CreateGitHubDeployments") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("PostTestMergeComment") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("PushTestMergeCommits") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("ShowTestMergeCommitters") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("UpdateSubmodules") + .IsRequired() + .HasColumnType("boolean"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("RepositorySettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevInfoTestMerge", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("RevisionInformationId") + .HasColumnType("bigint"); + + b.Property("TestMergeId") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("RevisionInformationId"); + + b.HasIndex("TestMergeId"); + + b.ToTable("RevInfoTestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("character varying(40)"); + + b.Property("InstanceId") + .HasColumnType("bigint"); + + b.Property("OriginCommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("character varying(40)"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId", "CommitSha") + .IsUnique(); + + b.ToTable("RevisionInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Author") + .IsRequired() + .HasColumnType("text"); + + b.Property("BodyAtMerge") + .IsRequired() + .HasColumnType("text"); + + b.Property("Comment") + .HasMaxLength(10000) + .HasColumnType("character varying(10000)"); + + b.Property("MergedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("MergedById") + .HasColumnType("bigint"); + + b.Property("Number") + .HasColumnType("integer"); + + b.Property("PrimaryRevisionInformationId") + .IsRequired() + .HasColumnType("bigint"); + + b.Property("TargetCommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("character varying(40)"); + + b.Property("TitleAtMerge") + .IsRequired() + .HasColumnType("text"); + + b.Property("Url") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("MergedById"); + + b.HasIndex("PrimaryRevisionInformationId") + .IsUnique(); + + b.ToTable("TestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CanonicalName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("CreatedAt") + .IsRequired() + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedById") + .HasColumnType("bigint"); + + b.Property("Enabled") + .IsRequired() + .HasColumnType("boolean"); + + b.Property("GroupId") + .HasColumnType("bigint"); + + b.Property("LastPasswordUpdate") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("PasswordHash") + .HasColumnType("text"); + + b.Property("SystemIdentifier") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.HasKey("Id"); + + b.HasIndex("CanonicalName") + .IsUnique(); + + b.HasIndex("CreatedById"); + + b.HasIndex("GroupId"); + + b.HasIndex("SystemIdentifier") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.UserGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Groups"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("ChatSettings") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatChannel", b => + { + b.HasOne("Tgstation.Server.Host.Models.ChatBot", "ChatSettings") + .WithMany("Channels") + .HasForeignKey("ChatSettingsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ChatSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.CompileJob", b => + { + b.HasOne("Tgstation.Server.Host.Models.Job", "Job") + .WithOne() + .HasForeignKey("Tgstation.Server.Host.Models.CompileJob", "JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "RevisionInformation") + .WithMany("CompileJobs") + .HasForeignKey("RevisionInformationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Job"); + + b.Navigation("RevisionInformation"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamDaemonSettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("DreamDaemonSettings") + .HasForeignKey("Tgstation.Server.Host.Models.DreamDaemonSettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamMakerSettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("DreamMakerSettings") + .HasForeignKey("Tgstation.Server.Host.Models.DreamMakerSettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.InstancePermissionSet", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("InstancePermissionSets") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.PermissionSet", "PermissionSet") + .WithMany("InstancePermissionSets") + .HasForeignKey("PermissionSetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + + b.Navigation("PermissionSet"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Job", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "CancelledBy") + .WithMany() + .HasForeignKey("CancelledById"); + + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("Jobs") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.User", "StartedBy") + .WithMany() + .HasForeignKey("StartedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CancelledBy"); + + b.Navigation("Instance"); + + b.Navigation("StartedBy"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.OAuthConnection", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "User") + .WithMany("OAuthConnections") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.HasOne("Tgstation.Server.Host.Models.UserGroup", "Group") + .WithOne("PermissionSet") + .HasForeignKey("Tgstation.Server.Host.Models.PermissionSet", "GroupId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("Tgstation.Server.Host.Models.User", "User") + .WithOne("PermissionSet") + .HasForeignKey("Tgstation.Server.Host.Models.PermissionSet", "UserId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("Group"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ReattachInformation", b => + { + b.HasOne("Tgstation.Server.Host.Models.CompileJob", "CompileJob") + .WithMany() + .HasForeignKey("CompileJobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob") + .WithMany() + .HasForeignKey("InitialCompileJobId"); + + b.Navigation("CompileJob"); + + b.Navigation("InitialCompileJob"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("RepositorySettings") + .HasForeignKey("Tgstation.Server.Host.Models.RepositorySettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevInfoTestMerge", b => + { + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "RevisionInformation") + .WithMany("ActiveTestMerges") + .HasForeignKey("RevisionInformationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.TestMerge", "TestMerge") + .WithMany("RevisonInformations") + .HasForeignKey("TestMergeId") + .OnDelete(DeleteBehavior.ClientNoAction) + .IsRequired(); + + b.Navigation("RevisionInformation"); + + b.Navigation("TestMerge"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("RevisionInformations") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "MergedBy") + .WithMany("TestMerges") + .HasForeignKey("MergedById") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "PrimaryRevisionInformation") + .WithOne("PrimaryTestMerge") + .HasForeignKey("Tgstation.Server.Host.Models.TestMerge", "PrimaryRevisionInformationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MergedBy"); + + b.Navigation("PrimaryRevisionInformation"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "CreatedBy") + .WithMany("CreatedUsers") + .HasForeignKey("CreatedById"); + + b.HasOne("Tgstation.Server.Host.Models.UserGroup", "Group") + .WithMany("Users") + .HasForeignKey("GroupId"); + + b.Navigation("CreatedBy"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.Navigation("Channels"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Instance", b => + { + b.Navigation("ChatSettings"); + + b.Navigation("DreamDaemonSettings"); + + b.Navigation("DreamMakerSettings"); + + b.Navigation("InstancePermissionSets"); + + b.Navigation("Jobs"); + + b.Navigation("RepositorySettings"); + + b.Navigation("RevisionInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.Navigation("InstancePermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.Navigation("ActiveTestMerges"); + + b.Navigation("CompileJobs"); + + b.Navigation("PrimaryTestMerge"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.Navigation("RevisonInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.Navigation("CreatedUsers"); + + b.Navigation("OAuthConnections"); + + b.Navigation("PermissionSet"); + + b.Navigation("TestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.UserGroup", b => + { + b.Navigation("PermissionSet") + .IsRequired(); + + b.Navigation("Users"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Tgstation.Server.Host/Database/Migrations/20231105004814_PGAddJobCodes.cs b/src/Tgstation.Server.Host/Database/Migrations/20231105004814_PGAddJobCodes.cs new file mode 100644 index 00000000000..5bd1b863f68 --- /dev/null +++ b/src/Tgstation.Server.Host/Database/Migrations/20231105004814_PGAddJobCodes.cs @@ -0,0 +1,33 @@ +using System; + +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Tgstation.Server.Host.Database.Migrations +{ + /// + public partial class PGAddJobCodes : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + ArgumentNullException.ThrowIfNull(migrationBuilder); + migrationBuilder.AddColumn( + name: "JobCode", + table: "Jobs", + type: "smallint", + nullable: false, + defaultValue: (byte)0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + ArgumentNullException.ThrowIfNull(migrationBuilder); + migrationBuilder.DropColumn( + name: "JobCode", + table: "Jobs"); + } + } +} diff --git a/src/Tgstation.Server.Host/Database/Migrations/20231105004820_SLAddJobCodes.Designer.cs b/src/Tgstation.Server.Host/Database/Migrations/20231105004820_SLAddJobCodes.Designer.cs new file mode 100644 index 00000000000..718743a339c --- /dev/null +++ b/src/Tgstation.Server.Host/Database/Migrations/20231105004820_SLAddJobCodes.Designer.cs @@ -0,0 +1,1041 @@ +// +using System; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Tgstation.Server.Host.Database.Migrations +{ + [DbContext(typeof(SqliteDatabaseContext))] + [Migration("20231105004820_SLAddJobCodes")] + partial class SLAddJobCodes + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "7.0.13"); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ChannelLimit") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("ConnectionString") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("InstanceId") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("Provider") + .HasColumnType("INTEGER"); + + b.Property("ReconnectionInterval") + .IsRequired() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId", "Name") + .IsUnique(); + + b.ToTable("ChatBots"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatChannel", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ChatSettingsId") + .HasColumnType("INTEGER"); + + b.Property("DiscordChannelId") + .HasColumnType("INTEGER"); + + b.Property("IrcChannel") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("IsAdminChannel") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("IsSystemChannel") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("IsUpdatesChannel") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("IsWatchdogChannel") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("Tag") + .HasMaxLength(10000) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ChatSettingsId", "DiscordChannelId") + .IsUnique(); + + b.HasIndex("ChatSettingsId", "IrcChannel") + .IsUnique(); + + b.ToTable("ChatChannels"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.CompileJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ByondVersion") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DMApiMajorVersion") + .HasColumnType("INTEGER"); + + b.Property("DMApiMinorVersion") + .HasColumnType("INTEGER"); + + b.Property("DMApiPatchVersion") + .HasColumnType("INTEGER"); + + b.Property("DirectoryName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DmeName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("GitHubDeploymentId") + .HasColumnType("INTEGER"); + + b.Property("GitHubRepoId") + .HasColumnType("INTEGER"); + + b.Property("JobId") + .HasColumnType("INTEGER"); + + b.Property("MinimumSecurityLevel") + .HasColumnType("INTEGER"); + + b.Property("Output") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("RepositoryOrigin") + .HasColumnType("TEXT"); + + b.Property("RevisionInformationId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("DirectoryName"); + + b.HasIndex("JobId") + .IsUnique(); + + b.HasIndex("RevisionInformationId"); + + b.ToTable("CompileJobs"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamDaemonSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AdditionalParameters") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("TEXT"); + + b.Property("AllowWebClient") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("AutoStart") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("DumpOnHealthCheckRestart") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("HealthCheckSeconds") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("InstanceId") + .HasColumnType("INTEGER"); + + b.Property("LogOutput") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("MapThreads") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("Port") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("SecurityLevel") + .HasColumnType("INTEGER"); + + b.Property("StartProfiler") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("StartupTimeout") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("TopicRequestTimeout") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("Visibility") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DreamDaemonSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamMakerSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ApiValidationPort") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("ApiValidationSecurityLevel") + .HasColumnType("INTEGER"); + + b.Property("InstanceId") + .HasColumnType("INTEGER"); + + b.Property("ProjectName") + .HasMaxLength(10000) + .HasColumnType("TEXT"); + + b.Property("RequireDMApiValidation") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("Timeout") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DreamMakerSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Instance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AutoUpdateInterval") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("ChatBotLimit") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("ConfigurationType") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("Online") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("Path") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SwarmIdentifer") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Path", "SwarmIdentifer") + .IsUnique(); + + b.ToTable("Instances"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.InstancePermissionSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ByondRights") + .HasColumnType("INTEGER"); + + b.Property("ChatBotRights") + .HasColumnType("INTEGER"); + + b.Property("ConfigurationRights") + .HasColumnType("INTEGER"); + + b.Property("DreamDaemonRights") + .HasColumnType("INTEGER"); + + b.Property("DreamMakerRights") + .HasColumnType("INTEGER"); + + b.Property("InstanceId") + .HasColumnType("INTEGER"); + + b.Property("InstancePermissionSetRights") + .HasColumnType("INTEGER"); + + b.Property("PermissionSetId") + .HasColumnType("INTEGER"); + + b.Property("RepositoryRights") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId"); + + b.HasIndex("PermissionSetId", "InstanceId") + .IsUnique(); + + b.ToTable("InstancePermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CancelRight") + .HasColumnType("INTEGER"); + + b.Property("CancelRightsType") + .HasColumnType("INTEGER"); + + b.Property("Cancelled") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("CancelledById") + .HasColumnType("INTEGER"); + + b.Property("Description") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ErrorCode") + .HasColumnType("INTEGER"); + + b.Property("ExceptionDetails") + .HasColumnType("TEXT"); + + b.Property("InstanceId") + .HasColumnType("INTEGER"); + + b.Property("JobCode") + .HasColumnType("INTEGER"); + + b.Property("StartedAt") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("StartedById") + .HasColumnType("INTEGER"); + + b.Property("StoppedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CancelledById"); + + b.HasIndex("InstanceId"); + + b.HasIndex("StartedById"); + + b.ToTable("Jobs"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.OAuthConnection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ExternalUserId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("Provider") + .HasColumnType("INTEGER"); + + b.Property("UserId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.HasIndex("Provider", "ExternalUserId") + .IsUnique(); + + b.ToTable("OAuthConnections"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AdministrationRights") + .HasColumnType("INTEGER"); + + b.Property("GroupId") + .HasColumnType("INTEGER"); + + b.Property("InstanceManagerRights") + .HasColumnType("INTEGER"); + + b.Property("UserId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("GroupId") + .IsUnique(); + + b.HasIndex("UserId") + .IsUnique(); + + b.ToTable("PermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ReattachInformation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AccessIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CompileJobId") + .HasColumnType("INTEGER"); + + b.Property("InitialCompileJobId") + .HasColumnType("INTEGER"); + + b.Property("LaunchSecurityLevel") + .HasColumnType("INTEGER"); + + b.Property("LaunchVisibility") + .HasColumnType("INTEGER"); + + b.Property("Port") + .HasColumnType("INTEGER"); + + b.Property("ProcessId") + .HasColumnType("INTEGER"); + + b.Property("RebootState") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("CompileJobId"); + + b.HasIndex("InitialCompileJobId"); + + b.ToTable("ReattachInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AccessToken") + .HasMaxLength(10000) + .HasColumnType("TEXT"); + + b.Property("AccessUser") + .HasMaxLength(10000) + .HasColumnType("TEXT"); + + b.Property("AutoUpdatesKeepTestMerges") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("AutoUpdatesSynchronize") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("CommitterEmail") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("TEXT"); + + b.Property("CommitterName") + .IsRequired() + .HasMaxLength(10000) + .HasColumnType("TEXT"); + + b.Property("CreateGitHubDeployments") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("InstanceId") + .HasColumnType("INTEGER"); + + b.Property("PostTestMergeComment") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("PushTestMergeCommits") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("ShowTestMergeCommitters") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("UpdateSubmodules") + .IsRequired() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("RepositorySettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevInfoTestMerge", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("RevisionInformationId") + .HasColumnType("INTEGER"); + + b.Property("TestMergeId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("RevisionInformationId"); + + b.HasIndex("TestMergeId"); + + b.ToTable("RevInfoTestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("TEXT"); + + b.Property("InstanceId") + .HasColumnType("INTEGER"); + + b.Property("OriginCommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("TEXT"); + + b.Property("Timestamp") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId", "CommitSha") + .IsUnique(); + + b.ToTable("RevisionInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Author") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("BodyAtMerge") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Comment") + .HasMaxLength(10000) + .HasColumnType("TEXT"); + + b.Property("MergedAt") + .HasColumnType("TEXT"); + + b.Property("MergedById") + .HasColumnType("INTEGER"); + + b.Property("Number") + .HasColumnType("INTEGER"); + + b.Property("PrimaryRevisionInformationId") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("TargetCommitSha") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("TEXT"); + + b.Property("TitleAtMerge") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Url") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("MergedById"); + + b.HasIndex("PrimaryRevisionInformationId") + .IsUnique(); + + b.ToTable("TestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CanonicalName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CreatedById") + .HasColumnType("INTEGER"); + + b.Property("Enabled") + .IsRequired() + .HasColumnType("INTEGER"); + + b.Property("GroupId") + .HasColumnType("INTEGER"); + + b.Property("LastPasswordUpdate") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("PasswordHash") + .HasColumnType("TEXT"); + + b.Property("SystemIdentifier") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CanonicalName") + .IsUnique(); + + b.HasIndex("CreatedById"); + + b.HasIndex("GroupId"); + + b.HasIndex("SystemIdentifier") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.UserGroup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Groups"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("ChatSettings") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatChannel", b => + { + b.HasOne("Tgstation.Server.Host.Models.ChatBot", "ChatSettings") + .WithMany("Channels") + .HasForeignKey("ChatSettingsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ChatSettings"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.CompileJob", b => + { + b.HasOne("Tgstation.Server.Host.Models.Job", "Job") + .WithOne() + .HasForeignKey("Tgstation.Server.Host.Models.CompileJob", "JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "RevisionInformation") + .WithMany("CompileJobs") + .HasForeignKey("RevisionInformationId") + .OnDelete(DeleteBehavior.ClientNoAction) + .IsRequired(); + + b.Navigation("Job"); + + b.Navigation("RevisionInformation"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamDaemonSettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("DreamDaemonSettings") + .HasForeignKey("Tgstation.Server.Host.Models.DreamDaemonSettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.DreamMakerSettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("DreamMakerSettings") + .HasForeignKey("Tgstation.Server.Host.Models.DreamMakerSettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.InstancePermissionSet", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("InstancePermissionSets") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.PermissionSet", "PermissionSet") + .WithMany("InstancePermissionSets") + .HasForeignKey("PermissionSetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + + b.Navigation("PermissionSet"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Job", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "CancelledBy") + .WithMany() + .HasForeignKey("CancelledById"); + + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("Jobs") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.User", "StartedBy") + .WithMany() + .HasForeignKey("StartedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CancelledBy"); + + b.Navigation("Instance"); + + b.Navigation("StartedBy"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.OAuthConnection", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "User") + .WithMany("OAuthConnections") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.HasOne("Tgstation.Server.Host.Models.UserGroup", "Group") + .WithOne("PermissionSet") + .HasForeignKey("Tgstation.Server.Host.Models.PermissionSet", "GroupId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("Tgstation.Server.Host.Models.User", "User") + .WithOne("PermissionSet") + .HasForeignKey("Tgstation.Server.Host.Models.PermissionSet", "UserId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("Group"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ReattachInformation", b => + { + b.HasOne("Tgstation.Server.Host.Models.CompileJob", "CompileJob") + .WithMany() + .HasForeignKey("CompileJobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob") + .WithMany() + .HasForeignKey("InitialCompileJobId"); + + b.Navigation("CompileJob"); + + b.Navigation("InitialCompileJob"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithOne("RepositorySettings") + .HasForeignKey("Tgstation.Server.Host.Models.RepositorySettings", "InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevInfoTestMerge", b => + { + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "RevisionInformation") + .WithMany("ActiveTestMerges") + .HasForeignKey("RevisionInformationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.TestMerge", "TestMerge") + .WithMany("RevisonInformations") + .HasForeignKey("TestMergeId") + .OnDelete(DeleteBehavior.ClientNoAction) + .IsRequired(); + + b.Navigation("RevisionInformation"); + + b.Navigation("TestMerge"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.HasOne("Tgstation.Server.Host.Models.Instance", "Instance") + .WithMany("RevisionInformations") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Instance"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "MergedBy") + .WithMany("TestMerges") + .HasForeignKey("MergedById") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("Tgstation.Server.Host.Models.RevisionInformation", "PrimaryRevisionInformation") + .WithOne("PrimaryTestMerge") + .HasForeignKey("Tgstation.Server.Host.Models.TestMerge", "PrimaryRevisionInformationId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("MergedBy"); + + b.Navigation("PrimaryRevisionInformation"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.HasOne("Tgstation.Server.Host.Models.User", "CreatedBy") + .WithMany("CreatedUsers") + .HasForeignKey("CreatedById"); + + b.HasOne("Tgstation.Server.Host.Models.UserGroup", "Group") + .WithMany("Users") + .HasForeignKey("GroupId"); + + b.Navigation("CreatedBy"); + + b.Navigation("Group"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => + { + b.Navigation("Channels"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.Instance", b => + { + b.Navigation("ChatSettings"); + + b.Navigation("DreamDaemonSettings"); + + b.Navigation("DreamMakerSettings"); + + b.Navigation("InstancePermissionSets"); + + b.Navigation("Jobs"); + + b.Navigation("RepositorySettings"); + + b.Navigation("RevisionInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.PermissionSet", b => + { + b.Navigation("InstancePermissionSets"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.RevisionInformation", b => + { + b.Navigation("ActiveTestMerges"); + + b.Navigation("CompileJobs"); + + b.Navigation("PrimaryTestMerge"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.TestMerge", b => + { + b.Navigation("RevisonInformations"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.User", b => + { + b.Navigation("CreatedUsers"); + + b.Navigation("OAuthConnections"); + + b.Navigation("PermissionSet"); + + b.Navigation("TestMerges"); + }); + + modelBuilder.Entity("Tgstation.Server.Host.Models.UserGroup", b => + { + b.Navigation("PermissionSet") + .IsRequired(); + + b.Navigation("Users"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Tgstation.Server.Host/Database/Migrations/20231105004820_SLAddJobCodes.cs b/src/Tgstation.Server.Host/Database/Migrations/20231105004820_SLAddJobCodes.cs new file mode 100644 index 00000000000..76a2be02ee4 --- /dev/null +++ b/src/Tgstation.Server.Host/Database/Migrations/20231105004820_SLAddJobCodes.cs @@ -0,0 +1,33 @@ +using System; + +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Tgstation.Server.Host.Database.Migrations +{ + /// + public partial class SLAddJobCodes : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + ArgumentNullException.ThrowIfNull(migrationBuilder); + migrationBuilder.AddColumn( + name: "JobCode", + table: "Jobs", + type: "INTEGER", + nullable: false, + defaultValue: (byte)0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + ArgumentNullException.ThrowIfNull(migrationBuilder); + migrationBuilder.DropColumn( + name: "JobCode", + table: "Jobs"); + } + } +} diff --git a/src/Tgstation.Server.Host/Database/Migrations/MySqlDatabaseContextModelSnapshot.cs b/src/Tgstation.Server.Host/Database/Migrations/MySqlDatabaseContextModelSnapshot.cs index f70cac88acf..c5e6c701d7c 100644 --- a/src/Tgstation.Server.Host/Database/Migrations/MySqlDatabaseContextModelSnapshot.cs +++ b/src/Tgstation.Server.Host/Database/Migrations/MySqlDatabaseContextModelSnapshot.cs @@ -11,12 +11,11 @@ namespace Tgstation.Server.Host.Database.Migrations [DbContext(typeof(MySqlDatabaseContext))] partial class MySqlDatabaseContextModelSnapshot : ModelSnapshot { - /// protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "7.0.7") + .HasAnnotation("ProductVersion", "7.0.13") .HasAnnotation("Relational:MaxIdentifierLength", 64); modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => @@ -416,6 +415,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("InstanceId") .HasColumnType("bigint"); + b.Property("JobCode") + .HasColumnType("tinyint unsigned"); + b.Property("StartedAt") .IsRequired() .HasColumnType("datetime(6)"); diff --git a/src/Tgstation.Server.Host/Database/Migrations/PostgresSqlDatabaseContextModelSnapshot.cs b/src/Tgstation.Server.Host/Database/Migrations/PostgresSqlDatabaseContextModelSnapshot.cs index aa74a6f86d5..f550058f18e 100644 --- a/src/Tgstation.Server.Host/Database/Migrations/PostgresSqlDatabaseContextModelSnapshot.cs +++ b/src/Tgstation.Server.Host/Database/Migrations/PostgresSqlDatabaseContextModelSnapshot.cs @@ -11,12 +11,11 @@ namespace Tgstation.Server.Host.Database.Migrations [DbContext(typeof(PostgresSqlDatabaseContext))] partial class PostgresSqlDatabaseContextModelSnapshot : ModelSnapshot { - /// protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "7.0.7") + .HasAnnotation("ProductVersion", "7.0.13") .HasAnnotation("Relational:MaxIdentifierLength", 63); NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); @@ -396,6 +395,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("InstanceId") .HasColumnType("bigint"); + b.Property("JobCode") + .HasColumnType("smallint"); + b.Property("StartedAt") .IsRequired() .HasColumnType("timestamp with time zone"); diff --git a/src/Tgstation.Server.Host/Database/Migrations/SqlServerDatabaseContextModelSnapshot.cs b/src/Tgstation.Server.Host/Database/Migrations/SqlServerDatabaseContextModelSnapshot.cs index 4534c99cf69..f8a83396a92 100644 --- a/src/Tgstation.Server.Host/Database/Migrations/SqlServerDatabaseContextModelSnapshot.cs +++ b/src/Tgstation.Server.Host/Database/Migrations/SqlServerDatabaseContextModelSnapshot.cs @@ -11,12 +11,11 @@ namespace Tgstation.Server.Host.Database.Migrations [DbContext(typeof(SqlServerDatabaseContext))] partial class SqlServerDatabaseContextModelSnapshot : ModelSnapshot { - /// protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "7.0.7") + .HasAnnotation("ProductVersion", "7.0.13") .HasAnnotation("Relational:MaxIdentifierLength", 128); SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); @@ -399,6 +398,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("InstanceId") .HasColumnType("bigint"); + b.Property("JobCode") + .HasColumnType("tinyint"); + b.Property("StartedAt") .IsRequired() .HasColumnType("datetimeoffset"); diff --git a/src/Tgstation.Server.Host/Database/Migrations/SqliteDatabaseContextModelSnapshot.cs b/src/Tgstation.Server.Host/Database/Migrations/SqliteDatabaseContextModelSnapshot.cs index b9526482875..c5f426c4701 100644 --- a/src/Tgstation.Server.Host/Database/Migrations/SqliteDatabaseContextModelSnapshot.cs +++ b/src/Tgstation.Server.Host/Database/Migrations/SqliteDatabaseContextModelSnapshot.cs @@ -11,11 +11,10 @@ namespace Tgstation.Server.Host.Database.Migrations [DbContext(typeof(SqliteDatabaseContext))] partial class SqliteDatabaseContextModelSnapshot : ModelSnapshot { - /// protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "7.0.7"); + modelBuilder.HasAnnotation("ProductVersion", "7.0.13"); modelBuilder.Entity("Tgstation.Server.Host.Models.ChatBot", b => { @@ -386,6 +385,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("InstanceId") .HasColumnType("INTEGER"); + b.Property("JobCode") + .HasColumnType("INTEGER"); + b.Property("StartedAt") .IsRequired() .HasColumnType("TEXT"); diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index 2212bba37dc..b1aa9c1e0a7 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -120,7 +120,7 @@ public ValueTask RegisterOperation(Job job, JobEntrypoint operation, Cancellatio job.Instance = new Models.Instance { - Id = job.Instance.Id ?? throw new InvalidOperationException("Instance associated with job does not have an Id!"), + Id = job.Instance.Id.Value, }; databaseContext.Instances.Attach(job.Instance); @@ -172,14 +172,14 @@ public Task StartAsync(CancellationToken cancellationToken) .Jobs .AsQueryable() .Where(y => !y.StoppedAt.HasValue) - .Select(y => y.Id) + .Select(y => y.Id.Value) .ToListAsync(cancellationToken); if (badJobIds.Count > 0) { logger.LogTrace("Cleaning {unfinishedJobCount} unfinished jobs...", badJobIds.Count); foreach (var badJobId in badJobIds) { - var job = new Job { Id = badJobId }; + var job = new Job(badJobId); databaseContext.Jobs.Attach(job); job.Cancelled = true; job.StoppedAt = DateTimeOffset.UtcNow; @@ -201,10 +201,7 @@ public Task StopAsync(CancellationToken cancellationToken) { noMoreJobsShouldStart = true; joinTasks = jobs.Select(x => CancelJob( - new Job - { - Id = x.Key, - }, + new Job(x.Key), null, true, cancellationToken)) @@ -234,7 +231,7 @@ await databaseContextFactory.UseContext(async databaseContext => { user ??= await databaseContext.Users.GetTgsUser(cancellationToken); - var updatedJob = new Job { Id = job.Id }; + var updatedJob = new Job(job.Id.Value); databaseContext.Jobs.Attach(updatedJob); var attachedUser = new User { Id = user.Id }; databaseContext.Users.Attach(attachedUser); @@ -425,10 +422,7 @@ await operation( await databaseContextFactory.UseContext(async databaseContext => { - var attachedJob = new Job - { - Id = job.Id, - }; + var attachedJob = new Job(job.Id.Value); databaseContext.Jobs.Attach(attachedJob); attachedJob.StoppedAt = DateTimeOffset.UtcNow; diff --git a/src/Tgstation.Server.Host/Models/Job.cs b/src/Tgstation.Server.Host/Models/Job.cs index a05b83b811b..8193402a6a5 100644 --- a/src/Tgstation.Server.Host/Models/Job.cs +++ b/src/Tgstation.Server.Host/Models/Job.cs @@ -1,6 +1,11 @@ -using System.ComponentModel.DataAnnotations; +using System; +using System.ComponentModel; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; +using Tgstation.Server.Api.Rights; namespace Tgstation.Server.Host.Models { @@ -26,10 +31,88 @@ public sealed class Job : Api.Models.Internal.Job, IApiTransformable + /// Creates a new job for registering in the . + /// + /// The of . + /// The value of . will be derived from this. + /// The value of . If , the user will be used. + /// The used to generate the value of . + /// The value of . will be derived from this. + /// A new ready to be registered with the . + public static Job Create(JobCode code, User startedBy, Api.Models.Instance instance, TRight cancelRight) + where TRight : Enum + => new ( + code, + startedBy, + instance, + RightsHelper.TypeToRight(), + (ulong)(object)cancelRight); + + /// + /// Creates a new job for registering in the . + /// + /// The value of . will be derived from this. + /// The value of . If , the user will be used. + /// The used to generate the value of . + /// A new ready to be registered with the . + public static Job Create(JobCode code, User startedBy, Api.Models.Instance instance) + => new ( + code, + startedBy, + instance, + null, + null); + + /// + /// Initializes a new instance of the class. + /// + [Obsolete("For use by EFCore only", true)] + public Job() + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public Job(long id) + { + Id = id; + } + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + /// The value of . + /// The value of . + Job(JobCode code, User startedBy, Api.Models.Instance instance, RightsType? cancelRightsType, ulong? cancelRight) + { + StartedBy = startedBy; + ArgumentNullException.ThrowIfNull(instance); + Instance = new Instance + { + Id = instance.Id ?? throw new InvalidOperationException("Instance associated with job does not have an Id!"), + }; + Description = typeof(JobCode) + .GetField(code.ToString()) + .GetCustomAttributes(false) + .OfType() + .First() + .Description; + JobCode = code; + CancelRight = cancelRight; + CancelRightsType = cancelRightsType; + } + /// public JobResponse ToApi() => new () { Id = Id, + JobCode = JobCode.Value, InstanceId = Instance.Id.Value, StartedAt = StartedAt, StoppedAt = StoppedAt, diff --git a/tests/Tgstation.Server.Api.Tests/Models/TestJobCode.cs b/tests/Tgstation.Server.Api.Tests/Models/TestJobCode.cs new file mode 100644 index 00000000000..0d5a020767c --- /dev/null +++ b/tests/Tgstation.Server.Api.Tests/Models/TestJobCode.cs @@ -0,0 +1,27 @@ +using System; +using System.Linq; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Tgstation.Server.Api.Models.Tests +{ + [TestClass] + public sealed class TestJobCode + { + [TestMethod] + public void TestAllCodesHaveDescription() + { + var jobCodeType = typeof(JobCode); + foreach (var code in Enum.GetValues(typeof(JobCode)).Cast()) + Assert.IsFalse( + String.IsNullOrWhiteSpace( + jobCodeType + .GetField(code.ToString()) + .GetCustomAttributes(false) + .OfType() + .FirstOrDefault() + ?.Description), + $"JobCode {code} is missing a description!"); + } + } +} diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs index c561d7abde4..51e5587c642 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs @@ -22,11 +22,15 @@ public JobsRequiredTest(IJobsClient jobsClient) public async Task WaitForJob(JobResponse originalJob, int timeout, bool? expectFailure, ErrorCode? expectedCode, CancellationToken cancellationToken) { + Assert.IsNotNull(originalJob.Id); + Assert.IsNotNull(originalJob.JobCode); var job = originalJob; do { await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); job = await JobsClient.GetId(job, cancellationToken); + Assert.IsNotNull(job.Id); + Assert.IsNotNull(job.JobCode); --timeout; } while (!job.StoppedAt.HasValue && timeout > 0); @@ -49,11 +53,15 @@ public async Task WaitForJob(JobResponse originalJob, int timeout, protected async Task WaitForJobProgress(JobResponse originalJob, int timeout, CancellationToken cancellationToken) { + Assert.IsNotNull(originalJob.Id); + Assert.IsNotNull(originalJob.JobCode); var job = originalJob; do { await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); job = await JobsClient.GetId(job, cancellationToken); + Assert.IsNotNull(job.Id); + Assert.IsNotNull(job.JobCode); --timeout; } while (!job.Progress.HasValue && job.Stage == null && timeout > 0); @@ -66,6 +74,8 @@ protected async Task WaitForJobProgress(JobResponse originalJob, in protected async Task WaitForJobProgressThenCancel(JobResponse originalJob, int timeout, CancellationToken cancellationToken) { + Assert.IsNotNull(originalJob.Id); + Assert.IsNotNull(originalJob.JobCode); var start = DateTimeOffset.UtcNow; var job = await WaitForJobProgress(originalJob, timeout, cancellationToken); From edcd26e44f62ff02fe0701644885a35f887c72ab Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 23:27:57 -0400 Subject: [PATCH 103/138] Allow GET `/Jobs` request spam again Now that we have a hub as a better alternative, it should be less prevalent. --- .../Controllers/ApiController.cs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index b3136a115f0..ffa4e849600 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -154,15 +154,14 @@ protected override async ValueTask HookExecuteAction(Func e if (ApiHeaders != null) { var isGet = HttpMethods.IsGet(Request.Method); - if (!(isGet && Request.Path.StartsWithSegments(Routes.Jobs, StringComparison.OrdinalIgnoreCase))) - Logger.Log( - isGet - ? LogLevel.Trace - : LogLevel.Debug, - "Starting API request: Version: {clientApiVersion}. {userAgentHeaderName}: {clientUserAgent}", - ApiHeaders.ApiVersion.Semver(), - HeaderNames.UserAgent, - ApiHeaders.RawUserAgent); + Logger.Log( + isGet + ? LogLevel.Trace + : LogLevel.Debug, + "Starting API request: Version: {clientApiVersion}. {userAgentHeaderName}: {clientUserAgent}", + ApiHeaders.ApiVersion.Semver(), + HeaderNames.UserAgent, + ApiHeaders.RawUserAgent); } else if (Request.Headers.TryGetValue(HeaderNames.UserAgent, out var userAgents)) Logger.LogDebug( From ec81fcdf4125595991e071dba792ea4d490128e1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 23:28:09 -0400 Subject: [PATCH 104/138] Remove no-op call --- src/Tgstation.Server.Host/Controllers/HomeController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index b33c5bf77d9..659a379591d 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -246,9 +246,9 @@ public async ValueTask CreateToken(CancellationToken cancellation // trust the system over the database because a user's name can change while still having the same SID systemIdentity = await systemIdentityFactory.CreateSystemIdentity(ApiHeaders.Username, ApiHeaders.Password, cancellationToken); } - catch (NotImplementedException ex) + catch (NotImplementedException) { - RequiresPosixSystemIdentity(ex); + // Intentionally suppressed } using (systemIdentity) From dd2e2644778397fe02a501080149fdfd1644f19f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 23:29:26 -0400 Subject: [PATCH 105/138] Document new authentication pipeline Also fix location of `TgsAuthorizeAttribute` .md documentation. --- .../Controllers/README.md | 1 - src/Tgstation.Server.Host/Security/README.md | 70 ++++++++++++++++++- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/README.md b/src/Tgstation.Server.Host/Controllers/README.md index fa7f1c5a1f1..e1214f91f27 100644 --- a/src/Tgstation.Server.Host/Controllers/README.md +++ b/src/Tgstation.Server.Host/Controllers/README.md @@ -12,4 +12,3 @@ Some notable exceptions: - Returns 401 If an `IAuthenticationContext` could not be created for a request. - [BridgeController](./BridgeController.cs) is a special controller accessible only from localhost and is used to receive bridge request from DreamDaemon - [HomeController](./HomeController.cs) contains the code to initially log in and generate an API token for a user. -- [TgsAuthorizeAttribute](./TgsAuthorizeAttribute.cs) is a special attribute applied to controller methods to define which rights are required to run a verb. diff --git a/src/Tgstation.Server.Host/Security/README.md b/src/Tgstation.Server.Host/Security/README.md index 80f5afaccf6..da4d16464ea 100644 --- a/src/Tgstation.Server.Host/Security/README.md +++ b/src/Tgstation.Server.Host/Security/README.md @@ -1,12 +1,78 @@ # Security Classes - [IAuthenticationContext](./IAuthenticationContext.cs) and [implementation](./AuthenticationContext.cs) is what contains information about an authenticated user for a request. Includes things like the relevant `InstanceUser` and any associated rights. -- [IAuthenticationContextFactory](./IAuthenticationContextFactory.cs) and [implementation](AuthenticationContextFactory.cs) is a factory for `IAuthenticationContext`s. It handles things related to the database for a user's authentication. This includes loading their rights/associated instance user. It will also stop the request if the users token was issued before the last time their password or enabled status was updated. -- [IClaimsInjector](./IClaimsInjector.cs) and [implementation](./ClaimsInjector.cs) is used to associate rights with a request context so that it may properly pass appropriate `TgsAuthorizeAttribute`s. +- [IAuthenticationContextFactory](./IAuthenticationContextFactory.cs) and [implementation](AuthenticationContextFactory.cs) is a factory for `IAuthenticationContext`s. It handles things related to the database for a user's authentication. This includes loading their rights/associated instance user. It will also stop the request if the users token was issued before the last time their password or `Enabled` status was updated. +- [AuthenticationContextClaimsTransformation](./AuthenticationContextClaimsTransformation.cs) is used to associate rights with a request context so that it may properly pass appropriate `TgsAuthorizeAttribute`s. - [ICrytopgraphySuite](./ICrytopgraphySuite.cs) and [implementation](./CrytopgraphySuite.cs) is used to generate secure strings and byte arrays. It also contains the password hashing and validation logic. - [IIdentityCache](./IIdentityCache.cs) and [implementation](./IdentityCache.cs) is used to store `ISystemIdentity`s for the duration of their associated tokens as [IdentityCacheObject](./IdentityCacheObject.cs)s. - [ITokenFactory](./ITokenFactory.cs) and [implementation](./TokenFactory.cs) is used to generate the Json Web Token for a session after a user successfully authenticates. - [ISystemIdentity](./ISystemIdentity.cs)s represent a logon session with the operating system for a given user. It contains a method to run code under the security context of said user. - [ISystemIdentityFactory](./ISystemIdentityFactory.cs) is used to create `ISystemIdentity`s by attempting to log the user in with the OS with a given username and password. +- [TgsAuthorizeAttribute](./TgsAuthorizeAttribute.cs) is a special attribute applied to controller methods to define which rights are required to run a verb. - [OAuth](./OAuth) contains classes related to OAuth 2.0 authentication + +# A Basic Rundown of the Authenticaton Pipeline + +## For the login request (`POST /`) + +1. An attempt to parse the `ApiHeaders` is made. If they were valid. The API version check is performed. If it fails, HTTP 426 with an `ErrorMessageResponse` will be returned. +1. If, for some reason, the user attempts to use a JWT to authenticate this request, steps 2-4 of the non-login pipeline list below are performed. +1. The `ApiController` base class inspects the request. + - At this point, if the `ApiHeaders` (MINUS the `Authorization` header) cannot be properly parsed, HTTP 400 with an `ErrorMessageResponse` is returned. +1. The `HomeController` inspects the request. + 1. If the `ApiHeaders` could not be properly parsed, HTTP 400 (or 406 if the `Accept` header was bad) with an `ErrorMessageResponse` is returned. + - The `WWW-Authenticate` header will be set in this response. + 1. If authentication succeeded using a JWT `Bearer` token, HTTP 400 with an `ErrorMessageResponse` is returned. Refreshing a login using a token is not permitted. + - If the user is using a username/password combo + 1. The username and password combination is tried against the OS authentication system (currently a no-op on Linux). + - If it succeeds, the session is held on to for future reference and the database is queried for a user matching the SID/UID of the login session. + - Otherwise, the database is queried for a user matching the canonicalized username. + - If the user is using an OAuth code: + 1. If the OAuth provider is disabled in the configuration, HTTP 400 with an `ErrorMessageResponse` is returned. + 1. The code is sent to the external provider for validation + - If the provider is GitHub, there's a chance that this could fail due to rate limiting. In this case, HTTP 429 is returned. + - If the provider rejects the OAuth code, HTTP 401 is returned. + 1. The database is queried for a user matching the OAuth provider and external user identifier sent with the OAuth provider's response. + 1. If the query selected above produces no results, HTTP 401 is returned. + 1. For non-OAuth logins, maintenance is performed on the user's DB entry at this point + - For non-OS logins: + 1. The provided password is hashed and checked against the database entry. If it does not match, HTTP 401 will be returned. + - This can potentially cause a change to the DB's stored `PasswordHash` if TGS has updated its dependencies and Microsoft has decided to deprecated the previous hashing method since the user last logged in. + - If this occurs, it invalidates all previous logins for the user. + - For OS logins: + 1. If the `PasswordHash` in the DB isn't null, it is set as such. This invalidates all previous logins for the user. + 1. If the `Name` in the DB does not match the user's OS login, it is updated. + 1. If the user's database entry says they are not enabled, HTTP 403 is returned. + 1. A token is generated from the [ITokenFactory](./ITokenFactory.cs). + 1. For OS logins, the user's login session is cached for the duration of the token's validity plus 15 seconds. + 1. The token is returned as a `TokenResponse` with an HTTP 200 status code. + +## For all other authenticated requests + +1. An attempt to parse the `ApiHeaders` is made. If they were valid. The API version check is performed. If it fails, HTTP 426 with an `ErrorMessageResponse` will be returned. +1. The JWT, if present, is validated. If it is, the scope's [AuthenticationContextFactory](./AuthenticationContextFactory.cs) has `SetTokenNbf` called. If not, HTTP 401 will be returned. + - Inside ASP.NET Core, this initializes the calling user's identity principal and sets the "sub" claim to the TGS user ID parsed out of the JWT. + - We know it's the user ID because we set it up like that in the [TokenFactory](./TokenFactory.cs) +1. The [AuthenticationContextClaimsTransformation](./AuthenticationContextClaimsTransformation.cs) is run (this does not short circuit to responses). + 1. This invokes `IAuthenticationContextFactory.CreateAuthenticationContext` using the "sub" claim from the user's identity and the "nbf" timestamp set earlier (We don't get this from the scope's [IApiHeadersProvider](./IApiHeadersProvider.cs) because there may be other errors preventing the `ApiHeaders` from being parsed). + - At this point, the database lookup using the user ID occurs. This hyrates the scope's [AuthenticationContext](./AuthenticationContext.cs) (which is available at the start of the request, but uninitialized). If the user is a system user, their login session is pulled from the cache. This is also where the instance data for a request is loaded if the user has a valid `InstancePermissionSet` for that instance. The user needs to have a few prerequisites for a valid [IAuthenticationContext](./IAuthenticationContext.cs) to be generated: + - The user with the matching ID must exist in the database. + - The last time the user's password or `Enabled` status changed must be before the "nbf" of their token" + - If the user logged in with an OS login, the session is retrieved from the cache here and added to the scope's [AuthenticationContext](./AuthenticationContext.cs). + 1. If a valid authentication context is returned from the [IAuthenticationContextFactory](./IAuthenticationContextFactory.cs), the [AuthenticationContextClaimsTransformation](./AuthenticationContextClaimsTransformation.cs) uses the context to add claims for each permission bit to the user's identity principal. + - Internally, ASP.NET Core uses this to determine whether or not a request to an endpoint will 403 or not based on the parameters of its [TgsAuthorizeAttribute](./TgsAuthorizeAttribute.cs). +1. The authorization filter is invoked + - For non-SignalR hub requests, this is the [AuthenticationContextAuthorizationFilter](./AuthenticationContextAuthorizationFilter.cs). It does two simple things: + 1. It checks the validity of the scope's [IAuthenticationContext](./IAuthenticationContext.cs). If it is invalid (indicating the user is not authorized either due to not existing (Only possible with a forged and signed JWT) or if their token was outdated compared to the last time their password or `Enabled` status was updated), HTTP 401 will be returned. + 1. It checks the user's `Enabled` status. If the user is disabled, HTTP 403 will be returned. + - For SignalR hub requests, this is the [AuthorizationContextHubFilter](./AuthorizationContextHubFilter.cs). + - If either [IAuthenticationContext](./IAuthenticationContext.cs) is either invalid OR unauthorized, it invokes `IErrorHandlingHub.AbortingConnection` with `ConnectionAbortReason.TokenInvalid` on the client before aborting the connection. +1. The `ApiController` base class inspects the request. + 1. If the `ApiHeaders` could not be properly parsed, HTTP 400 (or 406 if the `Accept` header was bad) with an `ErrorMessageResponse` is returned. + 1. If the request is to an Instance component path: + 1. If there is no valid `Instance` header, HTTP 400 with an `ErrorMessageResponse` is returned. + 1. If the active [IAuthenticationContext](./IAuthenticationContext.cs) has no instance data loaded (indicating the user is not authorized to access said instance), HTTP 403 is returned. + 1. If the instance is offline, HTTP 409 with an `ErrorMessageResponse` is returned. + 1. If the request takes an API model as a parameters and the model included in the request body encountered validation errors, HTTP 400 with an `ErrorMessageResponse` is returned. +1. The request at this point, is considered authorized. Remaining behaviour is left up to each individual route to implement. From 31ac130b45317e4aa48399560a27a299d2a9b58e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 23:31:33 -0400 Subject: [PATCH 106/138] Version bumps - Api - ApiLibrary - ClientLibrary - Core --- build/Version.props | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/Version.props b/build/Version.props index ecaea3d8d97..f63dc186b28 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,12 +3,12 @@ - 5.16.4 + 5.17.0 4.7.1 - 9.12.0 + 9.13.0 7.0.0 - 11.1.2 - 13.0.0 + 12.0.0 + 14.0.0 6.6.2 5.6.2 1.4.0 From 9044eb041d1e4062245de3d6f4543dd617e2a381 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 4 Nov 2023 23:34:19 -0400 Subject: [PATCH 107/138] Properly document `DeploymentActivation` event --- src/Tgstation.Server.Host/Components/Events/EventType.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index 2b9575c8561..ef7571ac833 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -163,7 +163,7 @@ public enum EventType DeploymentCleanup, /// - /// Whenever a deployment is about to be used by the game server. May fire multiple times per deployment. + /// Whenever a deployment is about to be used by the game server. May fire multiple times per deployment. Parameters: Game directory path /// [EventScript("DeploymentActivation")] DeploymentActivation, From cab8ce68f5e560ac0ef39c1c821b7fe2aa8336d4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 08:40:18 -0500 Subject: [PATCH 108/138] Simplify this job type check using new JobCodes --- tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index f78352772d0..ccd158b3724 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -9,6 +9,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Tgstation.Server.Api.Hubs; +using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Request; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Client; @@ -186,9 +187,9 @@ await permedUser.Instances.Update(new InstanceUpdateRequest } else { - var wasMissableJob = job.Description.StartsWith("Reconnect chat bot") - || job.Description.StartsWith("Instance startup watchdog reattach") - || job.Description.StartsWith("Instance startup watchdog launch"); + var wasMissableJob = job.JobCode == JobCode.ReconnectChatBot + || job.JobCode == JobCode.StartupWatchdogLaunch + || job.JobCode == JobCode.StartupWatchdogReattach; Assert.IsTrue(wasMissableJob); ++missableMissedJobs; } From 2c82da46f6a14efd6b1df5bfbc9db8492337d1d3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 09:00:44 -0500 Subject: [PATCH 109/138] Simplify logic flow in JobsHubTests --- .../Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index ccd158b3724..228d9296ce2 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -92,13 +92,12 @@ public Task ReceiveJobUpdate(JobResponse job, CancellationToken cancellationToke public async Task Run(CancellationToken cancellationToken) { - var neverReceiverTcs = new TaskCompletionSource(); var neverReceiver = new ShouldNeverReceiveUpdates() { Callback = job => { if (!permlessIsPermed) - neverReceiverTcs.TrySetException(new Exception($"ShouldNeverReceiveUpdates received an update for job {job.Id}!")); + finishTcs.TrySetException(new Exception($"ShouldNeverReceiveUpdates received an update for job {job.Id}!")); else lock (permlessSeenJobs) permlessSeenJobs.Add(job.Id.Value); @@ -131,13 +130,9 @@ public async Task Run(CancellationToken cancellationToken) return Task.CompletedTask; }; - var completedTask = await Task.WhenAny(finishTcs.Task, neverReceiverTcs.Task); - await completedTask; + await finishTcs.Task; } - neverReceiverTcs.TrySetResult(); - await neverReceiverTcs.Task; - var allInstances = await permedUser.Instances.List(null, cancellationToken); async ValueTask> CheckInstance(InstanceResponse instance) From d3563394fb0e5bda5825b1b575657db321513c3a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 09:01:20 -0500 Subject: [PATCH 110/138] `OperationCanceledException`s are not test errors --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 1f51de5f599..0ef060c2f36 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1621,7 +1621,7 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest Console.WriteLine($"[{DateTimeOffset.UtcNow}] TEST ERROR: {ex.ErrorCode}: {ex.Message}\n{ex.AdditionalServerData}"); throw; } - catch (Exception ex) + catch (Exception ex) when (ex is not OperationCanceledException) { Console.WriteLine($"[{DateTimeOffset.UtcNow}] TEST ERROR: {ex}"); throw; From ed8453f08ebc0513963f6c4733c2e5bb886e450f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 09:38:57 -0500 Subject: [PATCH 111/138] Remove hub abort notifications Because SignalR buffers messages, we can't guarantee these will be delivered before the connection is aborted. We'll have to rely on the client not being pants-on-head. --- .../Hubs/ConnectionAbortReason.cs | 18 ------- .../Hubs/IErrorHandlingHub.cs | 19 ------- src/Tgstation.Server.Api/Hubs/IJobsHub.cs | 2 +- .../Extensions/ServiceCollectionExtensions.cs | 3 +- .../Jobs/JobsHubGroupMapper.cs | 3 +- .../Security/AuthorizationContextHubFilter.cs | 12 ++--- src/Tgstation.Server.Host/Security/README.md | 2 +- .../Utils/SignalR/ComprehensiveHubContext.cs | 33 ++----------- .../Utils/SignalR/ConnectionMappingHub.cs | 5 +- .../SignalR/IConnectionMappedHubContext.cs | 9 ++-- .../Utils/SignalR/IHubConnectionMapper.cs | 5 +- .../Live/Instance/JobsHubTests.cs | 26 ---------- .../Live/RawRequestTests.cs | 49 ++++++------------- 13 files changed, 35 insertions(+), 151 deletions(-) delete mode 100644 src/Tgstation.Server.Api/Hubs/ConnectionAbortReason.cs delete mode 100644 src/Tgstation.Server.Api/Hubs/IErrorHandlingHub.cs diff --git a/src/Tgstation.Server.Api/Hubs/ConnectionAbortReason.cs b/src/Tgstation.Server.Api/Hubs/ConnectionAbortReason.cs deleted file mode 100644 index 05b1cac9b43..00000000000 --- a/src/Tgstation.Server.Api/Hubs/ConnectionAbortReason.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace Tgstation.Server.Api.Hubs -{ - /// - /// The reason an aborts a connection. - /// - public enum ConnectionAbortReason - { - /// - /// The provided token is no longer authenticated or authorized to keep the connection. - /// - TokenInvalid, - - /// - /// The server is restarting. - /// - ServerRestart, - } -} diff --git a/src/Tgstation.Server.Api/Hubs/IErrorHandlingHub.cs b/src/Tgstation.Server.Api/Hubs/IErrorHandlingHub.cs deleted file mode 100644 index 844c621d4f1..00000000000 --- a/src/Tgstation.Server.Api/Hubs/IErrorHandlingHub.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; - -namespace Tgstation.Server.Api.Hubs -{ - /// - /// Hub for handling communication errors. - /// - public interface IErrorHandlingHub - { - /// - /// Called if a hub connection or call is attempted with an invalid or unauthorized token. After calling this, the connection is aborted. - /// - /// The . - /// The for the operation. - /// A representing the running operation. - Task AbortingConnection(ConnectionAbortReason reason, CancellationToken cancellationToken); - } -} diff --git a/src/Tgstation.Server.Api/Hubs/IJobsHub.cs b/src/Tgstation.Server.Api/Hubs/IJobsHub.cs index 01b4aec8bde..4c595362cbf 100644 --- a/src/Tgstation.Server.Api/Hubs/IJobsHub.cs +++ b/src/Tgstation.Server.Api/Hubs/IJobsHub.cs @@ -8,7 +8,7 @@ namespace Tgstation.Server.Api.Hubs /// /// SignalR client methods for receiving s. /// - public interface IJobsHub : IErrorHandlingHub + public interface IJobsHub { /// /// Push a update to the client. diff --git a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs index 6693ca670ef..c90433f1ff0 100644 --- a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs @@ -11,7 +11,6 @@ using Serilog.Configuration; using Serilog.Sinks.Elasticsearch; -using Tgstation.Server.Api.Hubs; using Tgstation.Server.Host.Components.Chat.Providers; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; @@ -230,7 +229,7 @@ public static IServiceCollection SetupLogging( /// The to add the to. public static void AddHub(this IServiceCollection services) where THub : ConnectionMappingHub - where THubMethods : class, IErrorHandlingHub + where THubMethods : class { ArgumentNullException.ThrowIfNull(services); diff --git a/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs b/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs index a05d3283b58..4f46c900147 100644 --- a/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs +++ b/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs @@ -71,7 +71,8 @@ public ValueTask UserDisabled(User user, CancellationToken cancellationToken) throw new InvalidOperationException("user.Id was null!"); logger.LogTrace("UserDisabled"); - return hub.NotifyAndAbortUnauthedConnections(user, cancellationToken); + hub.AbortUnauthedConnections(user); + return ValueTask.CompletedTask; } /// diff --git a/src/Tgstation.Server.Host/Security/AuthorizationContextHubFilter.cs b/src/Tgstation.Server.Host/Security/AuthorizationContextHubFilter.cs index 7820d66494c..38360cab9ba 100644 --- a/src/Tgstation.Server.Host/Security/AuthorizationContextHubFilter.cs +++ b/src/Tgstation.Server.Host/Security/AuthorizationContextHubFilter.cs @@ -5,8 +5,6 @@ using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Logging; -using Tgstation.Server.Api.Hubs; - namespace Tgstation.Server.Host.Security { /// @@ -41,7 +39,7 @@ public AuthorizationContextHubFilter( public async Task OnConnectedAsync(HubLifetimeContext context, Func next) { ArgumentNullException.ThrowIfNull(context); - if (await ValidateAuthenticationContext(context.Hub)) + if (ValidateAuthenticationContext(context.Hub)) await next(context); } @@ -49,7 +47,7 @@ public async Task OnConnectedAsync(HubLifetimeContext context, Func InvokeMethodAsync(HubInvocationContext invocationContext, Func> next) { ArgumentNullException.ThrowIfNull(invocationContext); - if (await ValidateAuthenticationContext(invocationContext.Hub)) + if (ValidateAuthenticationContext(invocationContext.Hub)) return await next(invocationContext); return null; @@ -60,7 +58,7 @@ public async ValueTask InvokeMethodAsync(HubInvocationContext invocation /// /// The current . /// if the hub call should continue, if it shouldn't and has been aborted. - async ValueTask ValidateAuthenticationContext(Hub hub) + bool ValidateAuthenticationContext(Hub hub) { if (!authenticationContext.Valid) logger.LogTrace("The token for connection {connectionId} is no longer authenticated! Aborting...", hub.Context.ConnectionId); @@ -78,10 +76,6 @@ async ValueTask ValidateAuthenticationContext(Hub hub) var callerProperty = clients.GetType().GetProperty(nameof(hub.Clients.Caller)); var caller = callerProperty.GetValue(clients); - if (caller is not IErrorHandlingHub specifiedHub) - throw new InvalidOperationException("This filter only supports IErrorHandlingHubs"); - - await specifiedHub.AbortingConnection(ConnectionAbortReason.TokenInvalid, hub.Context.ConnectionAborted); hub.Context.Abort(); return false; } diff --git a/src/Tgstation.Server.Host/Security/README.md b/src/Tgstation.Server.Host/Security/README.md index da4d16464ea..104db33ea4e 100644 --- a/src/Tgstation.Server.Host/Security/README.md +++ b/src/Tgstation.Server.Host/Security/README.md @@ -67,7 +67,7 @@ 1. It checks the validity of the scope's [IAuthenticationContext](./IAuthenticationContext.cs). If it is invalid (indicating the user is not authorized either due to not existing (Only possible with a forged and signed JWT) or if their token was outdated compared to the last time their password or `Enabled` status was updated), HTTP 401 will be returned. 1. It checks the user's `Enabled` status. If the user is disabled, HTTP 403 will be returned. - For SignalR hub requests, this is the [AuthorizationContextHubFilter](./AuthorizationContextHubFilter.cs). - - If either [IAuthenticationContext](./IAuthenticationContext.cs) is either invalid OR unauthorized, it invokes `IErrorHandlingHub.AbortingConnection` with `ConnectionAbortReason.TokenInvalid` on the client before aborting the connection. + - If either [IAuthenticationContext](./IAuthenticationContext.cs) is either invalid OR unauthorized, it unceremoniously aborts the connection. 1. The `ApiController` base class inspects the request. 1. If the `ApiHeaders` could not be properly parsed, HTTP 400 (or 406 if the `Accept` header was bad) with an `ErrorMessageResponse` is returned. 1. If the request is to an Instance component path: diff --git a/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs b/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs index 411c424f088..9e2b053a656 100644 --- a/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs +++ b/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs @@ -8,9 +8,6 @@ using Microsoft.AspNetCore.SignalR; using Microsoft.Extensions.Logging; -using Tgstation.Server.Api.Hubs; -using Tgstation.Server.Common.Extensions; -using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; @@ -20,10 +17,10 @@ namespace Tgstation.Server.Host.Utils.SignalR /// An implementation of with connection ID mapping. /// /// The the is for. - /// The interface for implementing methods. - sealed class ComprehensiveHubContext : IConnectionMappedHubContext, IHubConnectionMapper, IRestartHandler + /// The for implementing methods. + sealed class ComprehensiveHubContext : IConnectionMappedHubContext, IHubConnectionMapper where THub : ConnectionMappingHub - where THubMethods : class, IErrorHandlingHub + where THubMethods : class { /// public IHubClients Clients => wrappedHubContext.Clients; @@ -53,20 +50,15 @@ sealed class ComprehensiveHubContext : IConnectionMappedHubCo /// Initializes a new instance of the class. /// /// The value of . - /// The to with. /// The value of . public ComprehensiveHubContext( IHubContext wrappedHubContext, - IServerControl serverControl, ILogger> logger) { this.wrappedHubContext = wrappedHubContext ?? throw new ArgumentNullException(nameof(wrappedHubContext)); - ArgumentNullException.ThrowIfNull(serverControl); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); userConnections = new ConcurrentDictionary>(); - - serverControl.RegisterForRestart(this); } /// @@ -123,7 +115,7 @@ public void UserDisconnected(string connectionId) } /// - public ValueTask NotifyAndAbortUnauthedConnections(User user, CancellationToken cancellationToken) + public void AbortUnauthedConnections(User user) { ArgumentNullException.ThrowIfNull(user); logger.LogTrace("NotifyAndAbortUnauthedConnections. UID {userId}", user.Id.Value); @@ -143,23 +135,8 @@ public ValueTask NotifyAndAbortUnauthedConnections(User user, CancellationToken return old; }); - async ValueTask NotifyAndAbortConnection(HubCallerContext context) - { - await Clients - .Client(context.ConnectionId) - .AbortingConnection(ConnectionAbortReason.TokenInvalid, cancellationToken); + foreach (var context in connections) context.Abort(); - } - - return ValueTaskExtensions.WhenAll(connections.Select(NotifyAndAbortConnection)); - } - - /// - public async ValueTask HandleRestart(Version updateVersion, bool handlerMayDelayShutdownWithExtremelyLongRunningTasks, CancellationToken cancellationToken) - { - logger.LogTrace("HandleRestart. {connectionCount} active connections", userConnections.Count); - await Clients.All.AbortingConnection(ConnectionAbortReason.ServerRestart, cancellationToken); - userConnections.Clear(); } } } diff --git a/src/Tgstation.Server.Host/Utils/SignalR/ConnectionMappingHub.cs b/src/Tgstation.Server.Host/Utils/SignalR/ConnectionMappingHub.cs index 5f81010e8b2..05e2eb8742e 100644 --- a/src/Tgstation.Server.Host/Utils/SignalR/ConnectionMappingHub.cs +++ b/src/Tgstation.Server.Host/Utils/SignalR/ConnectionMappingHub.cs @@ -4,7 +4,6 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.SignalR; -using Tgstation.Server.Api.Hubs; using Tgstation.Server.Host.Security; namespace Tgstation.Server.Host.Utils.SignalR @@ -13,11 +12,11 @@ namespace Tgstation.Server.Host.Utils.SignalR /// Base for s that want to map their connection IDs to s. /// /// The child inheriting from the . - /// The interface for implementing methods. + /// The for implementing methods. [TgsAuthorize] abstract class ConnectionMappingHub : Hub where TChildHub : ConnectionMappingHub - where THubMethods : class, IErrorHandlingHub + where THubMethods : class { /// /// The used to map connections. diff --git a/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs b/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs index d44ad96108d..38778817029 100644 --- a/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs +++ b/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs @@ -5,7 +5,6 @@ using Microsoft.AspNetCore.SignalR; -using Tgstation.Server.Api.Hubs; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; @@ -18,7 +17,7 @@ namespace Tgstation.Server.Host.Utils.SignalR /// The interface for implementing methods. interface IConnectionMappedHubContext : IHubContext where THub : Hub - where THubMethods : class, IErrorHandlingHub + where THubMethods : class { /// /// Called when a user connects. Should return an of hub group names the given belongs in. @@ -33,11 +32,9 @@ interface IConnectionMappedHubContext : IHubContext UserConnectionIds(User user); /// - /// Calls with on and aborts the connections associated with the given . + /// Aborts the connections associated with the given . /// /// The to abort the connections of. - /// The for the operation. - /// A representing the running operation. - ValueTask NotifyAndAbortUnauthedConnections(User user, CancellationToken cancellationToken); + void AbortUnauthedConnections(User user); } } diff --git a/src/Tgstation.Server.Host/Utils/SignalR/IHubConnectionMapper.cs b/src/Tgstation.Server.Host/Utils/SignalR/IHubConnectionMapper.cs index f941ac28fe0..4521c87fcce 100644 --- a/src/Tgstation.Server.Host/Utils/SignalR/IHubConnectionMapper.cs +++ b/src/Tgstation.Server.Host/Utils/SignalR/IHubConnectionMapper.cs @@ -3,7 +3,6 @@ using Microsoft.AspNetCore.SignalR; -using Tgstation.Server.Api.Hubs; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; @@ -13,10 +12,10 @@ namespace Tgstation.Server.Host.Utils.SignalR /// Handles mapping connection IDs to s for a given . /// /// The whose connections are being mapped. - /// The interface for implementing methods. + /// The for implementing methods. interface IHubConnectionMapper where THub : ConnectionMappingHub - where THubMethods : class, IErrorHandlingHub + where THubMethods : class { /// /// To be called when a hub connection is made. diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index 228d9296ce2..bf683503c11 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -19,8 +19,6 @@ namespace Tgstation.Server.Tests.Live.Instance { sealed class JobsHubTests : IJobsHub { - const int ActiveConnections = 2; - readonly IServerClient permedUser; readonly IServerClient permlessUser; @@ -31,7 +29,6 @@ sealed class JobsHubTests : IJobsHub readonly HashSet permlessSeenJobs; HubConnection conn1, conn2; - int expectedReboots; bool permlessIsPermed; long? permlessPsId; @@ -78,10 +75,6 @@ public Task ReceiveJobUpdate(JobResponse job, CancellationToken cancellationToke class ShouldNeverReceiveUpdates : IJobsHub { public Action Callback { get; set; } - public Func Error { get; set; } - - public Task AbortingConnection(ConnectionAbortReason reason, CancellationToken cancellationToken) - => Error(reason, cancellationToken); public Task ReceiveJobUpdate(JobResponse job, CancellationToken cancellationToken) { @@ -102,7 +95,6 @@ public async Task Run(CancellationToken cancellationToken) lock (permlessSeenJobs) permlessSeenJobs.Add(job.Id.Value); }, - Error = AbortingConnection, }; await using (conn1 = (HubConnection)await permedUser.SubscribeToJobUpdates( @@ -208,19 +200,16 @@ await permedUser.Instances.Update(new InstanceUpdateRequest Assert.AreEqual(HubConnectionState.Connected, conn3.State); await permlessUser.DisposeAsync(); await permedUser.DisposeAsync(); - Assert.AreEqual(0, expectedReboots); } public void ExpectShutdown() { - Assert.AreEqual(0, Interlocked.Exchange(ref expectedReboots, ActiveConnections)); Assert.AreEqual(HubConnectionState.Connected, conn1.State); Assert.AreEqual(HubConnectionState.Connected, conn2.State); } public async ValueTask WaitForReconnect(CancellationToken cancellationToken) { - Assert.AreEqual(0, expectedReboots); await Task.WhenAll(conn1.StopAsync(cancellationToken), conn2.StopAsync(cancellationToken)); Assert.AreEqual(HubConnectionState.Disconnected, conn1.State); @@ -270,20 +259,5 @@ await ic.PermissionSets.Delete(new InstancePermissionSetRequest } public void CompleteNow() => finishTcs.TrySetResult(); - - public Task AbortingConnection(ConnectionAbortReason reason, CancellationToken cancellationToken) - { - try - { - Assert.AreEqual(ConnectionAbortReason.ServerRestart, reason); - var remaining = Interlocked.Decrement(ref expectedReboots); - Assert.IsTrue(remaining >= 0); - } - catch (Exception ex) - { - finishTcs.TrySetException(ex); - } - return Task.CompletedTask; - } } } diff --git a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs index bb028397b48..7e4a1010015 100644 --- a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs +++ b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs @@ -359,10 +359,6 @@ static async Task RegressionTestForLeakedPasswordHashesBug(IServerClient serverC class FuncProxiedJobsHub : IJobsHub { public Func ProxyFunc { get; set; } - public Func ErrorFunc { get; set; } - - public Task AbortingConnection(ConnectionAbortReason reason, CancellationToken cancellationToken) - => ErrorFunc(reason); public Task ReceiveJobUpdate(JobResponse job, CancellationToken cancellationToken) => ProxyFunc(job, cancellationToken); @@ -402,13 +398,6 @@ static async Task TestSignalRUsage(IServerClientFactory serverClientFactory, ISe }); var proxy = new FuncProxiedJobsHub(); - var errorTcs = new TaskCompletionSource(); - proxy.ErrorFunc = reason => - { - errorTcs.SetException(new Exception($"Aborted: {reason}")); - return Task.CompletedTask; - }; - HubConnection hubConnection; HardFailLoggerProvider.BlockFails = true; try @@ -431,8 +420,6 @@ static async Task TestSignalRUsage(IServerClientFactory serverClientFactory, ISe Assert.AreEqual(HubConnectionState.Disconnected, hubConnection.State); - Assert.IsFalse(errorTcs.Task.IsCompleted); - var createRequest = new UserCreateRequest { Enabled = true, @@ -441,33 +428,27 @@ static async Task TestSignalRUsage(IServerClientFactory serverClientFactory, ISe }; var testUser = await serverClient.Users.Create(createRequest, cancellationToken); - await using (var testUserClient = await serverClientFactory.CreateFromLogin(serverClient.Url, createRequest.Name, createRequest.Password, cancellationToken: cancellationToken)) - { - errorTcs = new TaskCompletionSource(); - await using var testUserConn1 = await testUserClient.SubscribeToJobUpdates(proxy, cancellationToken: cancellationToken); + await using var testUserClient = await serverClientFactory.CreateFromLogin(serverClient.Url, createRequest.Name, createRequest.Password, cancellationToken: cancellationToken); + await using var testUserConn1 = (HubConnection)await testUserClient.SubscribeToJobUpdates(proxy, cancellationToken: cancellationToken); - Assert.IsFalse(errorTcs.Task.IsCompleted); + await serverClient.Users.Update(new UserUpdateRequest + { + Id = testUser.Id, + Enabled = false, + }, cancellationToken); - await serverClient.Users.Update(new UserUpdateRequest - { - Id = testUser.Id, - Enabled = false, - }, cancellationToken); + // need a second here + for (var i = 0; i < 10 && testUserConn1.State == HubConnectionState.Connected; ++i) + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); - // need a second here - for (var i = 0; i < 10 && !errorTcs.Task.IsCompleted; ++i) - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + Assert.AreNotEqual(HubConnectionState.Connected, testUserConn1.State); - Assert.IsTrue(errorTcs.Task.IsCompleted); + await using var testUserConn2 = (HubConnection)await testUserClient.SubscribeToJobUpdates(proxy, cancellationToken: cancellationToken); - errorTcs = new TaskCompletionSource(); - await using var testUserConn2 = await testUserClient.SubscribeToJobUpdates(proxy, cancellationToken: cancellationToken); - for (var i = 0; i < 10 && !errorTcs.Task.IsCompleted; ++i) - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); - } + for (var i = 0; i < 10 && testUserConn2.State == HubConnectionState.Connected; ++i) + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); - Assert.IsTrue(errorTcs.Task.IsCompleted); - await Assert.ThrowsExceptionAsync(() => errorTcs.Task); + Assert.AreNotEqual(HubConnectionState.Connected, testUserConn2.State); } finally { From fa13869ea269409d122e2c24aad2e1ce882efdb4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 11:23:47 -0500 Subject: [PATCH 112/138] Ensure that SignalR works with the webpanel - Fix issue with CORS preventing static file browsing. - AllowCredentials in CORS. Switch from `AllowAnyOrigin` to a wildcard matching `Func` to bypass the CORS specification that says you can't do that. - Add workaround for legacy SignalR `access_token` query string. - Get token `nbf` from claims rather than through the composition root. --- src/Tgstation.Server.Host/Core/Application.cs | 55 +++++++++++-------- ...thenticationContextClaimsTransformation.cs | 20 ++++++- .../Security/AuthenticationContextFactory.cs | 24 +------- .../Security/IAuthenticationContextFactory.cs | 6 +- 4 files changed, 58 insertions(+), 47 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index b97a21600f0..922011a9ac1 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -473,6 +473,24 @@ public void Configure( logger.LogTrace("Swagger API generation enabled"); } + // spa loading if necessary + if (controlPanelConfiguration.Enable) + { + logger.LogInformation("Web control panel enabled."); + applicationBuilder.UseFileServer(new FileServerOptions + { + RequestPath = ControlPanelController.ControlPanelRoute, + EnableDefaultFiles = true, + EnableDirectoryBrowsing = false, + }); + } + else +#if NO_WEBPANEL + logger.LogDebug("Web control panel was not included in TGS build!"); +#else + logger.LogTrace("Web control panel disabled!"); +#endif + // Enable endpoint routing applicationBuilder.UseRouting(); @@ -481,7 +499,7 @@ public void Configure( if (controlPanelConfiguration.AllowAnyOrigin) { logger.LogTrace("Access-Control-Allow-Origin: *"); - corsBuilder = builder => builder.AllowAnyOrigin(); + corsBuilder = builder => builder.SetIsOriginAllowed(_ => true); } else if (controlPanelConfiguration.AllowedOrigins?.Count > 0) { @@ -495,29 +513,12 @@ public void Configure( builder .AllowAnyHeader() .AllowAnyMethod() + .AllowCredentials() .SetPreflightMaxAge(TimeSpan.FromDays(1)); originalBuilder?.Invoke(builder); }; applicationBuilder.UseCors(corsBuilder); - // spa loading if necessary - if (controlPanelConfiguration.Enable) - { - logger.LogInformation("Web control panel enabled."); - applicationBuilder.UseFileServer(new FileServerOptions - { - RequestPath = ControlPanelController.ControlPanelRoute, - EnableDefaultFiles = true, - EnableDirectoryBrowsing = false, - }); - } - else -#if NO_WEBPANEL - logger.LogDebug("Web control panel was not included in TGS build!"); -#else - logger.LogTrace("Web control panel disabled!"); -#endif - // validate the API version applicationBuilder.UseApiCompatibility(); @@ -596,10 +597,20 @@ void ConfigureAuthenticationPipeline(IServiceCollection services) jwtBearerOptions.TokenValidationParameters = tokenFactory?.ValidationParameters ?? throw new InvalidOperationException("tokenFactory not initialized!"); jwtBearerOptions.Events = new JwtBearerEvents { - OnTokenValidated = tokenValidatedContext => + OnMessageReceived = context => { - var acf = tokenValidatedContext.HttpContext.RequestServices.GetRequiredService(); - acf.SetTokenNbf(tokenValidatedContext.SecurityToken.ValidFrom); + if (String.IsNullOrWhiteSpace(context.Token)) + { + var accessToken = context.Request.Query["access_token"]; + var path = context.HttpContext.Request.Path; + + if (!String.IsNullOrWhiteSpace(accessToken) && + path.StartsWithSegments(Routes.HubsRoot, StringComparison.OrdinalIgnoreCase)) + { + context.Token = accessToken; + } + } + return Task.CompletedTask; }, }; diff --git a/src/Tgstation.Server.Host/Security/AuthenticationContextClaimsTransformation.cs b/src/Tgstation.Server.Host/Security/AuthenticationContextClaimsTransformation.cs index 6cd5f303053..4f02bb0b0d7 100644 --- a/src/Tgstation.Server.Host/Security/AuthenticationContextClaimsTransformation.cs +++ b/src/Tgstation.Server.Host/Security/AuthenticationContextClaimsTransformation.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Authentication; +using Microsoft.IdentityModel.Tokens; using Tgstation.Server.Api; using Tgstation.Server.Api.Rights; @@ -48,7 +49,7 @@ public async Task TransformAsync(ClaimsPrincipal principal) var userIdClaim = principal.FindFirst(JwtRegisteredClaimNames.Sub); if (userIdClaim == default) - throw new InvalidOperationException("Missing required claim!"); + throw new InvalidOperationException($"Missing '{JwtRegisteredClaimNames.Sub}' claim!"); long userId; try @@ -60,9 +61,26 @@ public async Task TransformAsync(ClaimsPrincipal principal) throw new InvalidOperationException("Failed to parse user ID!", e); } + var nbfClaim = principal.FindFirst(JwtRegisteredClaimNames.Nbf); + if (nbfClaim == default) + throw new InvalidOperationException($"Missing '{JwtRegisteredClaimNames.Nbf}' claim!"); + + DateTimeOffset nbf; + try + { + nbf = new DateTimeOffset( + EpochTime.DateTime( + Int64.Parse(nbfClaim.Value, CultureInfo.InvariantCulture))); + } + catch (Exception ex) + { + throw new InvalidOperationException("Failed to parse nbf!", ex); + } + var authenticationContext = await authenticationContextFactory.CreateAuthenticationContext( userId, apiHeaders?.InstanceId, + nbf, CancellationToken.None); // DCT: None available if (authenticationContext.Valid) diff --git a/src/Tgstation.Server.Host/Security/AuthenticationContextFactory.cs b/src/Tgstation.Server.Host/Security/AuthenticationContextFactory.cs index 3b5035e69c2..f39daaa9897 100644 --- a/src/Tgstation.Server.Host/Security/AuthenticationContextFactory.cs +++ b/src/Tgstation.Server.Host/Security/AuthenticationContextFactory.cs @@ -46,11 +46,6 @@ sealed class AuthenticationContextFactory : IAuthenticationContextFactory, IDisp /// readonly AuthenticationContext currentAuthenticationContext; - /// - /// The the request's token must be valid after. - /// - DateTimeOffset? validAfter; - /// /// 1 if was initialized, 0 otherwise. /// @@ -80,27 +75,12 @@ public AuthenticationContextFactory( /// public void Dispose() => currentAuthenticationContext.Dispose(); - /// - /// Populate with a given . - /// - /// The an issued token is not valid before. - public void SetTokenNbf(DateTimeOffset tokenNbf) - { - if (validAfter.HasValue) - throw new InvalidOperationException("SetTokenNbf called multiple times!"); - - validAfter = tokenNbf; - } - /// - public async ValueTask CreateAuthenticationContext(long userId, long? instanceId, CancellationToken cancellationToken) + public async ValueTask CreateAuthenticationContext(long userId, long? instanceId, DateTimeOffset notBefore, CancellationToken cancellationToken) { if (Interlocked.Exchange(ref initialized, 1) != 0) throw new InvalidOperationException("Authentication context has already been loaded"); - if (!validAfter.HasValue) - throw new InvalidOperationException("SetTokenNbf has not been called!"); - var user = await databaseContext .Users .AsQueryable() @@ -122,7 +102,7 @@ public async ValueTask CreateAuthenticationContext(long systemIdentity = identityCache.LoadCachedIdentity(user); else { - if (user.LastPasswordUpdate.HasValue && user.LastPasswordUpdate > validAfter.Value) + if (user.LastPasswordUpdate.HasValue && user.LastPasswordUpdate > notBefore) { logger.LogDebug("Rejecting token for user {userId} created before last password update: {lastPasswordUpdate}", userId, user.LastPasswordUpdate.Value); return currentAuthenticationContext; diff --git a/src/Tgstation.Server.Host/Security/IAuthenticationContextFactory.cs b/src/Tgstation.Server.Host/Security/IAuthenticationContextFactory.cs index 02337005850..cb70a50b18e 100644 --- a/src/Tgstation.Server.Host/Security/IAuthenticationContextFactory.cs +++ b/src/Tgstation.Server.Host/Security/IAuthenticationContextFactory.cs @@ -1,4 +1,5 @@ -using System.Threading; +using System; +using System.Threading; using System.Threading.Tasks; namespace Tgstation.Server.Host.Security @@ -13,8 +14,9 @@ public interface IAuthenticationContextFactory /// /// The of the . /// The of the for the operation. + /// The the login must not be from before. /// The for the operation. /// A resulting in the created . - ValueTask CreateAuthenticationContext(long userId, long? instanceId, CancellationToken cancellationToken); + ValueTask CreateAuthenticationContext(long userId, long? instanceId, DateTimeOffset notBefore, CancellationToken cancellationToken); } } From b79d02c77ab98670ca15ba12cb813b8fe9791537 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 11:56:43 -0500 Subject: [PATCH 113/138] Push an update of all active jobs to all clients when a new one connects I know this is annoying in that we only need to target the connected client, but it's so much simpler this way. How big of an update can it even be if it's only including active jobs? --- src/Tgstation.Server.Host/Core/Application.cs | 4 +- .../Jobs/IJobsHubUpdater.cs | 13 ++ src/Tgstation.Server.Host/Jobs/JobService.cs | 142 +++++++++++------- .../Jobs/JobsHubGroupMapper.cs | 27 +++- .../Utils/SignalR/ComprehensiveHubContext.cs | 16 +- .../SignalR/IConnectionMappedHubContext.cs | 9 +- 6 files changed, 144 insertions(+), 67 deletions(-) create mode 100644 src/Tgstation.Server.Host/Jobs/IJobsHubUpdater.cs diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 922011a9ac1..a331feeb9f3 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -400,7 +400,9 @@ void AddTypedContext() services.AddGitHub(); // configure root services - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(x => x.GetRequiredService()); services.AddSingleton(); diff --git a/src/Tgstation.Server.Host/Jobs/IJobsHubUpdater.cs b/src/Tgstation.Server.Host/Jobs/IJobsHubUpdater.cs new file mode 100644 index 00000000000..ed9080ada72 --- /dev/null +++ b/src/Tgstation.Server.Host/Jobs/IJobsHubUpdater.cs @@ -0,0 +1,13 @@ +namespace Tgstation.Server.Host.Jobs +{ + /// + /// Allows manually triggering jobs hub updates. + /// + interface IJobsHubUpdater + { + /// + /// Queue a message to be sent to all clients with the current state of active jobs. + /// + void QueueActiveJobUpdates(); + } +} diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index b1aa9c1e0a7..7793eb582bd 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -24,7 +24,7 @@ namespace Tgstation.Server.Host.Jobs { /// - sealed class JobService : IJobService, IDisposable + sealed class JobService : IJobService, IJobsHubUpdater, IDisposable { /// /// The maximum rate at which hub clients can receive updates. @@ -56,6 +56,11 @@ sealed class JobService : IJobService, IDisposable /// readonly Dictionary jobs; + /// + /// of running s to s that will push immediate updates. + /// + readonly Dictionary hubUpdateActions; + /// /// to delay starting jobs until the server is ready. /// @@ -95,6 +100,7 @@ public JobService( this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); jobs = new Dictionary(); + hubUpdateActions = new Dictionary(); activationTcs = new TaskCompletionSource(); synchronizationLock = new object(); addCancelLock = new object(); @@ -305,6 +311,14 @@ public void Activate(IInstanceCoreProvider instanceCoreProvider) activationTcs.SetResult(instanceCoreProvider); } + /// + public void QueueActiveJobUpdates() + { + lock (hubUpdateActions) + foreach (var action in hubUpdateActions.Values) + action(); + } + /// /// Runner for s. /// @@ -325,39 +339,51 @@ async Task RunJob(Job job, JobEntrypoint operation, CancellationToken canc var result = false; Stopwatch stopwatch = null; - void QueueHubUpdate(JobResponse update) + void QueueHubUpdate(JobResponse update, bool final) { - var currentUpdatesTask = hubUpdatesTask; - async Task ChainHubUpdate() + void NextUpdate() { - await currentUpdatesTask; + var currentUpdatesTask = hubUpdatesTask; + async Task ChainHubUpdate() + { + await currentUpdatesTask; - // DCT: Cancellation token is for job, operation should always run - await hub - .Clients - .Group(JobsHub.HubGroupName(job)) - .ReceiveJobUpdate(update, CancellationToken.None); - } + // DCT: Cancellation token is for job, operation should always run + await hub + .Clients + .Group(JobsHub.HubGroupName(job)) + .ReceiveJobUpdate(update, CancellationToken.None); + } - Stopwatch enteredLock = null; - try - { - if (stopwatch != null) + Stopwatch enteredLock = null; + try { - Monitor.Enter(stopwatch); - enteredLock = stopwatch; - if (stopwatch.ElapsedMilliseconds * MaxHubUpdatesPerSecond < 1) - return; // don't spam client - } + if (stopwatch != null) + { + Monitor.Enter(stopwatch); + enteredLock = stopwatch; + if (stopwatch.ElapsedMilliseconds * MaxHubUpdatesPerSecond < 1) + return; // don't spam client + } - hubUpdatesTask = ChainHubUpdate(); - stopwatch = Stopwatch.StartNew(); - } - finally - { - if (enteredLock != null) - Monitor.Exit(enteredLock); + hubUpdatesTask = ChainHubUpdate(); + stopwatch = Stopwatch.StartNew(); + } + finally + { + if (enteredLock != null) + Monitor.Exit(enteredLock); + } } + + var jobId = update.Id.Value; + lock (hubUpdateActions) + if (final) + hubUpdateActions.Remove(jobId); + else + hubUpdateActions[jobId] = NextUpdate; + + NextUpdate(); } try @@ -382,12 +408,12 @@ void UpdateProgress(string stage, double? progress) var updatedJob = job.ToApi(); updatedJob.Stage = stage; updatedJob.Progress = newProgress; - QueueHubUpdate(updatedJob); + QueueHubUpdate(updatedJob, false); } } var instanceCoreProvider = await activationTcs.Task.WaitAsync(cancellationToken); - QueueHubUpdate(job.ToApi()); + QueueHubUpdate(job.ToApi(), false); logger.LogTrace("Starting job..."); await operation( @@ -420,35 +446,45 @@ await operation( LogException(e); } - await databaseContextFactory.UseContext(async databaseContext => + try { - var attachedJob = new Job(job.Id.Value); + await databaseContextFactory.UseContext(async databaseContext => + { + var attachedJob = new Job(job.Id.Value); - databaseContext.Jobs.Attach(attachedJob); - attachedJob.StoppedAt = DateTimeOffset.UtcNow; - attachedJob.ExceptionDetails = job.ExceptionDetails; - attachedJob.ErrorCode = job.ErrorCode; - attachedJob.Cancelled = job.Cancelled; + databaseContext.Jobs.Attach(attachedJob); + attachedJob.StoppedAt = DateTimeOffset.UtcNow; + attachedJob.ExceptionDetails = job.ExceptionDetails; + attachedJob.ErrorCode = job.ErrorCode; + attachedJob.Cancelled = job.Cancelled; - // DCT: Cancellation token is for job, operation should always run - await databaseContext.Save(CancellationToken.None); - }); + // DCT: Cancellation token is for job, operation should always run + await databaseContext.Save(CancellationToken.None); + }); - // Resetting the context here because I CBA to worry if the cache is being used - await databaseContextFactory.UseContext(async databaseContext => + // Resetting the context here because I CBA to worry if the cache is being used + await databaseContextFactory.UseContext(async databaseContext => + { + // Cancellation might be set in another async context, forced to reload here for the final hub update + // DCT: Cancellation token is for job, operation should always run + var finalJob = await databaseContext + .Jobs + .AsQueryable() + .Include(x => x.Instance) + .Include(x => x.StartedBy) + .Include(x => x.CancelledBy) + .Where(dbJob => dbJob.Id == job.Id.Value) + .FirstAsync(CancellationToken.None); + QueueHubUpdate(finalJob.ToApi(), true); + }); + } + catch { - // Cancellation might be set in another async context, forced to reload here for the final hub update - // DCT: Cancellation token is for job, operation should always run - var finalJob = await databaseContext - .Jobs - .AsQueryable() - .Include(x => x.Instance) - .Include(x => x.StartedBy) - .Include(x => x.CancelledBy) - .Where(dbJob => dbJob.Id == job.Id.Value) - .FirstAsync(CancellationToken.None); - QueueHubUpdate(finalJob.ToApi()); - }); + lock (hubUpdateActions) + hubUpdateActions.Remove(job.Id.Value); + + throw; + } try { diff --git a/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs b/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs index 4f46c900147..77ccf99b0b5 100644 --- a/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs +++ b/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs @@ -27,10 +27,15 @@ sealed class JobsHubGroupMapper : IPermissionsUpdateNotifyee readonly IConnectionMappedHubContext hub; /// - /// The for the . + /// The for the . /// readonly IDatabaseContextFactory databaseContextFactory; + /// + /// The for the . + /// + readonly IJobsHubUpdater jobsHubUpdater; + /// /// The for the . /// @@ -41,11 +46,17 @@ sealed class JobsHubGroupMapper : IPermissionsUpdateNotifyee /// /// The value of . /// The value of . + /// The value of . /// The value of . - public JobsHubGroupMapper(IConnectionMappedHubContext hub, IDatabaseContextFactory databaseContextFactory, ILogger logger) + public JobsHubGroupMapper( + IConnectionMappedHubContext hub, + IDatabaseContextFactory databaseContextFactory, + IJobsHubUpdater jobsHubUpdater, + ILogger logger) { this.hub = hub ?? throw new ArgumentNullException(nameof(hub)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); + this.jobsHubUpdater = jobsHubUpdater ?? throw new ArgumentNullException(nameof(jobsHubUpdater)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); hub.OnConnectionMapGroups += MapConnectionGroups; @@ -89,9 +100,13 @@ public ValueTask InstancePermissionSetDeleted(PermissionSet permissionSet, Cance /// Implementation of . /// /// The to map the groups for. + /// The taking the mapped group names as an of resulting in a to be ed. /// The for the operation. /// A resulting in an of the group names the user belongs in. - async ValueTask> MapConnectionGroups(IAuthenticationContext authenticationContext, CancellationToken cancellationToken) + async ValueTask MapConnectionGroups( + IAuthenticationContext authenticationContext, + Func, Task> mappingFunc, + CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(authenticationContext); @@ -105,7 +120,11 @@ await databaseContextFactory.UseContext( .Select(ips => ips.Id) .ToListAsync(cancellationToken)); - return permedInstanceIds.Select(JobsHub.HubGroupName); + await mappingFunc( + permedInstanceIds.Select( + JobsHub.HubGroupName)); + + jobsHubUpdater.QueueActiveJobUpdates(); } /// diff --git a/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs b/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs index 9e2b053a656..79999080529 100644 --- a/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs +++ b/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs @@ -44,7 +44,7 @@ sealed class ComprehensiveHubContext : IConnectionMappedHubCo readonly ConcurrentDictionary> userConnections; /// - public event Func>> OnConnectionMapGroups; + public event Func, Task>, CancellationToken, ValueTask> OnConnectionMapGroups; /// /// Initializes a new instance of the class. @@ -71,7 +71,7 @@ public List UserConnectionIds(User user) } /// - public async ValueTask UserConnected(IAuthenticationContext authenticationContext, THub hub, CancellationToken cancellationToken) + public ValueTask UserConnected(IAuthenticationContext authenticationContext, THub hub, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(authenticationContext); ArgumentNullException.ThrowIfNull(hub); @@ -83,7 +83,12 @@ public async ValueTask UserConnected(IAuthenticationContext authenticationContex userId, context.ConnectionId); - var mappedGroupsTask = OnConnectionMapGroups(authenticationContext, cancellationToken); + var mappingTask = OnConnectionMapGroups( + authenticationContext, + mappedGroups => Task.WhenAll( + mappedGroups.Select( + group => hub.Groups.AddToGroupAsync(context.ConnectionId, group, cancellationToken))), + cancellationToken); userConnections.AddOrUpdate( userId, _ => new Dictionary @@ -98,10 +103,7 @@ public async ValueTask UserConnected(IAuthenticationContext authenticationContex return old; }); - var mappedGroups = await mappedGroupsTask; - await Task.WhenAll( - mappedGroups.Select( - group => hub.Groups.AddToGroupAsync(context.ConnectionId, group, cancellationToken))); + return mappingTask; } /// diff --git a/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs b/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs index 38778817029..101e8e599e6 100644 --- a/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs +++ b/src/Tgstation.Server.Host/Utils/SignalR/IConnectionMappedHubContext.cs @@ -20,9 +20,14 @@ interface IConnectionMappedHubContext : IHubContext - /// Called when a user connects. Should return an of hub group names the given belongs in. + /// Called when a user connects. + /// Parameters: + /// - The of the authenticated user. + /// - An accepting an of the group names the user should have and returning a that should be ed. + /// - The for the operation. + /// Returns: A representing the running operation. /// - event Func>> OnConnectionMapGroups; + event Func, Task>, CancellationToken, ValueTask> OnConnectionMapGroups; /// /// Gets a of current connection IDs for a given . From 337eb15a7fbc04492e9d47d3e02e3cc14efc24a0 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 12:12:15 -0500 Subject: [PATCH 114/138] User connection updates should bypass the jobs hub message rate limiter --- src/Tgstation.Server.Host/Jobs/JobService.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index 7793eb582bd..41a68e56bff 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -341,7 +341,7 @@ async Task RunJob(Job job, JobEntrypoint operation, CancellationToken canc Stopwatch stopwatch = null; void QueueHubUpdate(JobResponse update, bool final) { - void NextUpdate() + void NextUpdate(bool bypassRate) { var currentUpdatesTask = hubUpdatesTask; async Task ChainHubUpdate() @@ -358,7 +358,7 @@ await hub Stopwatch enteredLock = null; try { - if (stopwatch != null) + if (!bypassRate && stopwatch != null) { Monitor.Enter(stopwatch); enteredLock = stopwatch; @@ -381,9 +381,9 @@ await hub if (final) hubUpdateActions.Remove(jobId); else - hubUpdateActions[jobId] = NextUpdate; + hubUpdateActions[jobId] = () => NextUpdate(true); - NextUpdate(); + NextUpdate(false); } try From 32d1f0ae5e4f5c508e5fc78f40a9f8c423d3267a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 13:13:01 -0500 Subject: [PATCH 115/138] Add FAQ link for DMAPI validation failures --- src/Tgstation.Server.Api/Models/ErrorCode.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Api/Models/ErrorCode.cs b/src/Tgstation.Server.Api/Models/ErrorCode.cs index e10d4577c84..ae2dd3341ca 100644 --- a/src/Tgstation.Server.Api/Models/ErrorCode.cs +++ b/src/Tgstation.Server.Api/Models/ErrorCode.cs @@ -326,7 +326,7 @@ public enum ErrorCode : uint /// /// The DMAPI never validated itself /// - [Description("DreamDaemon did not validate the DMAPI! This can occur if your world is encountering runtime errors during startup.")] + [Description("DMAPI validation failed! See FAQ at https://github.com/tgstation/tgstation-server/discussions/1695")] DreamMakerNeverValidated, /// From 399f19936af1ac43540882ee849d041e11d25f9e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 16:35:45 -0500 Subject: [PATCH 116/138] Fix grammar in auth docs --- src/Tgstation.Server.Host/Security/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Security/README.md b/src/Tgstation.Server.Host/Security/README.md index 104db33ea4e..852af14d879 100644 --- a/src/Tgstation.Server.Host/Security/README.md +++ b/src/Tgstation.Server.Host/Security/README.md @@ -16,7 +16,7 @@ ## For the login request (`POST /`) -1. An attempt to parse the `ApiHeaders` is made. If they were valid. The API version check is performed. If it fails, HTTP 426 with an `ErrorMessageResponse` will be returned. +1. An attempt to parse the `ApiHeaders` is made. If they were valid, the API version check is performed. If it fails, HTTP 426 with an `ErrorMessageResponse` will be returned. 1. If, for some reason, the user attempts to use a JWT to authenticate this request, steps 2-4 of the non-login pipeline list below are performed. 1. The `ApiController` base class inspects the request. - At this point, if the `ApiHeaders` (MINUS the `Authorization` header) cannot be properly parsed, HTTP 400 with an `ErrorMessageResponse` is returned. From 3032955d2519fc54ab51aaeae8f618b463218c1d Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 16:41:42 -0500 Subject: [PATCH 117/138] Make this indentation make sense Also add a missing colon --- src/Tgstation.Server.Host/Security/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Security/README.md b/src/Tgstation.Server.Host/Security/README.md index 852af14d879..c12dd0f587d 100644 --- a/src/Tgstation.Server.Host/Security/README.md +++ b/src/Tgstation.Server.Host/Security/README.md @@ -24,7 +24,8 @@ 1. If the `ApiHeaders` could not be properly parsed, HTTP 400 (or 406 if the `Accept` header was bad) with an `ErrorMessageResponse` is returned. - The `WWW-Authenticate` header will be set in this response. 1. If authentication succeeded using a JWT `Bearer` token, HTTP 400 with an `ErrorMessageResponse` is returned. Refreshing a login using a token is not permitted. - - If the user is using a username/password combo + 1. At this point, the path diverges based on the credential type. + - If the user is using a username/password combo: 1. The username and password combination is tried against the OS authentication system (currently a no-op on Linux). - If it succeeds, the session is held on to for future reference and the database is queried for a user matching the SID/UID of the login session. - Otherwise, the database is queried for a user matching the canonicalized username. From f71ac09722f7c46f427044bfd32367bbc3e29f8c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 16:42:33 -0500 Subject: [PATCH 118/138] Fix incorrect past tense --- src/Tgstation.Server.Host/Security/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Security/README.md b/src/Tgstation.Server.Host/Security/README.md index c12dd0f587d..404b27e1627 100644 --- a/src/Tgstation.Server.Host/Security/README.md +++ b/src/Tgstation.Server.Host/Security/README.md @@ -39,7 +39,7 @@ 1. For non-OAuth logins, maintenance is performed on the user's DB entry at this point - For non-OS logins: 1. The provided password is hashed and checked against the database entry. If it does not match, HTTP 401 will be returned. - - This can potentially cause a change to the DB's stored `PasswordHash` if TGS has updated its dependencies and Microsoft has decided to deprecated the previous hashing method since the user last logged in. + - This can potentially cause a change to the DB's stored `PasswordHash` if TGS has updated its dependencies and Microsoft has decided to deprecate the previous hashing method since the user last logged in. - If this occurs, it invalidates all previous logins for the user. - For OS logins: 1. If the `PasswordHash` in the DB isn't null, it is set as such. This invalidates all previous logins for the user. From b939d722c9ece1aff739be2f9e0f86f45c1fdcb9 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 5 Nov 2023 17:26:53 -0500 Subject: [PATCH 119/138] Bump webpanel version to 4.26.0 --- build/ControlPanelVersion.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ControlPanelVersion.props b/build/ControlPanelVersion.props index c2a4c37602f..f6f147afdcf 100644 --- a/build/ControlPanelVersion.props +++ b/build/ControlPanelVersion.props @@ -1,6 +1,6 @@ - 4.25.5 + 4.26.0 From 52fc12b7a4775e50ce9447c96cc6844c4df58171 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 6 Nov 2023 16:19:24 -0500 Subject: [PATCH 120/138] Fix `JobsHubGroupMapper` not being initialized --- src/Tgstation.Server.Host/Core/Application.cs | 4 +++- src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs | 9 ++++++++- .../Utils/SignalR/ComprehensiveHubContext.cs | 5 +++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index a331feeb9f3..a1cdabec961 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -404,7 +404,9 @@ void AddTypedContext() services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(x => x.GetRequiredService()); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(provider => provider.GetRequiredService()); + services.AddSingleton(x => x.GetRequiredService()); // bit of a hack, but we need this to load immediated services.AddSingleton(); services.AddSingleton(x => x.GetRequiredService()); diff --git a/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs b/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs index 77ccf99b0b5..719138e6ad2 100644 --- a/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs +++ b/src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs @@ -6,6 +6,7 @@ using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Tgstation.Server.Api.Hubs; @@ -19,7 +20,7 @@ namespace Tgstation.Server.Host.Jobs /// /// Handles mapping groups for the . /// - sealed class JobsHubGroupMapper : IPermissionsUpdateNotifyee + sealed class JobsHubGroupMapper : IPermissionsUpdateNotifyee, IHostedService { /// /// The for the . @@ -96,6 +97,12 @@ public ValueTask InstancePermissionSetDeleted(PermissionSet permissionSet, Cance cancellationToken); } + /// + public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + /// /// Implementation of . /// diff --git a/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs b/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs index 79999080529..3892aad56c1 100644 --- a/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs +++ b/src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs @@ -83,12 +83,13 @@ public ValueTask UserConnected(IAuthenticationContext authenticationContext, THu userId, context.ConnectionId); - var mappingTask = OnConnectionMapGroups( + var mappingTask = OnConnectionMapGroups?.Invoke( authenticationContext, mappedGroups => Task.WhenAll( mappedGroups.Select( group => hub.Groups.AddToGroupAsync(context.ConnectionId, group, cancellationToken))), - cancellationToken); + cancellationToken) + ?? ValueTask.CompletedTask; userConnections.AddOrUpdate( userId, _ => new Dictionary From 9bea7686b28b455a69b29f1592300773e498e5e3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 6 Nov 2023 16:39:23 -0500 Subject: [PATCH 121/138] Add automatic reconnect to client hubs --- build/Version.props | 2 +- src/Tgstation.Server.Client/ApiClient.cs | 4 +- .../ApiClientTokenRefreshRetryPolicy.cs | 54 +++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 src/Tgstation.Server.Client/ApiClientTokenRefreshRetryPolicy.cs diff --git a/build/Version.props b/build/Version.props index f63dc186b28..99f5a049eec 100644 --- a/build/Version.props +++ b/build/Version.props @@ -8,7 +8,7 @@ 9.13.0 7.0.0 12.0.0 - 14.0.0 + 14.1.0 6.6.2 5.6.2 1.4.0 diff --git a/src/Tgstation.Server.Client/ApiClient.cs b/src/Tgstation.Server.Client/ApiClient.cs index ed48bc7bc7d..f4a328d2857 100644 --- a/src/Tgstation.Server.Client/ApiClient.cs +++ b/src/Tgstation.Server.Client/ApiClient.cs @@ -372,13 +372,15 @@ public async ValueTask CreateHubConnection retryPolicy ??= new InfiniteThirtySecondMaxRetryPolicy(); + var wrappedPolicy = new ApiClientTokenRefreshRetryPolicy(this, retryPolicy); + HubConnection? hubConnection = null; var hubConnectionBuilder = new HubConnectionBuilder() .AddNewtonsoftJsonProtocol(options => { options.PayloadSerializerSettings = SerializerSettings; }) - .WithAutomaticReconnect(retryPolicy) + .WithAutomaticReconnect(wrappedPolicy) .WithUrl( new Uri(Url, Routes.JobsHub), HttpTransportType.ServerSentEvents, diff --git a/src/Tgstation.Server.Client/ApiClientTokenRefreshRetryPolicy.cs b/src/Tgstation.Server.Client/ApiClientTokenRefreshRetryPolicy.cs new file mode 100644 index 00000000000..3ec5c1f6a12 --- /dev/null +++ b/src/Tgstation.Server.Client/ApiClientTokenRefreshRetryPolicy.cs @@ -0,0 +1,54 @@ +using System; +using System.Threading; + +using Microsoft.AspNetCore.SignalR.Client; + +namespace Tgstation.Server.Client +{ + /// + /// A that attempts to refresh a given 's token on the first disconnect. + /// + sealed class ApiClientTokenRefreshRetryPolicy : IRetryPolicy + { + /// + /// The backing . + /// + readonly ApiClient apiClient; + + /// + /// The wrapped . + /// + readonly IRetryPolicy wrappedPolicy; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + public ApiClientTokenRefreshRetryPolicy(ApiClient apiClient, IRetryPolicy wrappedPolicy) + { + this.apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); + this.wrappedPolicy = wrappedPolicy ?? throw new ArgumentNullException(nameof(wrappedPolicy)); + } + + /// + public TimeSpan? NextRetryDelay(RetryContext retryContext) + { + if (retryContext == null) + throw new ArgumentNullException(nameof(retryContext)); + + if (retryContext.PreviousRetryCount == 0) + AttemptTokenRefresh(); + + return wrappedPolicy.NextRetryDelay(retryContext); + } + + /// + /// Attempt to refresh the s token asynchronously. + /// + async void AttemptTokenRefresh() + { + await apiClient.RefreshToken(CancellationToken.None); + } + } +} From abd1948d42efcd967ce6bc921735bf5a64aeb6fd Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 6 Nov 2023 16:39:51 -0500 Subject: [PATCH 122/138] Version bump to 5.17.1 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index 99f5a049eec..2ca5699ebf1 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 5.17.0 + 5.17.1 4.7.1 9.13.0 7.0.0 From 37adb228d03a1e38399f11688898409cbba7f06a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 6 Nov 2023 16:56:30 -0500 Subject: [PATCH 123/138] Bump webpanel version to latest --- build/ControlPanelVersion.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ControlPanelVersion.props b/build/ControlPanelVersion.props index f6f147afdcf..02d75e71425 100644 --- a/build/ControlPanelVersion.props +++ b/build/ControlPanelVersion.props @@ -1,6 +1,6 @@ - 4.26.0 + 4.26.2 From 549b65f03c15e16ca3102b58cff3ef4ed4eef497 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 6 Nov 2023 16:57:24 -0500 Subject: [PATCH 124/138] Update nuget packages for API library --- build/Version.props | 2 +- src/Tgstation.Server.Api/Tgstation.Server.Api.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index 2ca5699ebf1..ae1f43929d8 100644 --- a/build/Version.props +++ b/build/Version.props @@ -7,7 +7,7 @@ 4.7.1 9.13.0 7.0.0 - 12.0.0 + 12.0.1 14.1.0 6.6.2 5.6.2 diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index 3573822c52b..e499846add8 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -27,7 +27,7 @@ - + From 72388f8035b3abc9f84eec1c1cdc9e6cc9840a35 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 6 Nov 2023 17:44:45 -0500 Subject: [PATCH 125/138] Add necessary `ObjectDisposedException` to client --- src/Tgstation.Server.Client/ApiClient.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Tgstation.Server.Client/ApiClient.cs b/src/Tgstation.Server.Client/ApiClient.cs index f4a328d2857..bf0934a48b9 100644 --- a/src/Tgstation.Server.Client/ApiClient.cs +++ b/src/Tgstation.Server.Client/ApiClient.cs @@ -482,6 +482,9 @@ protected virtual async ValueTask RunRequest( if (content == null && (method == HttpMethod.Post || method == HttpMethod.Put)) throw new InvalidOperationException("content cannot be null for POST or PUT!"); + if (disposed) + throw new ObjectDisposedException(nameof(ApiClient)); + HttpResponseMessage response; var fullUri = new Uri(Url, route); var serializerSettings = SerializerSettings; From e68dc5b307d0436c917e0fbcd7d1dfe493fba1cd Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 6 Nov 2023 17:46:05 -0500 Subject: [PATCH 126/138] An uncaught exception was never a good idea --- .../ApiClientTokenRefreshRetryPolicy.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Client/ApiClientTokenRefreshRetryPolicy.cs b/src/Tgstation.Server.Client/ApiClientTokenRefreshRetryPolicy.cs index 3ec5c1f6a12..34556c24ecd 100644 --- a/src/Tgstation.Server.Client/ApiClientTokenRefreshRetryPolicy.cs +++ b/src/Tgstation.Server.Client/ApiClientTokenRefreshRetryPolicy.cs @@ -48,7 +48,14 @@ public ApiClientTokenRefreshRetryPolicy(ApiClient apiClient, IRetryPolicy wrappe /// async void AttemptTokenRefresh() { - await apiClient.RefreshToken(CancellationToken.None); + try + { + await apiClient.RefreshToken(CancellationToken.None); + } + catch + { + // intentionally ignored + } } } } From 90d41cc88d13a1e011eb0aa1df97f5edf4f7356c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 6 Nov 2023 17:56:03 -0500 Subject: [PATCH 127/138] Replace HTTP 426 with 400 Notable exception is the swarm API, which still uses 426 for version mismatches and would be a pain in the ass to change correctly. Closes #1693 --- docs/API.dox | 1 - src/Tgstation.Server.Client/ApiClient.cs | 5 +++-- src/Tgstation.Server.Host/Controllers/README.md | 1 - .../Extensions/ApplicationBuilderExtensions.cs | 5 +---- src/Tgstation.Server.Host/Security/README.md | 4 ++-- tests/Tgstation.Server.Tests/Live/RawRequestTests.cs | 4 ++-- 6 files changed, 8 insertions(+), 12 deletions(-) diff --git a/docs/API.dox b/docs/API.dox index a859e36c47e..be71673604e 100644 --- a/docs/API.dox +++ b/docs/API.dox @@ -72,7 +72,6 @@ TGS will only every return the response codes listed here - 410: Gone. Attempted to access/modify a resource that ideally should have been ready, but isn't or no longer is - 422: Unprocessable Entity: Used specifically when an operation that requires a server restart is unable to be performed due to the @ref Tgstation.Server.Host.Watchdog not being present in the deployment. Should not happen with a proper server configuration. Response body contains an @ref Tgstation.Server.Api.Models.ErrorMessage - 424: Failed Dependency: When a request that depends on an external API fails for a reason other than rate limiting. The response body will contain an @ref Tgstation.Server.Api.Models.ErrorMessage model detailing the error. -- 426: Upgrade required: Used when the client's API version is not compatible with the server's. Response body contains an @ref Tgstation.Server.Api.Models.ErrorMessage - 429: Rate limited. Used with operations that rely on GitHub.com. If a rate limit is hit for an operation this will be returned. Response will contain a Retry-After header with the amount of seconds to wait. - 500: Server error. Please report the request and response body to the code repository - 501: Not implemented. Functionality not available in the current server version diff --git a/src/Tgstation.Server.Client/ApiClient.cs b/src/Tgstation.Server.Client/ApiClient.cs index 28c5611bddd..3270be8ea28 100644 --- a/src/Tgstation.Server.Client/ApiClient.cs +++ b/src/Tgstation.Server.Client/ApiClient.cs @@ -131,8 +131,6 @@ static void HandleBadResponse(HttpResponseMessage response, string json) #pragma warning restore IDE0066 // Convert switch statement to expression #pragma warning restore IDE0010 // Add missing cases { - case HttpStatusCode.UpgradeRequired: - throw new VersionMismatchException(errorMessage, response); case HttpStatusCode.Unauthorized: throw new UnauthorizedException(errorMessage, response); case HttpStatusCode.InternalServerError: @@ -154,6 +152,9 @@ static void HandleBadResponse(HttpResponseMessage response, string json) case (HttpStatusCode)429: throw new RateLimitException(errorMessage, response); default: + if (errorMessage?.ErrorCode == ErrorCode.ApiMismatch) + throw new VersionMismatchException(errorMessage, response); + throw new ApiConflictException(errorMessage, response); } } diff --git a/src/Tgstation.Server.Host/Controllers/README.md b/src/Tgstation.Server.Host/Controllers/README.md index e1214f91f27..66c5df1244d 100644 --- a/src/Tgstation.Server.Host/Controllers/README.md +++ b/src/Tgstation.Server.Host/Controllers/README.md @@ -7,7 +7,6 @@ Some notable exceptions: - [ApiController](./ApiController.cs) is the base class of nearly all API related controllers. It does the following: - Contains code to deny the request if the instance is not present when it should be. - Contains the `IDatabaseContext` and `ILogger` properties for child controllers. - - Returns 426 Upgrade Required if the API version in the headers are incompatible with the request. - Returns 400 Bad Request if the headers or the PUT/POST'd model is invalid. - Returns 401 If an `IAuthenticationContext` could not be created for a request. - [BridgeController](./BridgeController.cs) is a special controller accessible only from localhost and is used to receive bridge request from DreamDaemon diff --git a/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs b/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs index 55086228dc6..40661c9a5e5 100644 --- a/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ApplicationBuilderExtensions.cs @@ -132,11 +132,8 @@ public static void UseApiCompatibility(this IApplicationBuilder applicationBuild var apiHeadersProvider = context.RequestServices.GetRequiredService(); if (apiHeadersProvider.ApiHeaders?.Compatible() == false) { - await new JsonResult( + await new BadRequestObjectResult( new ErrorMessageResponse(ErrorCode.ApiMismatch)) - { - StatusCode = (int)HttpStatusCode.UpgradeRequired, - } .ExecuteResultAsync(new ActionContext { HttpContext = context, diff --git a/src/Tgstation.Server.Host/Security/README.md b/src/Tgstation.Server.Host/Security/README.md index 404b27e1627..ac6cbbc08ff 100644 --- a/src/Tgstation.Server.Host/Security/README.md +++ b/src/Tgstation.Server.Host/Security/README.md @@ -16,7 +16,7 @@ ## For the login request (`POST /`) -1. An attempt to parse the `ApiHeaders` is made. If they were valid, the API version check is performed. If it fails, HTTP 426 with an `ErrorMessageResponse` will be returned. +1. An attempt to parse the `ApiHeaders` is made. If they were valid, the API version check is performed. If it fails, HTTP 400 with an `ErrorMessageResponse` will be returned. 1. If, for some reason, the user attempts to use a JWT to authenticate this request, steps 2-4 of the non-login pipeline list below are performed. 1. The `ApiController` base class inspects the request. - At this point, if the `ApiHeaders` (MINUS the `Authorization` header) cannot be properly parsed, HTTP 400 with an `ErrorMessageResponse` is returned. @@ -51,7 +51,7 @@ ## For all other authenticated requests -1. An attempt to parse the `ApiHeaders` is made. If they were valid. The API version check is performed. If it fails, HTTP 426 with an `ErrorMessageResponse` will be returned. +1. An attempt to parse the `ApiHeaders` is made. If they were valid. The API version check is performed. If it fails, HTTP 400 with an `ErrorMessageResponse` will be returned. 1. The JWT, if present, is validated. If it is, the scope's [AuthenticationContextFactory](./AuthenticationContextFactory.cs) has `SetTokenNbf` called. If not, HTTP 401 will be returned. - Inside ASP.NET Core, this initializes the calling user's identity principal and sets the "sub" claim to the TGS user ID parsed out of the JWT. - We know it's the user ID because we set it up like that in the [TokenFactory](./TokenFactory.cs) diff --git a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs index fbf5fe44e56..bf0d409e61a 100644 --- a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs +++ b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs @@ -87,7 +87,7 @@ static async Task TestRequestValidation(IServerClient serverClient, Cancellation request.Headers.Add(ApiHeaders.ApiVersionHeader, "Tgstation.Server.Api/6.0.0"); request.Headers.Authorization = new AuthenticationHeaderValue(ApiHeaders.BearerAuthenticationScheme, token); using var response = await httpClient.SendAsync(request, cancellationToken); - Assert.AreEqual(HttpStatusCode.UpgradeRequired, response.StatusCode); + Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode); var content = await response.Content.ReadAsStringAsync(cancellationToken); var message = JsonConvert.DeserializeObject(content); Assert.AreEqual(ErrorCode.ApiMismatch, message.ErrorCode); @@ -101,7 +101,7 @@ static async Task TestRequestValidation(IServerClient serverClient, Cancellation request.Headers.Add(ApiHeaders.ApiVersionHeader, "Tgstation.Server.Api/6.0.0"); request.Headers.Authorization = new AuthenticationHeaderValue(ApiHeaders.BearerAuthenticationScheme, token); using var response = await httpClient.SendAsync(request, cancellationToken); - Assert.AreEqual(HttpStatusCode.UpgradeRequired, response.StatusCode); + Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode); var content = await response.Content.ReadAsStringAsync(cancellationToken); var message = JsonConvert.DeserializeObject(content); Assert.AreEqual(ErrorCode.ApiMismatch, message.ErrorCode); From 7c9a1a6a4fb9f04d22cbef627937278dd1b7aef7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 6 Nov 2023 21:45:45 -0500 Subject: [PATCH 128/138] Fix a cause of spurious CI failures --- .../Live/Instance/JobsHubTests.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index bf683503c11..a52e6e1b1d6 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -158,19 +158,24 @@ await permedUser.Instances.Update(new InstanceUpdateRequest var seenThisJob = seenJobs.TryGetValue(job.Id.Value, out var hubJob); if (seenThisJob) { - Assert.AreEqual(job.StoppedAt, hubJob.StoppedAt); + if (hubJob.StoppedAt.HasValue) + { + Assert.AreEqual(job.StoppedAt, hubJob.StoppedAt); + Assert.AreEqual(job.ExceptionDetails, hubJob.ExceptionDetails); + Assert.AreEqual(job.Progress, hubJob.Progress); + Assert.AreEqual(job.Stage, hubJob.Stage); + Assert.AreEqual(job.ErrorCode, hubJob.ErrorCode); + Assert.AreEqual(job.CancelledBy?.Id, hubJob.CancelledBy?.Id); + Assert.AreEqual(job.Cancelled, hubJob.Cancelled); + } + Assert.AreEqual(job.InstanceId, hubJob.InstanceId); - Assert.AreEqual(job.ExceptionDetails, hubJob.ExceptionDetails); - Assert.AreEqual(job.Stage, hubJob.Stage); - Assert.AreEqual(job.CancelledBy?.Id, hubJob.CancelledBy?.Id); - Assert.AreEqual(job.Cancelled, hubJob.Cancelled); Assert.AreEqual(job.StartedBy?.Id, hubJob.StartedBy?.Id); Assert.AreEqual(job.CancelRight, hubJob.CancelRight); Assert.AreEqual(job.CancelRightsType, hubJob.CancelRightsType); - Assert.AreEqual(job.Progress, hubJob.Progress); Assert.AreEqual(job.Description, hubJob.Description); - Assert.AreEqual(job.ErrorCode, hubJob.ErrorCode); Assert.AreEqual(job.StartedAt, hubJob.StartedAt); + Assert.AreEqual(job.JobCode, hubJob.JobCode); } else { From 2f9fb5df0c66a20960c3472c2dcb45bdbc2980c6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 7 Nov 2023 08:52:01 -0500 Subject: [PATCH 129/138] Suppress the NU5104 while we're in preview Added to #1589 --- src/Tgstation.Server.Client/Tgstation.Server.Client.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 375fb7161a4..c946acf8414 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -7,6 +7,7 @@ Client library for tgstation-server. json web api tgstation-server tgstation ss13 byond client http $(TGS_NUGET_RELEASE_NOTES_CLIENT) + NU5104 From 469b16d815ec1804229a58a2fd348cf959045085 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 9 Nov 2023 08:53:29 -0500 Subject: [PATCH 130/138] Fix test errors caused by DB truncation --- tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index a52e6e1b1d6..d097aae7ebd 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -169,12 +169,17 @@ await permedUser.Instances.Update(new InstanceUpdateRequest Assert.AreEqual(job.Cancelled, hubJob.Cancelled); } + static DateTimeOffset PerformDBTruncation(DateTimeOffset original) + => new DateTimeOffset( + original.Ticks - (original.Ticks % TimeSpan.TicksPerSecond), + original.Offset); + Assert.AreEqual(job.InstanceId, hubJob.InstanceId); Assert.AreEqual(job.StartedBy?.Id, hubJob.StartedBy?.Id); Assert.AreEqual(job.CancelRight, hubJob.CancelRight); Assert.AreEqual(job.CancelRightsType, hubJob.CancelRightsType); Assert.AreEqual(job.Description, hubJob.Description); - Assert.AreEqual(job.StartedAt, hubJob.StartedAt); + Assert.AreEqual(PerformDBTruncation(job.StartedAt.Value), PerformDBTruncation(hubJob.StartedAt.Value)); // RHS may NOT be DB truncated, both sides because not all DBs do this Assert.AreEqual(job.JobCode, hubJob.JobCode); } else From 06dda10ac2a5106cbf793a926909d4b7d8652dc4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 10 Nov 2023 10:04:09 -0500 Subject: [PATCH 131/138] Change the swagger documentation path to `/doc/tgs_api.json` Change hosted site path to `/documentation` Closes #1586 --- .github/workflows/ci-pipeline.yml | 8 ++++---- .../Configuration/GeneralConfiguration.cs | 2 +- src/Tgstation.Server.Host/Core/Application.cs | 11 +++++++++-- .../Utils/SwaggerConfiguration.cs | 7 ++++++- src/Tgstation.Server.Host/appsettings.yml | 2 +- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 4 ++-- 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 3a5e143e6ba..123c939f48c 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -478,7 +478,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: openapi-spec - path: C:/swagger.json + path: C:/tgs_api.json - name: Package Server Service if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Basic' }} @@ -706,7 +706,7 @@ jobs: path: ./swagger - name: Lint OpenAPI Spec - run: npx lint-openapi -v -p -c build/OpenApiValidationSettings.json ./swagger/swagger.json + run: npx lint-openapi -v -p -c build/OpenApiValidationSettings.json ./swagger/tgs_api.json upload-code-coverage: name: Upload Code Coverage @@ -1311,7 +1311,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./swagger/swagger.json + asset_path: ./swagger/tgs_api.json asset_name: swagger.json asset_content_type: application/json @@ -1615,7 +1615,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./swagger/swagger.json + asset_path: ./swagger/tgs_api.json asset_name: swagger.json asset_content_type: application/json diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index 7b02f6d177f..1bb02a86e60 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -108,7 +108,7 @@ public sealed class GeneralConfiguration : ServerInformationBase public bool UseBasicWatchdog { get; set; } /// - /// If the swagger UI should be made avaiable. + /// If the swagger documentation and UI should be made avaiable. /// public bool HostApiDocumentation { get; set; } diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index d730b9de108..4d54a42b014 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -471,8 +471,15 @@ public void Configure( if (generalConfiguration.HostApiDocumentation) { - applicationBuilder.UseSwagger(); - applicationBuilder.UseSwaggerUI(c => c.SwaggerEndpoint("/swagger/v1/swagger.json", "TGS API")); + applicationBuilder.UseSwagger(options => + { + options.RouteTemplate = Routes.Root + "doc/{documentName}.{json|yaml}"; + }); + applicationBuilder.UseSwaggerUI(options => + { + options.RoutePrefix = "documentation"; + options.SwaggerEndpoint(Routes.Root + $"doc/{SwaggerConfiguration.DocumentName}.json", "TGS API"); + }); logger.LogTrace("Swagger API generation enabled"); } diff --git a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs index 89f77167329..927fc4f4fa7 100644 --- a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs +++ b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs @@ -27,6 +27,11 @@ namespace Tgstation.Server.Host.Utils /// sealed class SwaggerConfiguration : IOperationFilter, IDocumentFilter, ISchemaFilter, IRequestBodyFilter { + /// + /// The name of the swagger document. + /// + public const string DocumentName = "tgs_api"; + /// /// The name for password authentication. /// @@ -51,7 +56,7 @@ sealed class SwaggerConfiguration : IOperationFilter, IDocumentFilter, ISchemaFi public static void Configure(SwaggerGenOptions swaggerGenOptions, string assemblyDocumentationPath, string apiDocumentationPath) { swaggerGenOptions.SwaggerDoc( - "v1", + DocumentName, new OpenApiInfo { Title = "TGS API", diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index bebeb79c043..680c347e3d3 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -13,7 +13,7 @@ General: UserGroupLimit: 25 # Maximum number of allowed groups InstanceLimit: 10 # Maximum number of allowed instances ValidInstancePaths: # An array of directories instances may be created in (either directly or as a subdirectory). null removes the restriction - HostApiDocumentation: false # Make HTTP API documentation available at /swagger/v1/swagger.json + HostApiDocumentation: false # Make HTTP API documentation available at /doc/tgs_api.json SkipAddingByondFirewallException: false # Windows Only: Prevent running netsh.exe to add a firewall exception for installed DreamDaemon binaries DeploymentDirectoryCopyTasksPerCore: 100 # Maximum number of concurrent file copy operations PER available CPU core Session: diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 0ef060c2f36..f8cf0c78029 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1302,11 +1302,11 @@ async ValueTask CreateUserWithNoInstancePerms() // Dump swagger to disk // This is purely for CI using var httpClient = new HttpClient(); - var webRequestTask = httpClient.GetAsync(server.Url.ToString() + "swagger/v1/swagger.json", cancellationToken); + var webRequestTask = httpClient.GetAsync(server.Url.ToString() + "doc/tgs_api.json", cancellationToken); using var response = await webRequestTask; response.EnsureSuccessStatusCode(); await using var content = await response.Content.ReadAsStreamAsync(cancellationToken); - await using var output = new FileStream(@"C:\swagger.json", FileMode.Create); + await using var output = new FileStream(@"C:\tgs_api.json", FileMode.Create); await content.CopyToAsync(output, cancellationToken); } From 9e1806a2b4551b9369c80da0235b66e5774b3890 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 10 Nov 2023 12:11:28 -0500 Subject: [PATCH 132/138] Move all API functionality to `/api`. Move `BridgeController` route but BYOND forces us to support the legacy one as well. Add a basic homepage if it's not obvious where to redirect the user. Closes #1689 --- build/Version.props | 2 +- src/DMAPI/tgs/core/core.dm | 4 +- src/DMAPI/tgs/v5/__interop_version.dm | 2 +- src/DMAPI/tgs/v5/api.dm | 12 +- src/DMAPI/tgs/v5/bridge.dm | 4 +- src/Tgstation.Server.Api/Routes.cs | 32 ++--- src/Tgstation.Server.Client/ApiClient.cs | 2 +- src/Tgstation.Server.Client/IServerClient.cs | 2 +- src/Tgstation.Server.Client/ServerClient.cs | 2 +- .../ServerClientFactory.cs | 4 +- ...HomeController.cs => ApiRootController.cs} | 52 +++------ .../Controllers/BridgeController.cs | 11 +- .../Controllers/RootController.cs | 109 ++++++++++++++++++ src/Tgstation.Server.Host/Core/Application.cs | 6 +- .../Swarm/SwarmConstants.cs | 2 +- .../Utils/SwaggerConfiguration.cs | 7 +- .../Views/Root/Index.cshtml | 28 +++++ src/Tgstation.Server.Host/appsettings.yml | 2 +- tests/DMAPI/LongRunning/Test.dm | 30 +++++ .../Live/Instance/TestBridgeHandler.cs | 2 +- .../Live/Instance/WatchdogTest.cs | 10 ++ .../Live/LiveTestingServer.cs | 8 +- .../Live/RawRequestTests.cs | 18 +-- .../Live/TestLiveServer.cs | 57 ++++----- tests/Tgstation.Server.Tests/TestVersions.cs | 27 ++++- 25 files changed, 322 insertions(+), 113 deletions(-) rename src/Tgstation.Server.Host/Controllers/{HomeController.cs => ApiRootController.cs} (90%) create mode 100644 src/Tgstation.Server.Host/Controllers/RootController.cs create mode 100644 src/Tgstation.Server.Host/Views/Root/Index.cshtml diff --git a/build/Version.props b/build/Version.props index e3a0b04745f..5bba820f55e 100644 --- a/build/Version.props +++ b/build/Version.props @@ -10,7 +10,7 @@ 13.0.0 15.0.0 6.6.2 - 5.6.2 + 5.7.0 1.4.0 1.2.1 2.0.0 diff --git a/src/DMAPI/tgs/core/core.dm b/src/DMAPI/tgs/core/core.dm index b9a9f27a28a..4a408e89a23 100644 --- a/src/DMAPI/tgs/core/core.dm +++ b/src/DMAPI/tgs/core/core.dm @@ -42,11 +42,11 @@ var/datum/tgs_version/max_api_version = TgsMaximumApiVersion(); if(version.suite != null && version.minor != null && version.patch != null && version.deprecated_patch != null && version.deprefixed_parameter > max_api_version.deprefixed_parameter) - TGS_ERROR_LOG("Detected unknown API version! Defaulting to latest. Update the DMAPI to fix this problem.") + TGS_ERROR_LOG("Detected unknown Interop API version! Defaulting to latest. Update the DMAPI to fix this problem.") api_datum = /datum/tgs_api/latest if(!api_datum) - TGS_ERROR_LOG("Found unsupported API version: [raw_parameter]. If this is a valid version please report this, backporting is done on demand.") + TGS_ERROR_LOG("Found unsupported Interop API version: [raw_parameter]. If this is a valid version please report this, backporting is done on demand.") return TGS_INFO_LOG("Activating API for version [version.deprefixed_parameter]") diff --git a/src/DMAPI/tgs/v5/__interop_version.dm b/src/DMAPI/tgs/v5/__interop_version.dm index 1b52b31d6a7..83420d130a7 100644 --- a/src/DMAPI/tgs/v5/__interop_version.dm +++ b/src/DMAPI/tgs/v5/__interop_version.dm @@ -1 +1 @@ -"5.6.2" +"5.7.0" diff --git a/src/DMAPI/tgs/v5/api.dm b/src/DMAPI/tgs/v5/api.dm index 7226f29bba6..4a101d58dc1 100644 --- a/src/DMAPI/tgs/v5/api.dm +++ b/src/DMAPI/tgs/v5/api.dm @@ -17,6 +17,8 @@ var/list/chat_channels var/initialized = FALSE + var/initial_bridge_request_received = FALSE + var/datum/tgs_version/interop_version var/chunked_requests = 0 var/list/chunked_topics = list() @@ -25,7 +27,8 @@ /datum/tgs_api/v5/New() . = ..() - TGS_DEBUG_LOG("V5 API created") + interop_version = version + TGS_DEBUG_LOG("V5 API created: [json_encode(args)]") /datum/tgs_api/v5/ApiVersion() return new /datum/tgs_version( @@ -38,7 +41,7 @@ access_identifier = world.params[DMAPI5_PARAM_ACCESS_IDENTIFIER] var/datum/tgs_version/api_version = ApiVersion() - version = null + version = null // we want this to be the TGS version, not the interop version var/list/bridge_response = Bridge(DMAPI5_BRIDGE_COMMAND_STARTUP, list(DMAPI5_BRIDGE_PARAMETER_MINIMUM_SECURITY_LEVEL = minimum_required_security_level, DMAPI5_BRIDGE_PARAMETER_VERSION = api_version.raw_parameter, DMAPI5_PARAMETER_CUSTOM_COMMANDS = ListCustomCommands())) if(!istype(bridge_response)) TGS_ERROR_LOG("Failed initial bridge request!") @@ -53,7 +56,8 @@ TGS_INFO_LOG("DMAPI validation, exiting...") TerminateWorld() - version = new /datum/tgs_version(runtime_information[DMAPI5_RUNTIME_INFORMATION_SERVER_VERSION]) + initial_bridge_request_received = TRUE + version = new /datum/tgs_version(runtime_information[DMAPI5_RUNTIME_INFORMATION_SERVER_VERSION]) // reassigning this because it can change if TGS updates security_level = runtime_information[DMAPI5_RUNTIME_INFORMATION_SECURITY_LEVEL] visibility = runtime_information[DMAPI5_RUNTIME_INFORMATION_VISIBILITY] instance_name = runtime_information[DMAPI5_RUNTIME_INFORMATION_INSTANCE_NAME] @@ -105,7 +109,7 @@ /datum/tgs_api/v5/proc/RequireInitialBridgeResponse() TGS_DEBUG_LOG("RequireInitialBridgeResponse()") var/logged = FALSE - while(!version) + while(!initial_bridge_request_received) if(!logged) TGS_DEBUG_LOG("RequireInitialBridgeResponse: Starting sleep") logged = TRUE diff --git a/src/DMAPI/tgs/v5/bridge.dm b/src/DMAPI/tgs/v5/bridge.dm index 37f58bcdf63..8e35dc3b1e3 100644 --- a/src/DMAPI/tgs/v5/bridge.dm +++ b/src/DMAPI/tgs/v5/bridge.dm @@ -48,7 +48,9 @@ var/json = CreateBridgeData(command, data, TRUE) var/encoded_json = url_encode(json) - var/url = "http://127.0.0.1:[server_port]/Bridge?[DMAPI5_BRIDGE_DATA]=[encoded_json]" + var/api_prefix = interop_version.minor >= 7 ? "api/" : "" + + var/url = "http://127.0.0.1:[server_port]/[api_prefix]Bridge?[DMAPI5_BRIDGE_DATA]=[encoded_json]" return url /datum/tgs_api/v5/proc/CreateBridgeData(command, list/data, needs_auth) diff --git a/src/Tgstation.Server.Api/Routes.cs b/src/Tgstation.Server.Api/Routes.cs index b62d7816043..862dd3994e0 100644 --- a/src/Tgstation.Server.Api/Routes.cs +++ b/src/Tgstation.Server.Api/Routes.cs @@ -8,19 +8,19 @@ namespace Tgstation.Server.Api public static class Routes { /// - /// The root controller. + /// The root of API methods. /// - public const string Root = "/"; + public const string ApiRoot = "/api/"; /// /// The root route of all hubs. /// - public const string HubsRoot = Root + "hubs"; + public const string HubsRoot = ApiRoot + "hubs"; /// /// The server administration controller. /// - public const string Administration = Root + "Administration"; + public const string Administration = ApiRoot + "Administration"; /// /// The endpoint to download server logs. @@ -30,32 +30,32 @@ public static class Routes /// /// The user controller. /// - public const string User = Root + "User"; + public const string User = ApiRoot + "User"; /// /// The user group controller. /// - public const string UserGroup = Root + "UserGroup"; + public const string UserGroup = ApiRoot + "UserGroup"; /// /// The controller. /// - public const string InstanceManager = Root + "Instance"; + public const string InstanceManager = ApiRoot + "Instance"; /// /// The BYOND controller. /// - public const string Byond = Root + "Byond"; + public const string Byond = ApiRoot + "Byond"; /// /// The git repository controller. /// - public const string Repository = Root + "Repository"; + public const string Repository = ApiRoot + "Repository"; /// /// The DreamDaemon controller. /// - public const string DreamDaemon = Root + "DreamDaemon"; + public const string DreamDaemon = ApiRoot + "DreamDaemon"; /// /// For accessing DD diagnostics. @@ -65,7 +65,7 @@ public static class Routes /// /// The configuration controller. /// - public const string Configuration = Root + "Config"; + public const string Configuration = ApiRoot + "Config"; /// /// To be paired with for accessing s. @@ -80,27 +80,27 @@ public static class Routes /// /// The instance permission set controller. /// - public const string InstancePermissionSet = Root + "InstancePermissionSet"; + public const string InstancePermissionSet = ApiRoot + "InstancePermissionSet"; /// /// The chat bot controller. /// - public const string Chat = Root + "Chat"; + public const string Chat = ApiRoot + "Chat"; /// /// The deployment controller. /// - public const string DreamMaker = Root + "DreamMaker"; + public const string DreamMaker = ApiRoot + "DreamMaker"; /// /// The jobs controller. /// - public const string Jobs = Root + "Job"; + public const string Jobs = ApiRoot + "Job"; /// /// The transfer controller. /// - public const string Transfer = Root + "Transfer"; + public const string Transfer = ApiRoot + "Transfer"; /// /// The postfix for list operations. diff --git a/src/Tgstation.Server.Client/ApiClient.cs b/src/Tgstation.Server.Client/ApiClient.cs index d759ccbbdd1..f691dd0e905 100644 --- a/src/Tgstation.Server.Client/ApiClient.cs +++ b/src/Tgstation.Server.Client/ApiClient.cs @@ -349,7 +349,7 @@ public async ValueTask RefreshToken(CancellationToken cancellationToken) if (startingToken != headers.Token) return true; - var token = await RunRequest(Routes.Root, new object(), HttpMethod.Post, null, true, cancellationToken).ConfigureAwait(false); + var token = await RunRequest(Routes.ApiRoot, new object(), HttpMethod.Post, null, true, cancellationToken).ConfigureAwait(false); headers = new ApiHeaders(headers.UserAgent!, token); } finally diff --git a/src/Tgstation.Server.Client/IServerClient.cs b/src/Tgstation.Server.Client/IServerClient.cs index 65014f66b1e..317e012ed08 100644 --- a/src/Tgstation.Server.Client/IServerClient.cs +++ b/src/Tgstation.Server.Client/IServerClient.cs @@ -16,7 +16,7 @@ namespace Tgstation.Server.Client public interface IServerClient : IAsyncDisposable { /// - /// The connected server . + /// The connected server's root . /// Uri Url { get; } diff --git a/src/Tgstation.Server.Client/ServerClient.cs b/src/Tgstation.Server.Client/ServerClient.cs index 186e32745b2..6fce35fcf9c 100644 --- a/src/Tgstation.Server.Client/ServerClient.cs +++ b/src/Tgstation.Server.Client/ServerClient.cs @@ -66,7 +66,7 @@ public ServerClient(IApiClient apiClient) public ValueTask DisposeAsync() => apiClient.DisposeAsync(); /// - public ValueTask ServerInformation(CancellationToken cancellationToken) => apiClient.Read(Routes.Root, cancellationToken); + public ValueTask ServerInformation(CancellationToken cancellationToken) => apiClient.Read(Routes.ApiRoot, cancellationToken); /// public void AddRequestLogger(IRequestLogger requestLogger) => apiClient.AddRequestLogger(requestLogger); diff --git a/src/Tgstation.Server.Client/ServerClientFactory.cs b/src/Tgstation.Server.Client/ServerClientFactory.cs index 155198996a2..98a51e8dc5d 100644 --- a/src/Tgstation.Server.Client/ServerClientFactory.cs +++ b/src/Tgstation.Server.Client/ServerClientFactory.cs @@ -138,7 +138,7 @@ public async ValueTask GetServerInformation( if (timeout.HasValue) api.Timeout = timeout.Value; - return await api.Read(Routes.Root, cancellationToken).ConfigureAwait(false); + return await api.Read(Routes.ApiRoot, cancellationToken).ConfigureAwait(false); } /// @@ -169,7 +169,7 @@ async ValueTask CreateWithNewToken( if (timeout.HasValue) api.Timeout = timeout.Value; - token = await api.Update(Routes.Root, cancellationToken).ConfigureAwait(false); + token = await api.Update(Routes.ApiRoot, cancellationToken).ConfigureAwait(false); } var apiHeaders = new ApiHeaders(productHeaderValue, token); diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/ApiRootController.cs similarity index 90% rename from src/Tgstation.Server.Host/Controllers/HomeController.cs rename to src/Tgstation.Server.Host/Controllers/ApiRootController.cs index 659a379591d..fbfc1d44659 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiRootController.cs @@ -32,66 +32,66 @@ namespace Tgstation.Server.Host.Controllers /// /// Root for the . /// - [Route(Routes.Root)] - public sealed class HomeController : ApiController + [Route(Routes.ApiRoot)] + public sealed class ApiRootController : ApiController { /// - /// The for the . + /// The for the . /// readonly ITokenFactory tokenFactory; /// - /// The for the . + /// The for the . /// readonly ISystemIdentityFactory systemIdentityFactory; /// - /// The for the . + /// The for the . /// readonly ICryptographySuite cryptographySuite; /// - /// The for the . + /// The for the . /// readonly IAssemblyInformationProvider assemblyInformationProvider; /// - /// The for the . + /// The for the . /// readonly IIdentityCache identityCache; /// - /// The for the . + /// The for the . /// readonly IOAuthProviders oAuthProviders; /// - /// The for the . + /// The for the . /// readonly IPlatformIdentifier platformIdentifier; /// - /// The for the . + /// The for the . /// readonly ISwarmService swarmService; /// - /// The for the . + /// The for the . /// readonly IServerControl serverControl; /// - /// The for the . + /// The for the . /// readonly GeneralConfiguration generalConfiguration; /// - /// The for the . + /// The for the . /// readonly ControlPanelConfiguration controlPanelConfiguration; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The for the . /// The for the . @@ -108,7 +108,7 @@ public sealed class HomeController : ApiController /// The containing the value of . /// The for the . /// The for the . - public HomeController( + public ApiRootController( IDatabaseContext databaseContext, IAuthenticationContext authenticationContext, ITokenFactory tokenFactory, @@ -122,7 +122,7 @@ public HomeController( IServerControl serverControl, IOptions generalConfigurationOptions, IOptions controlPanelConfigurationOptions, - ILogger logger, + ILogger logger, IApiHeadersProvider apiHeadersProvider) : base( databaseContext, @@ -154,29 +154,12 @@ public HomeController( [HttpGet] [AllowAnonymous] [ProducesResponseType(typeof(ServerInformationResponse), 200)] -#pragma warning disable CA1506 - public IActionResult Home() + public IActionResult ServerInfo() { - if (controlPanelConfiguration.Enable) - Response.Headers.Add( - HeaderNames.Vary, - new StringValues(ApiHeaders.ApiVersionHeader)); - // if they tried to authenticate in any form and failed, let them know immediately bool failIfUnauthed; if (ApiHeaders == null) { - if (controlPanelConfiguration.Enable && !Request.Headers.TryGetValue(ApiHeaders.ApiVersionHeader, out _)) - { - Logger.LogDebug("No API headers on request, redirecting to control panel..."); - - var controlPanelRoute = controlPanelConfiguration.PublicPath; - if (String.IsNullOrWhiteSpace(controlPanelRoute)) - controlPanelRoute = ControlPanelController.ControlPanelRoute; - - return Redirect(controlPanelRoute); - } - try { // we only allow authorization header issues @@ -211,7 +194,6 @@ public IActionResult Home() UpdateInProgress = serverControl.UpdateInProgress, }); } -#pragma warning restore CA1506 /// /// Attempt to authenticate a using . diff --git a/src/Tgstation.Server.Host/Controllers/BridgeController.cs b/src/Tgstation.Server.Host/Controllers/BridgeController.cs index 4ae2e0497c8..86d67d19c05 100644 --- a/src/Tgstation.Server.Host/Controllers/BridgeController.cs +++ b/src/Tgstation.Server.Host/Controllers/BridgeController.cs @@ -8,9 +8,12 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; + using Newtonsoft.Json; + using Serilog.Context; +using Tgstation.Server.Api; using Tgstation.Server.Host.Components.Interop; using Tgstation.Server.Host.Components.Interop.Bridge; using Tgstation.Server.Host.Utils; @@ -20,10 +23,16 @@ namespace Tgstation.Server.Host.Controllers /// /// for recieving DMAPI requests from DreamDaemon. /// - [Route("/Bridge")] + [Route("/" + RouteExtension)] // obsolete route, but BYOND can't handle a simple fucking 301 + [Route(Routes.ApiRoot + RouteExtension)] [ApiExplorerSettings(IgnoreApi = true)] public sealed class BridgeController : ApiControllerBase { + /// + /// The route to the . + /// + const string RouteExtension = "Bridge"; + /// /// If the content of bridge requests and responses should be logged. /// diff --git a/src/Tgstation.Server.Host/Controllers/RootController.cs b/src/Tgstation.Server.Host/Controllers/RootController.cs new file mode 100644 index 00000000000..fd1bf36b112 --- /dev/null +++ b/src/Tgstation.Server.Host/Controllers/RootController.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; + +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; + +using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils; + +namespace Tgstation.Server.Host.Controllers +{ + /// + /// The root path . + /// + [Route("/")] + public sealed class RootController : Controller + { + /// + /// The route to the TGS logo .svg in the on Windows. + /// + public const string ProjectLogoSvgRouteWindows = "/0176d5d8b7d307f158e0.svg"; + + /// + /// The route to the TGS logo .svg in the on Linux. + /// + public const string ProjectLogoSvgRouteLinux = "/b5616c99bf2052a6bbd7.svg"; + + /// + /// The for the . + /// + readonly IAssemblyInformationProvider assemblyInformationProvider; + + /// + /// The for the . + /// + readonly IPlatformIdentifier platformIdentifier; + + /// + /// The for the . + /// + readonly GeneralConfiguration generalConfiguration; + + /// + /// The for the . + /// + readonly ControlPanelConfiguration controlPanelConfiguration; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The containing the value of . + /// The containing the value of . + public RootController( + IAssemblyInformationProvider assemblyInformationProvider, + IPlatformIdentifier platformIdentifier, + IOptions generalConfigurationOptions, + IOptions controlPanelConfigurationOptions) + { + this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); + this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); + generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); + controlPanelConfiguration = controlPanelConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(controlPanelConfigurationOptions)); + } + + /// + /// Gets the server's homepage. + /// + /// The appropriate . + [HttpGet] + [AllowAnonymous] + public IActionResult Index() + { + const string ApiDocumentationRoute = "/" + SwaggerConfiguration.DocumentationSiteRouteExtension; + var panelEnabled = controlPanelConfiguration.Enable; + var apiDocsEnabled = generalConfiguration.HostApiDocumentation; + + if (panelEnabled ^ apiDocsEnabled) + if (panelEnabled) + return Redirect(ControlPanelController.ControlPanelRoute); + else + return Redirect(ApiDocumentationRoute); + + Dictionary links; + if (panelEnabled) + links = new Dictionary() + { + { "Web Control Panel", ControlPanelController.ControlPanelRoute.TrimStart('/') }, + { "API Documentation", SwaggerConfiguration.DocumentationSiteRouteExtension }, + }; + else + links = null; + + var model = new + { + Links = links, + Svg = platformIdentifier.IsWindows // these are different because of motherfucking line endings -_- + ? ProjectLogoSvgRouteWindows + : ProjectLogoSvgRouteLinux, + Title = assemblyInformationProvider.VersionString, + }; + + return View(model); + } + } +} diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 4d54a42b014..c3a4907f281 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -473,12 +473,12 @@ public void Configure( { applicationBuilder.UseSwagger(options => { - options.RouteTemplate = Routes.Root + "doc/{documentName}.{json|yaml}"; + options.RouteTemplate = Routes.ApiRoot + "doc/{documentName}.{json|yaml}"; }); applicationBuilder.UseSwaggerUI(options => { - options.RoutePrefix = "documentation"; - options.SwaggerEndpoint(Routes.Root + $"doc/{SwaggerConfiguration.DocumentName}.json", "TGS API"); + options.RoutePrefix = SwaggerConfiguration.DocumentationSiteRouteExtension; + options.SwaggerEndpoint(Routes.ApiRoot + $"doc/{SwaggerConfiguration.DocumentName}.json", "TGS API"); }); logger.LogTrace("Swagger API generation enabled"); } diff --git a/src/Tgstation.Server.Host/Swarm/SwarmConstants.cs b/src/Tgstation.Server.Host/Swarm/SwarmConstants.cs index fe2b892c327..f5485cc2805 100644 --- a/src/Tgstation.Server.Host/Swarm/SwarmConstants.cs +++ b/src/Tgstation.Server.Host/Swarm/SwarmConstants.cs @@ -14,7 +14,7 @@ static class SwarmConstants /// /// The base route for . /// - public const string ControllerRoute = Routes.Root + "Swarm"; + public const string ControllerRoute = Routes.ApiRoot + "Swarm"; /// /// The header used to pass in the . diff --git a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs index 927fc4f4fa7..8f8b0195bd7 100644 --- a/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs +++ b/src/Tgstation.Server.Host/Utils/SwaggerConfiguration.cs @@ -32,6 +32,11 @@ sealed class SwaggerConfiguration : IOperationFilter, IDocumentFilter, ISchemaFi /// public const string DocumentName = "tgs_api"; + /// + /// The path to the hosted documentation site. + /// + public const string DocumentationSiteRouteExtension = "documentation"; + /// /// The name for password authentication. /// @@ -410,7 +415,7 @@ public void Apply(OpenApiOperation operation, OperationFilterContext context) twoHundredResponseContents.Add(MediaTypeNames.Application.Octet, fileContent); } } - else if (context.MethodInfo.Name == nameof(HomeController.CreateToken)) + else if (context.MethodInfo.Name == nameof(ApiRootController.CreateToken)) { var passwordScheme = new OpenApiSecurityScheme { diff --git a/src/Tgstation.Server.Host/Views/Root/Index.cshtml b/src/Tgstation.Server.Host/Views/Root/Index.cshtml new file mode 100644 index 00000000000..209a6ad5ac9 --- /dev/null +++ b/src/Tgstation.Server.Host/Views/Root/Index.cshtml @@ -0,0 +1,28 @@ +@{ + var svgPath = Model.Svg; + var title = Model.Title; + + + + @title + + + + + @{ + if (Model.Links != null) + foreach (KeyValuePair kvp in Model.Links) + { +

+ @kvp.Key +

+ } + } + + +} diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 680c347e3d3..4ea862d7da8 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -13,7 +13,7 @@ General: UserGroupLimit: 25 # Maximum number of allowed groups InstanceLimit: 10 # Maximum number of allowed instances ValidInstancePaths: # An array of directories instances may be created in (either directly or as a subdirectory). null removes the restriction - HostApiDocumentation: false # Make HTTP API documentation available at /doc/tgs_api.json + HostApiDocumentation: false # Make HTTP API documentation available at /api/doc/tgs_api.json SkipAddingByondFirewallException: false # Windows Only: Prevent running netsh.exe to add a firewall exception for installed DreamDaemon binaries DeploymentDirectoryCopyTasksPerCore: 100 # Maximum number of concurrent file copy operations PER available CPU core Session: diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index c2e274f4d41..c9cbf5cab11 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -180,6 +180,11 @@ var/run_bridge_test kajigger_test = TRUE return "we love casting spells" + var/its_sad = data["im_out_of_memes"] + if(its_sad) + TestLegacyBridge() + return "yeah gimmie a sec" + TgsChatBroadcast(new /datum/tgs_message_content("Recieved non-tgs topic: `[T]`")) return "feck" @@ -349,3 +354,28 @@ var/suppress_bridge_spam = FALSE FailTest("Failed to end bridge limit test! [(istype(final_result) ? json_encode(final_result): (final_result || "null"))]") api.access_identifier = old_ai + +/proc/TestLegacyBridge() + set waitfor = FALSE + + sleep(10) + + var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs) + if(api.interop_version.suite != 5) + FailTest("Legacy bridge test not required anymore?") + + var/old_minor_version = api.interop_version.minor + api.interop_version.minor = 6 // before api repath + + var/result + var/bridge_request = api.CreateBridgeRequest(5, list("chatMessage" = list("text" = "legacy bridge test", "channelIds" = list()))) + try + result = api.PerformBridgeRequest(bridge_request) + catch(var/exception/e2) + world.log << "Caught exception: [e2]" + result = null + + if(!result || lastTgsError) + FailTest("Failed bridge request redirect test!") + + api.interop_version.minor = old_minor_version diff --git a/tests/Tgstation.Server.Tests/Live/Instance/TestBridgeHandler.cs b/tests/Tgstation.Server.Tests/Live/Instance/TestBridgeHandler.cs index 53e0c43a517..4276067b192 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/TestBridgeHandler.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/TestBridgeHandler.cs @@ -88,7 +88,7 @@ public async ValueTask ProcessBridgeRequest(BridgeParameters par Assert.AreEqual("payload", coreMessage); var serializedRequest = JsonConvert.SerializeObject(parameters, DMApiConstants.SerializerSettings); - var actualLastRequest = $"http://127.0.0.1:{serverPort}/Bridge?data=" + HttpUtility.UrlEncode(serializedRequest); + var actualLastRequest = $"http://127.0.0.1:{serverPort}/api/Bridge?data=" + HttpUtility.UrlEncode(serializedRequest); lastBridgeRequestSize = actualLastRequest.Length; return new BridgeResponseHack { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 0a67dc8c029..7c64451fd14 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -219,6 +219,8 @@ async Task InteropTestsForLongRunningDme(CancellationToken cancellationToken) await RegressionTest1550(cancellationToken); + await TestLegacyBridgeEndpoint(cancellationToken); + var deleteJobTask = TestDeleteByondInstallErrorCasesAndQueing(cancellationToken); SessionController.LogTopicRequests = false; @@ -1299,5 +1301,13 @@ async Task CheckDMApiFail(CompileJobResponse compileJob, CancellationToken cance var logtext = await File.ReadAllTextAsync(logfile.FullName, cancellationToken); Assert.IsFalse(String.IsNullOrWhiteSpace(logtext)); } + + async ValueTask TestLegacyBridgeEndpoint(CancellationToken cancellationToken) + { + var result = await topicClient.SendTopic(IPAddress.Loopback, "im_out_of_memes=1", ddPort, cancellationToken); + Assert.AreEqual("yeah gimmie a sec", result.StringData); + await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken); + await CheckDMApiFail((await instanceClient.DreamDaemon.Read(cancellationToken)).ActiveCompileJob, cancellationToken); + } } } diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 5f78b526b16..f43798005fb 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -10,6 +10,7 @@ using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Api; using Tgstation.Server.Api.Models; using Tgstation.Server.Host; using Tgstation.Server.Host.Configuration; @@ -50,7 +51,9 @@ static async Task Cleanup(string directory) } } - public Uri Url { get; } + public Uri ApiUrl { get; } + + public Uri RootUrl { get; } public string Directory { get; } @@ -82,7 +85,8 @@ public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth Directory = Path.Combine(Directory, Guid.NewGuid().ToString()); System.IO.Directory.CreateDirectory(Directory); string urlString = $"http://localhost:{port}"; - Url = new Uri(urlString); + RootUrl = new Uri(urlString); + ApiUrl = new Uri(urlString + Routes.ApiRoot); //so we need a db //we have to rely on env vars diff --git a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs index bf0d409e61a..50e9b211692 100644 --- a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs +++ b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs @@ -37,7 +37,7 @@ static async Task TestRequestValidation(IServerClient serverClient, Cancellation var token = serverClient.Token.Bearer; // check that 400s are returned appropriately using var httpClient = new HttpClient(); - using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString())) + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + Routes.ApiRoot.TrimStart('/'))) { request.Headers.Accept.Clear(); request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); @@ -45,7 +45,7 @@ static async Task TestRequestValidation(IServerClient serverClient, Cancellation Assert.AreEqual(HttpStatusCode.NotAcceptable, response.StatusCode); } - using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString())) + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + Routes.ApiRoot.TrimStart('/'))) { request.Headers.Accept.Clear(); request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); @@ -54,7 +54,7 @@ static async Task TestRequestValidation(IServerClient serverClient, Cancellation Assert.AreEqual(HttpStatusCode.NotAcceptable, response.StatusCode); } - using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString())) + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + Routes.ApiRoot.TrimStart('/'))) { request.Headers.Accept.Clear(); request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); @@ -66,7 +66,7 @@ static async Task TestRequestValidation(IServerClient serverClient, Cancellation Assert.AreEqual(ErrorCode.BadHeaders, message.ErrorCode); } - using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString())) + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + Routes.ApiRoot.TrimStart('/'))) { request.Headers.Accept.Clear(); request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); @@ -79,7 +79,7 @@ static async Task TestRequestValidation(IServerClient serverClient, Cancellation Assert.AreEqual(ApiHeaders.Version, message.ApiVersion); } - using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString())) + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + Routes.ApiRoot.TrimStart('/'))) { request.Headers.Accept.Clear(); request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); @@ -149,7 +149,7 @@ static async Task TestRequestValidation(IServerClient serverClient, Cancellation Assert.AreEqual(ErrorCode.InstanceHeaderRequired, message.ErrorCode); } - using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString())) + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + Routes.ApiRoot.TrimStart('/'))) { request.Headers.Accept.Clear(); request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); @@ -163,7 +163,7 @@ static async Task TestRequestValidation(IServerClient serverClient, Cancellation Assert.AreEqual(ErrorCode.BadHeaders, message.ErrorCode); } - using (var request = new HttpRequestMessage(HttpMethod.Post, url.ToString())) + using (var request = new HttpRequestMessage(HttpMethod.Post, url.ToString() + Routes.ApiRoot.TrimStart('/'))) { request.Headers.Accept.Clear(); request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); @@ -178,7 +178,7 @@ static async Task TestRequestValidation(IServerClient serverClient, Cancellation Assert.AreEqual(ErrorCode.BadHeaders, message.ErrorCode); } - using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString())) + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + Routes.ApiRoot.TrimStart('/'))) { request.Headers.Accept.Clear(); request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); @@ -232,7 +232,7 @@ static async Task TestOAuthFails(IServerClient serverClient, CancellationToken c // just hitting each type of oauth provider for coverage foreach (var I in Enum.GetValues(typeof(OAuthProvider))) - using (var request = new HttpRequestMessage(HttpMethod.Post, url.ToString())) + using (var request = new HttpRequestMessage(HttpMethod.Post, url.ToString() + Routes.ApiRoot.TrimStart('/'))) { request.Headers.Accept.Clear(); request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index f8cf0c78029..3daea816f76 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -255,11 +255,11 @@ async ValueTask TestWithoutAndWithPermission(Func( () => adminClient.Administration.Update( new ServerUpdateRequest @@ -442,7 +442,7 @@ public async Task TestOneServerSwarmUpdate() try { - await using var controllerClient = await CreateAdminClient(controller.Url, cancellationToken); + await using var controllerClient = await CreateAdminClient(controller.ApiUrl, cancellationToken); var controllerInfo = await controllerClient.ServerInformation(cancellationToken); @@ -544,9 +544,9 @@ public async Task TestSwarmSynchronizationAndUpdates() try { - await using var controllerClient = await CreateAdminClient(controller.Url, cancellationToken); - await using var node1Client = await CreateAdminClient(node1.Url, cancellationToken); - await using var node2Client = await CreateAdminClient(node2.Url, cancellationToken); + await using var controllerClient = await CreateAdminClient(controller.ApiUrl, cancellationToken); + await using var node1Client = await CreateAdminClient(node1.ApiUrl, cancellationToken); + await using var node2Client = await CreateAdminClient(node2.ApiUrl, cancellationToken); var controllerInfo = await controllerClient.ServerInformation(cancellationToken); @@ -615,7 +615,7 @@ await Task.WhenAny( newUser.Name, "asdfasdfasdfasdf"); - await using var node1BadClient = clientFactory.CreateFromToken(node1.Url, controllerUserClient.Token); + await using var node1BadClient = clientFactory.CreateFromToken(node1.RootUrl, controllerUserClient.Token); await ApiAssert.ThrowsException(() => node1BadClient.Administration.Read(cancellationToken)); // check instance info is not shared @@ -685,8 +685,8 @@ void CheckServerUpdated(LiveTestingServer server) controller.Run(cancellationToken).AsTask(), node1.Run(cancellationToken).AsTask()); - await using var controllerClient2 = await CreateAdminClient(controller.Url, cancellationToken); - await using var node1Client2 = await CreateAdminClient(node1.Url, cancellationToken); + await using var controllerClient2 = await CreateAdminClient(controller.ApiUrl, cancellationToken); + await using var node1Client2 = await CreateAdminClient(node1.ApiUrl, cancellationToken); await ApiAssert.ThrowsException(() => controllerClient2.Administration.Update( new ServerUpdateRequest @@ -701,7 +701,7 @@ await ApiAssert.ThrowsException(() = serverTask, node2.Run(cancellationToken).AsTask()); - await using var node2Client2 = await CreateAdminClient(node2.Url, cancellationToken); + await using var node2Client2 = await CreateAdminClient(node2.ApiUrl, cancellationToken); async Task WaitForSwarmServerUpdate2() { @@ -815,9 +815,9 @@ public async Task TestSwarmReconnection() try { - await using var controllerClient = await CreateAdminClient(controller.Url, cancellationToken); - await using var node1Client = await CreateAdminClient(node1.Url, cancellationToken); - await using var node2Client = await CreateAdminClient(node2.Url, cancellationToken); + await using var controllerClient = await CreateAdminClient(controller.ApiUrl, cancellationToken); + await using var node1Client = await CreateAdminClient(node1.ApiUrl, cancellationToken); + await using var node2Client = await CreateAdminClient(node2.ApiUrl, cancellationToken); var controllerInfo = await controllerClient.ServerInformation(cancellationToken); @@ -897,7 +897,7 @@ await Task.WhenAny( Assert.IsTrue(controllerTask.IsCompleted); controllerTask = controller.Run(cancellationToken).AsTask(); - await using var controllerClient2 = await CreateAdminClient(controller.Url, cancellationToken); + await using var controllerClient2 = await CreateAdminClient(controller.ApiUrl, cancellationToken); // node 2 should reconnect once it's health check triggers await Task.WhenAny( @@ -934,7 +934,7 @@ await ApiAssert.ThrowsException( ErrorCode.SwarmIntegrityCheckFailed); node2Task = node2.Run(cancellationToken).AsTask(); - await using var node2Client2 = await CreateAdminClient(node2.Url, cancellationToken); + await using var node2Client2 = await CreateAdminClient(node2.ApiUrl, cancellationToken); // should re-register await Task.WhenAny( @@ -991,7 +991,7 @@ async ValueTask TestTgstation(bool interactive) var serverTask = server.Run(cancellationToken); try { - await using var adminClient = await CreateAdminClient(server.Url, cancellationToken); + await using var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken); var instanceManagerTest = new InstanceManagerTest(adminClient, server.Directory); var instance = await instanceManagerTest.CreateTestInstance("TgTestInstance", cancellationToken); @@ -1273,7 +1273,7 @@ async Task TestTgsInternal(CancellationToken hardCancellationToken) { Api.Models.Instance instance; long initialStaged, initialActive; - await using var firstAdminClient = await CreateAdminClient(server.Url, cancellationToken); + await using var firstAdminClient = await CreateAdminClient(server.ApiUrl, cancellationToken); async ValueTask CreateUserWithNoInstancePerms() { @@ -1291,7 +1291,7 @@ async ValueTask CreateUserWithNoInstancePerms() var user = await firstAdminClient.Users.Create(createRequest, cancellationToken); Assert.IsTrue(user.Enabled); - return await clientFactory.CreateFromLogin(server.Url, createRequest.Name, createRequest.Password, cancellationToken: cancellationToken); + return await clientFactory.CreateFromLogin(server.RootUrl, createRequest.Name, createRequest.Password, cancellationToken: cancellationToken); } var jobsHubTest = new JobsHubTests(firstAdminClient, await CreateUserWithNoInstancePerms()); @@ -1302,7 +1302,7 @@ async ValueTask CreateUserWithNoInstancePerms() // Dump swagger to disk // This is purely for CI using var httpClient = new HttpClient(); - var webRequestTask = httpClient.GetAsync(server.Url.ToString() + "doc/tgs_api.json", cancellationToken); + var webRequestTask = httpClient.GetAsync(server.ApiUrl.ToString() + "doc/tgs_api.json", cancellationToken); using var response = await webRequestTask; response.EnsureSuccessStatusCode(); await using var content = await response.Content.ReadAsStreamAsync(cancellationToken); @@ -1343,7 +1343,7 @@ async Task FailFast(Task task) firstAdminClient.Instances, fileDownloader, GetInstanceManager(), - (ushort)server.Url.Port); + (ushort)server.ApiUrl.Port); async Task RunInstanceTests() { @@ -1415,7 +1415,7 @@ await FailFast( using var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp); blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true); blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false); - blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.Url.Port)); + blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.ApiUrl.Port)); // bind test run await server.Run(cancellationToken); Assert.Fail("Expected server task to end with a SocketException"); @@ -1437,7 +1437,7 @@ await FailFast( // chat bot start and DD reattach test serverTask = server.Run(cancellationToken).AsTask(); - await using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) + await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) { await jobsHubTest.WaitForReconnect(cancellationToken); var instanceClient = adminClient.Instances.CreateClient(instance); @@ -1541,7 +1541,7 @@ async Task WaitForInitialJobs(IInstanceClient instanceClient) serverTask = server.Run(cancellationToken).AsTask(); long expectedCompileJobId, expectedStaged; var edgeByond = await ByondTest.GetEdgeVersion(fileDownloader, cancellationToken); - await using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) + await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) { var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); @@ -1552,7 +1552,7 @@ async Task WaitForInitialJobs(IInstanceClient instanceClient) Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); - var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); + var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1590,7 +1590,7 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest // post/entity deletion tests serverTask = server.Run(cancellationToken).AsTask(); - await using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) + await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) { var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); @@ -1601,7 +1601,7 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); - var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); + var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); currentDD = await wdt.TellWorldToReboot(cancellationToken); Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value); Assert.IsNull(currentDD.StagedCompileJob); @@ -1644,6 +1644,7 @@ await instanceClient.DreamDaemon.Update(new DreamDaemonRequest async Task CreateAdminClient(Uri url, CancellationToken cancellationToken) { + url = new Uri(url.ToString().Replace(Routes.ApiRoot, String.Empty)); var giveUpAt = DateTimeOffset.UtcNow.AddMinutes(2); for (var I = 1; ; ++I) { diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index 41c48a1196d..68ac407ef69 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -1,9 +1,11 @@ using System; using System.IO; using System.IO.Compression; +using System.Globalization; using System.Linq; using System.Net.Http; using System.Reflection; +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using System.Xml.Linq; @@ -27,7 +29,7 @@ using Tgstation.Server.Host.Database; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.System; -using System.Net; +using Tgstation.Server.Host.Controllers; namespace Tgstation.Server.Tests { @@ -380,6 +382,29 @@ static string GetMigrationTimestampString(Type type) => type Assert.AreEqual(latestMigrationSL, DatabaseContext.SLLatestMigration); } + [TestMethod] + public async Task CheckWebRootPathForTgsLogo() + { + var directory = Path.GetFullPath("../../../../../src/Tgstation.Server.Host/wwwroot"); + if (!Directory.Exists(directory)) + Assert.Inconclusive("Webpanel not built?"); + + var logo = new PlatformIdentifier().IsWindows + ? RootController.ProjectLogoSvgRouteWindows + : RootController.ProjectLogoSvgRouteLinux; + + var path = $"../../../../../src/Tgstation.Server.Host/wwwroot{logo}"; + Assert.IsTrue(File.Exists(path)); + + var content = await File.ReadAllBytesAsync(path); + var hash = String.Join(String.Empty, SHA1.HashData(content).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); + Assert.AreEqual( + new PlatformIdentifier().IsWindows + ? "c5e4709774c14a6f376dbb5100bd80a0114a2287" + : "9eba2fac24c5c7e0008721690d07c3df575a00d6", + hash); + } + static async Task> GetByondVersionPriorTo(IByondInstaller byondInstaller, Version version) { var minusOneMinor = new Version(version.Major, version.Minor - 1); From 2413dc84f0d22ddecb46f93544b7cfce4d8e50e7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 10 Nov 2023 23:21:06 -0500 Subject: [PATCH 133/138] Fix .deb `postinst` conditional. --- build/package/deb/debian/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/deb/debian/postinst b/build/package/deb/debian/postinst index 9f71a1931f2..a9ce7b96961 100755 --- a/build/package/deb/debian/postinst +++ b/build/package/deb/debian/postinst @@ -2,7 +2,7 @@ #DEBHELPER# -if [ "$1" = "configure" ]; then +if [ -z "$2" ]; then chmod 600 /etc/tgstation-server deb-systemd-helper stop 'tgstation-server.service' >/dev/null || true From 22bd831c8c89d027eac699db8945e2ca017119fe Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 10 Nov 2023 23:23:19 -0500 Subject: [PATCH 134/138] Only create `tgstation-server` user + other permission fixes on first install. --- build/package/deb/debian/postinst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/build/package/deb/debian/postinst b/build/package/deb/debian/postinst index 9a1cb1b13c0..664ce89a97c 100755 --- a/build/package/deb/debian/postinst +++ b/build/package/deb/debian/postinst @@ -1,10 +1,12 @@ #!/bin/sh -e -adduser --system tgstation-server -mkdir -m 754 -p /var/log/tgstation-server -chown -R tgstation-server /etc/tgstation-server -chown -R tgstation-server /opt/tgstation-server -chown -R tgstation-server /var/log/tgstation-server +if [ -z "$2" ]; then + adduser --system tgstation-server + mkdir -m 754 -p /var/log/tgstation-server + chown -R tgstation-server /etc/tgstation-server + chown -R tgstation-server /opt/tgstation-server + chown -R tgstation-server /var/log/tgstation-server +fi #DEBHELPER# From 1527900571b25393a57c6dc28e81212cbf43f4c4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 10 Nov 2023 23:23:46 -0500 Subject: [PATCH 135/138] Fix .deb installation directory ownership --- build/package/deb/debian/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/deb/debian/postinst b/build/package/deb/debian/postinst index 664ce89a97c..a40084e5eb6 100755 --- a/build/package/deb/debian/postinst +++ b/build/package/deb/debian/postinst @@ -4,7 +4,7 @@ if [ -z "$2" ]; then adduser --system tgstation-server mkdir -m 754 -p /var/log/tgstation-server chown -R tgstation-server /etc/tgstation-server - chown -R tgstation-server /opt/tgstation-server + chown -R tgstation-server /opt/tgstation-server/lib chown -R tgstation-server /var/log/tgstation-server fi From 30293c3e39052ead17c1a67ee2b549f6409ad9eb Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 11 Nov 2023 21:31:07 -0500 Subject: [PATCH 136/138] Fix repository fetching possibly not fetching all tags --- .../Components/Repository/Repository.cs | 1 + .../Tgstation.Server.Tests/TestRepository.cs | 80 ++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 46cc06fd8e5..ea7844c2a80 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -450,6 +450,7 @@ await Task.Factory.StartNew( OnTransferProgress = TransferProgressHandler(progressReporter.CreateSection("Fetch Origin", 1.0), cancellationToken), OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), + TagFetchMode = TagFetchMode.All, }, "Fetch origin commits"); } diff --git a/tests/Tgstation.Server.Tests/TestRepository.cs b/tests/Tgstation.Server.Tests/TestRepository.cs index e03fbe984a7..d0549409f9c 100644 --- a/tests/Tgstation.Server.Tests/TestRepository.cs +++ b/tests/Tgstation.Server.Tests/TestRepository.cs @@ -1,8 +1,11 @@ using System; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using LibGit2Sharp; + using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -28,7 +31,7 @@ public async Task TestRepoParentLookup() { LibGit2Sharp.Repository.Clone("https://github.com/Cyberboss/test", tempPath); var libGit2Repo = new LibGit2Sharp.Repository(tempPath); - using var repo = new Repository( + using var repo = new Host.Components.Repository.Repository( libGit2Repo, new LibGit2Commands(), Mock.Of(), @@ -36,7 +39,7 @@ public async Task TestRepoParentLookup() Mock.Of(), Mock.Of(), Mock.Of(), - Mock.Of>(), + Mock.Of>(), new GeneralConfiguration(), () => { }); @@ -56,5 +59,78 @@ public async Task TestRepoParentLookup() CancellationToken.None); } } + + [TestMethod] + public async Task TestFetchingAdditionalCommits() + { + var tempPath = Path.Combine(Path.GetTempPath(), "TGS-Repository-Integration-Test", Guid.NewGuid().ToString()); + var repoFac = + new LibGit2RepositoryFactory( + Mock.Of>()); + var commands = new LibGit2Commands(); + using var manager = new RepositoryManager( + repoFac, + commands, + new ResolvingIOManager( + new DefaultIOManager(), + tempPath), + Mock.Of(), + new WindowsPostWriteHandler(), + Mock.Of(), + Mock.Of>(), + Mock.Of>(), + new GeneralConfiguration()); + try + { + using (await manager.CloneRepository( + new Uri("https://github.com/Cyberboss/common_core"), + null, + null, + null, + new JobProgressReporter(Mock.Of>(), null, (_, _) => { }), + true, + default)) + { + } + + using (var repo = await repoFac.CreateFromPath(tempPath, default)) + { + repo.Network.Remotes.Update("origin", updater => + { + updater.Url = "https://github.com/tgstation/common_core"; + }); + + var targetCommit = repo.Lookup("5b0d0a38057a2c8306a852ccbd6cd6f4ae766a33"); + Assert.IsNull(targetCommit); + } + + using (var repo2 = await manager.LoadRepository(default)) + { + await repo2.FetchOrigin( + new JobProgressReporter(Mock.Of>(), null, (_, _) => { }), + null, + null, + false, + default); + } + + using var repo3 = await repoFac.CreateFromPath(tempPath, default); + var remote = repo3.Network.Remotes.First(); + commands.Fetch(repo3, remote.FetchRefSpecs.Select(x => x.Specification), remote, new FetchOptions + { + TagFetchMode = TagFetchMode.All, + Prune = true, + }, "test"); + + var targetCommit2 = repo3.Lookup("5b0d0a38057a2c8306a852ccbd6cd6f4ae766a33"); + Assert.IsNotNull(targetCommit2); + } + finally + { + await new DefaultIOManager().DeleteDirectory( + Path.GetDirectoryName(tempPath), + CancellationToken.None); + } + } } } From 9ea468b8244bda363ccff8c1934598318532da49 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 11 Nov 2023 21:31:38 -0500 Subject: [PATCH 137/138] Version bump to 5.17.2 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index ae1f43929d8..c79cd908222 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 5.17.1 + 5.17.2 4.7.1 9.13.0 7.0.0 From a252c5f95a108a2244fd11ade9ec3d2abc96b1ef Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 11 Nov 2023 21:50:38 -0500 Subject: [PATCH 138/138] Update dotnet redistributable to latest --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index c79cd908222..28f434af34b 100644 --- a/build/Version.props +++ b/build/Version.props @@ -17,7 +17,7 @@ netstandard2.0 6 - https://dotnetcli.azureedge.net/dotnet/aspnetcore/Runtime/6.0.23/dotnet-hosting-6.0.23-win.exe + https://dotnetcli.azureedge.net/dotnet/aspnetcore/Runtime/6.0.24/dotnet-hosting-6.0.24-win.exe 10.11.5 https://ftp.osuosl.org/pub/mariadb//mariadb-10.11.5/winx64-packages/mariadb-10.11.5-winx64.msi