diff --git a/Docs/QuasarArchitecture.md b/Docs/QuasarArchitecture.md index 20c0d5a..691f701 100644 --- a/Docs/QuasarArchitecture.md +++ b/Docs/QuasarArchitecture.md @@ -493,6 +493,9 @@ Linux-first cutover ownership: - activation promotes the staged payload into `ManagedRuntime/WebService//` and writes `Updates/active-release.json` - Bootstrap observes the pointer change, drains the old worker, then starts the managed worker +- UI-plugin install/update/remove restarts write `Updates/worker-restart-request.json`; + Bootstrap consumes it, drains the old worker, then starts the same active release + so dynamic plugin assemblies are loaded from the updated install root - the browser and `Quasar.Agent` reconnect after the short listener gap - retiring workers only delete the discovery manifest if the on-disk worker id and process id still match themselves, so an old worker cannot remove the new @@ -919,7 +922,7 @@ As of this document: - deep per-server profiler telemetry now exists: `Quasar.Agent` runs a continuous in-process profiler with `SafeContinuous` enabled by default, with per-server persisted `AgentProfilerMode` values and a global `Quasar:AgentProfilerMode` / `QUASAR_AGENT_PROFILER_MODE` fallback for older definitions. Safe mode uses Harmony prefix/postfix timing only for named high-level paths: frame/update, programmable-block script, physics, replication/network/session, GPS, and block-limit work. It deliberately avoids broad entity update method patching and detailed network-event hooks so the always-on default stays low overhead. Deep mode adds detailed network-event method hooks plus Magnetar-compatible Harmony IL call-site transpilers for `MySession.Update` / `UpdateComponents`, session component calls, replication simulation, entity update dispatch, parallel waits/callbacks, and Havok physics stepping internals. Runtime mode changes reconfigure Harmony patches so Safe, Deep, and Off can be selected without restarting the server. Hot-path measurements use numeric call-site ids and rolling accumulators, split main-thread vs off-thread time, and publish one-second windows with bounded top-lists for grids, scripts, entity types, system methods, physics detail, and network/replication/session work where the active patch depth can observe them. Patch failures are logged and the agent keeps the remaining profiler surface; entity call-site misses stay at high-level timing rather than adding broad method wrapping. Each `ProfilerSnapshot` rides the regular agent snapshot, is validated, and is kept in a small recent in-memory `ProfilerStoreService` ring (~720 samples per server, about 12 minutes at one snapshot per second), then surfaced on the Analytics page as game-loop timing and top grid/entity-type chart panels - Discord per-server options now include chat relay and simspeed alert rules. Discord-to-game chat is injected as `[Discord] : ` so in-game readers see the Discord sender, and `DiscordChatRelayService` suppresses the matching game-history echo before it can post back to Discord as the server/bot author. `DiscordSimSpeedAlertService` evaluates fresh raw metric samples for connected/running agents on the registry change path, sending alerts through the configured simspeed channel or the server's analytics channel. Baseline rules detect sharp sample-to-sample drops across every unseen raw sample pair and sustained low average simspeed, and the Discord page exposes thresholds, windows, cooldowns, and per-rule enable switches. `DiscordBotService` also publishes aggregate managed-server state through Discord presence: the bot status reflects unhealthy/faulted vs active vs idle server instances, and its activity text shows active/total servers, player count, and issue/warning counts. - a unified GitHub-release-based update/publish pipeline now exists covering both Linux and Windows in a single combined release (`.github/workflows/release.yml`): each build produces `quasar-installer-linux.tar.gz` / `quasar-web-linux-x64.tar.gz` (Linux) and `quasar-installer-windows.zip` / `quasar-web-win-x64.zip` (Windows) under one tag; tag pushes and `main` publish full releases while pull requests publish draft prereleases; installer archives contain a single top-level `Quasar` directory for clean manual extraction; the release carries one combined `SHA256SUMS` covering every archive; release identity is normalized from `AssemblyInformationalVersion` and the active-release pointer (not numeric `AssemblyVersion`); four-part build tags such as `1.0.0.37` are canonical; every downloaded asset is verified against `SHA256SUMS`; the UI stages web updates and queues them for explicit activation from `/settings/updates`; Bootstrap self-upgrades from the launcher stream only when an actually-newer asset appears (see [Linux Deployment and Updates](LinuxDeploymentAndUpdates.md) and [Windows Deployment and Updates](WindowsDeploymentAndUpdates.md)) -- `Quasar.Bootstrap` runs as the stable launcher that owns the public port on both Linux (systemd service) and Windows (Scheduled Task): it activates web releases through the `Updates/active-release.json` pointer after staged payloads are promoted into `ManagedRuntime/WebService//`, and performs worker cutover by draining the old worker and starting the managed one on the same port — a launcher, not yet a reverse proxy — so the public endpoint stays stable across the short listener gap while managed Magnetar servers keep running; the UI shutdown action writes a launcher drain request, stops the worker, and leaves Bootstrap alive without respawning a worker until the service, task, or foreground launcher is restarted; on Linux the launcher exits with code 75 so systemd restarts it for self-upgrade; on Windows the launcher spawns a detached replacement `Quasar.exe serve --quiet` and exits 0, with the Scheduled Task restart-on-failure as the safety net +- `Quasar.Bootstrap` runs as the stable launcher that owns the public port on both Linux (systemd service) and Windows (Scheduled Task): it activates web releases through the `Updates/active-release.json` pointer after staged payloads are promoted into `ManagedRuntime/WebService//`, and performs worker cutover by draining the old worker and starting the managed one on the same port — a launcher, not yet a reverse proxy — so the public endpoint stays stable across the short listener gap while managed Magnetar servers keep running; UI plugin restarts write `Updates/worker-restart-request.json` so Bootstrap performs the same drain/start sequence for the current active release; the UI shutdown action writes a launcher drain request, stops the worker, and leaves Bootstrap alive without respawning a worker until the service, task, or foreground launcher is restarted; on Linux the launcher exits with code 75 so systemd restarts it for self-upgrade; on Windows the launcher spawns a detached replacement `Quasar.exe serve --quiet` and exits 0, with the Scheduled Task restart-on-failure as the safety net - Windows deployment exists via `install.ps1`/`uninstall.ps1`: extracted release installs default to the installer root, source installs default to `%ProgramFiles%\Quasar`, and the installer registers a Scheduled Task (`Quasar`) that starts at boot and restarts the launcher on failure; the task runs Bootstrap directly with `serve --quiet --service` - staged relaunch now persists supervisor runtime state so managed DS processes survive worker turnover - obsolete `webui/` is removed from the repository diff --git a/Docs/QuasarPluginSystem.md b/Docs/QuasarPluginSystem.md index 4160cdc..34244e6 100644 --- a/Docs/QuasarPluginSystem.md +++ b/Docs/QuasarPluginSystem.md @@ -221,11 +221,14 @@ Install, update, and remove operations change files on disk only. The active plugin assembly set is still loaded at Quasar worker startup, so each operation requires a Quasar restart before it takes effect. When that restart is triggered from the UI Plugins page, the browser shows restart progress and polls -`/api/health` until the replacement worker is ready. The restart watcher is -stored in browser session storage, so refreshing the page resumes the same -progress overlay and health polling. Managed Space Engineers server processes -stay detached during the Quasar worker restart and are adopted as running by the -replacement worker when their process id is still alive. +`/api/health` until the replacement worker is ready. The worker writes +`Updates/worker-restart-request.json`; Bootstrap consumes that request, drains +the current worker through `/api/internal/drain`, and starts the same active +release again so the plugin assembly set is rebuilt from disk. The restart +watcher is stored in browser session storage, so refreshing the page resumes the +same progress overlay and health polling. Managed Space Engineers server +processes stay detached during the Quasar worker restart and are adopted as +running by the replacement worker when their process id is still alive. QuasarHub descriptors can set `true` for reviewed plugins that should be present by default. Quasar installs or updates diff --git a/Docs/StateMachines/SelfUpdateAndRelease.md b/Docs/StateMachines/SelfUpdateAndRelease.md index 8f4c46a..2bca70a 100644 --- a/Docs/StateMachines/SelfUpdateAndRelease.md +++ b/Docs/StateMachines/SelfUpdateAndRelease.md @@ -65,10 +65,11 @@ stateDiagram-v2 WorkerLaunching --> ForceKilled: health timeout WorkerHealthy --> Running: serving the public port Running --> Draining: active-release.json changed + Running --> Draining: worker-restart-request.json consumed Draining --> Retired: worker exits within grace window Draining --> ForceKilled: grace timeout -> Kill - Retired --> WorkerLaunching: start new-release worker - ForceKilled --> WorkerLaunching: start new-release worker + Retired --> WorkerLaunching: start target worker + ForceKilled --> WorkerLaunching: start target worker Running --> Restarting: worker exits unexpectedly Restarting --> WorkerLaunching: relaunch (force) Running --> Drained: UI Shutdown Quasar / launcher drain request @@ -84,7 +85,7 @@ stateDiagram-v2 | `NoWorker` | Startup before a worker exists; may download an initial UI worker. | | `WorkerLaunching` | Worker process started; polling `/api/health` (up to 60s). | | `WorkerHealthy` / `Running` | Worker serving the public port. | -| `Draining` | Pointer change detected; the launcher posts `/api/internal/drain` (authenticated with the per-session launcher token) and waits for graceful exit. | +| `Draining` | Pointer change or `Updates/worker-restart-request.json` detected; the launcher posts `/api/internal/drain` (authenticated with the per-session launcher token) and waits for graceful exit. Pointer changes start the new active release; worker-restart requests start the same active release again. | | `Retired` / `ForceKilled` | Old worker exited within the grace window, or was killed after timeout. | | `Restarting` | Worker exited unexpectedly (not a launcher request); relaunched with `force`. | | `Drained` | UI **Shutdown Quasar** requested a launcher drain; Bootstrap stays alive without respawning a worker until the service, task, or foreground launcher is restarted. | diff --git a/Docs/StateMachines/diagrams/bootstrap-cutover.mmd b/Docs/StateMachines/diagrams/bootstrap-cutover.mmd index 4c51f42..4e08833 100644 --- a/Docs/StateMachines/diagrams/bootstrap-cutover.mmd +++ b/Docs/StateMachines/diagrams/bootstrap-cutover.mmd @@ -5,10 +5,11 @@ stateDiagram-v2 WorkerLaunching --> ForceKilled: health timeout WorkerHealthy --> Running: serving the public port Running --> Draining: active-release.json changed + Running --> Draining: worker-restart-request.json consumed Draining --> Retired: worker exits within grace window Draining --> ForceKilled: grace timeout -> Kill - Retired --> WorkerLaunching: start new-release worker - ForceKilled --> WorkerLaunching: start new-release worker + Retired --> WorkerLaunching: start target worker + ForceKilled --> WorkerLaunching: start target worker Running --> Restarting: worker exits unexpectedly Restarting --> WorkerLaunching: relaunch (force) Running --> Drained: UI Shutdown Quasar / launcher drain request diff --git a/Docs/StateMachines/diagrams/bootstrap-cutover.png b/Docs/StateMachines/diagrams/bootstrap-cutover.png index 0ee82f0..aacb758 100644 Binary files a/Docs/StateMachines/diagrams/bootstrap-cutover.png and b/Docs/StateMachines/diagrams/bootstrap-cutover.png differ diff --git a/Magnetar.Protocol/Runtime/MagnetarPaths.cs b/Magnetar.Protocol/Runtime/MagnetarPaths.cs index ad5e2db..5fdb33b 100644 --- a/Magnetar.Protocol/Runtime/MagnetarPaths.cs +++ b/Magnetar.Protocol/Runtime/MagnetarPaths.cs @@ -171,6 +171,9 @@ public static string GetQuasarAppSettingsBasePath() => public static string GetQuasarBootstrapUpdateRequestPath() => Path.Combine(GetQuasarUpdatesDirectory(), "bootstrap-update-request.json"); + public static string GetQuasarWorkerRestartRequestPath() => + Path.Combine(GetQuasarUpdatesDirectory(), "worker-restart-request.json"); + // ------------------------------------------------------------------------- // Managed runtime (auto-downloaded Magnetar + DS install) // ------------------------------------------------------------------------- diff --git a/Quasar.Bootstrap/Program.cs b/Quasar.Bootstrap/Program.cs index c593a22..50a41c0 100644 --- a/Quasar.Bootstrap/Program.cs +++ b/Quasar.Bootstrap/Program.cs @@ -802,6 +802,8 @@ internal sealed class LauncherCoordinator : IHostedService, IDisposable private Task? _bootstrapUpdateMonitorTask; private FileSystemWatcher? _bootstrapUpdateRequestWatcher; private CancellationTokenSource? _bootstrapUpdateRequestDebounce; + private FileSystemWatcher? _workerRestartRequestWatcher; + private CancellationTokenSource? _workerRestartRequestDebounce; private readonly SemaphoreSlim _bootstrapUpdateLock = new(1, 1); private WorkerProcessHandle? _currentWorker; private bool _isStopping; @@ -885,11 +887,14 @@ public async Task StartAsync(CancellationToken cancellationToken) Directory.CreateDirectory(MagnetarPaths.GetQuasarUpdatesDirectory()); if (TryConsumeLauncherShutdownRequest()) _logger.LogInformation("Cleared stale Quasar launcher drain request on startup."); + if (TryConsumeWorkerRestartRequest()) + _logger.LogInformation("Cleared stale Quasar worker restart request on startup."); await EnsureInitialWebReleaseAvailableAsync(cancellationToken).ConfigureAwait(false); EnsureActiveReleasePointerExists(); await ActivateCurrentReleaseAsync(force: false, cancellationToken).ConfigureAwait(false); StartWatchingReleasePointer(); + StartWatchingWorkerRestartRequests(); StartBootstrapUpdateMonitor(); StartWatchingBootstrapUpdateRequests(); } @@ -904,6 +909,9 @@ public async Task StopAsync(CancellationToken cancellationToken) _bootstrapUpdateRequestWatcher?.Dispose(); _bootstrapUpdateRequestDebounce?.Cancel(); _bootstrapUpdateRequestDebounce?.Dispose(); + _workerRestartRequestWatcher?.Dispose(); + _workerRestartRequestDebounce?.Cancel(); + _workerRestartRequestDebounce?.Dispose(); WorkerProcessHandle? worker; lock (_sync) @@ -937,6 +945,9 @@ public void Dispose() _bootstrapUpdateRequestWatcher?.Dispose(); _bootstrapUpdateRequestDebounce?.Cancel(); _bootstrapUpdateRequestDebounce?.Dispose(); + _workerRestartRequestWatcher?.Dispose(); + _workerRestartRequestDebounce?.Cancel(); + _workerRestartRequestDebounce?.Dispose(); _activationLock.Dispose(); _bootstrapUpdateLock.Dispose(); _healthClient.Dispose(); @@ -954,6 +965,64 @@ private void StartBootstrapUpdateMonitor() CancellationToken.None); } + private void StartWatchingWorkerRestartRequests() + { + var path = MagnetarPaths.GetQuasarWorkerRestartRequestPath(); + var directory = Path.GetDirectoryName(path)!; + Directory.CreateDirectory(directory); + + _workerRestartRequestWatcher = new FileSystemWatcher(directory) + { + Filter = Path.GetFileName(path), + NotifyFilter = NotifyFilters.FileName | NotifyFilters.LastWrite | NotifyFilters.CreationTime | NotifyFilters.Size, + }; + + _workerRestartRequestWatcher.Changed += HandleWorkerRestartRequestChanged; + _workerRestartRequestWatcher.Created += HandleWorkerRestartRequestChanged; + _workerRestartRequestWatcher.Renamed += HandleWorkerRestartRequestChanged; + _workerRestartRequestWatcher.EnableRaisingEvents = true; + + if (File.Exists(path)) + QueueWorkerRestartRequest(); + } + + private void HandleWorkerRestartRequestChanged(object sender, FileSystemEventArgs args) + { + QueueWorkerRestartRequest(); + } + + private void QueueWorkerRestartRequest() + { + CancellationTokenSource debounce; + lock (_sync) + { + _workerRestartRequestDebounce?.Cancel(); + _workerRestartRequestDebounce?.Dispose(); + _workerRestartRequestDebounce = new CancellationTokenSource(); + debounce = _workerRestartRequestDebounce; + } + + _ = Task.Run(async () => + { + try + { + await Task.Delay(TimeSpan.FromMilliseconds(250), debounce.Token); + if (!TryConsumeWorkerRestartRequest()) + return; + + _logger.LogInformation("Quasar worker restart requested by current worker."); + await RestartCurrentWorkerAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (OperationCanceledException) when (debounce.IsCancellationRequested) + { + } + catch (Exception exception) + { + _logger.LogWarning(exception, "Requested Quasar worker restart failed."); + } + }, CancellationToken.None); + } + private void StartWatchingBootstrapUpdateRequests() { if (!_options.UpdatesEnabled || (!OperatingSystem.IsLinux() && !OperatingSystem.IsWindows())) @@ -1630,6 +1699,64 @@ await DrainAndRetireWorkerAsync( } } + private async Task RestartCurrentWorkerAsync(CancellationToken cancellationToken) + { + if (IsDrained()) + return; + + await _activationLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var pointer = ReadActiveReleasePointer(); + if (pointer is null) + return; + + pointer = Normalize(pointer); + + WorkerProcessHandle? current; + lock (_sync) + { + if (_isDrained) + return; + + current = _currentWorker; + if (current is not null && ReferenceEquals(_currentWorker, current)) + _currentWorker = null; + } + + if (current is not null && !current.Process.HasExited) + { + await DrainAndRetireWorkerAsync( + current, + TimeSpan.FromSeconds(2), + stopManagedServers: false, + cancellationToken).ConfigureAwait(false); + } + + if (IsDrained()) + return; + + var nextWorker = await StartWorkerAsync(pointer, cancellationToken).ConfigureAwait(false); + if (nextWorker is null) + { + _logger.LogWarning("Quasar worker restart request could not start active release {Version}.", pointer.Version); + return; + } + + lock (_sync) + { + _currentWorker = nextWorker; + } + + CleanupInactiveManagedWebReleases(pointer.WorkingDirectory); + _logger.LogInformation("Restarted Quasar worker version {Version} at {BaseUri}.", pointer.Version, nextWorker.BaseUri); + } + finally + { + _activationLock.Release(); + } + } + private async Task ActivateSpecificReleaseAsync(QuasarActiveReleasePointer pointer, CancellationToken cancellationToken) { if (IsDrained()) @@ -1988,6 +2115,23 @@ private static bool TryConsumeLauncherShutdownRequest() return true; } + private static bool TryConsumeWorkerRestartRequest() + { + var path = MagnetarPaths.GetQuasarWorkerRestartRequestPath(); + if (!File.Exists(path)) + return false; + + try + { + File.Delete(path); + } + catch + { + } + + return true; + } + private static void TryDisposeProcess(Process process) { try diff --git a/Quasar/Services/QuasarShutdownService.cs b/Quasar/Services/QuasarShutdownService.cs index 343ddec..893f504 100644 --- a/Quasar/Services/QuasarShutdownService.cs +++ b/Quasar/Services/QuasarShutdownService.cs @@ -1,3 +1,4 @@ +using System.Text.Json; using Magnetar.Protocol.Runtime; using Quasar.Models; @@ -9,6 +10,8 @@ namespace Quasar.Services; /// public sealed class QuasarShutdownService { + private static readonly TimeSpan LauncherRestartFallbackDelay = TimeSpan.FromSeconds(8); + private readonly IHostApplicationLifetime _lifetime; private readonly DedicatedServerSupervisor _supervisor; private readonly WebServiceOptions _options; @@ -95,9 +98,16 @@ public void RestartWorker(IProgress? progress = null) { // BeginLauncherDrain marks the supervisor to preserve servers on this stop and // persists the runtime snapshot (including PIDs) so the next worker can adopt - // them; StopApplication then exits the worker for the launcher to respawn. + // them. progress?.Report("Restarting Quasar worker…"); _supervisor.BeginLauncherDrain(); + + if (TryRequestLauncherWorkerRestart()) + { + ScheduleRestartFallback(); + return; + } + _lifetime.StopApplication(); } @@ -131,4 +141,63 @@ private void RequestLauncherShutdown() private static string GetLauncherShutdownRequestPath() => Path.Combine(MagnetarPaths.GetQuasarDirectory(), "launcher-shutdown-request"); + + private bool TryRequestLauncherWorkerRestart() + { + if (string.IsNullOrWhiteSpace(_options.LauncherToken)) + return false; + + var path = MagnetarPaths.GetQuasarWorkerRestartRequestPath(); + try + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var json = JsonSerializer.Serialize(new + { + requestedAtUtc = DateTimeOffset.UtcNow, + reason = "worker-restart", + processId = Environment.ProcessId, + }); + File.WriteAllText(path, json); + _logger.LogInformation("Requested Quasar worker restart through Bootstrap at {Path}.", path); + return true; + } + catch (Exception exception) + { + _logger.LogWarning(exception, "Failed writing Quasar worker restart request."); + return false; + } + } + + private void ScheduleRestartFallback() + { + _ = Task.Run(async () => + { + try + { + await Task.Delay(LauncherRestartFallbackDelay).ConfigureAwait(false); + TryDeleteWorkerRestartRequest(); + _logger.LogWarning( + "Bootstrap did not consume the Quasar worker restart request within {DelaySeconds:0} seconds; stopping worker directly.", + LauncherRestartFallbackDelay.TotalSeconds); + _lifetime.StopApplication(); + } + catch (Exception exception) + { + _logger.LogWarning(exception, "Quasar worker restart fallback failed."); + } + }); + } + + private static void TryDeleteWorkerRestartRequest() + { + try + { + var path = MagnetarPaths.GetQuasarWorkerRestartRequestPath(); + if (File.Exists(path)) + File.Delete(path); + } + catch + { + } + } }