diff --git a/source/Calamari.Tests/KubernetesFixtures/Commands/KubernetesVerifyResourcesCommandFixture.cs b/source/Calamari.Tests/KubernetesFixtures/Commands/KubernetesVerifyResourcesCommandFixture.cs new file mode 100644 index 0000000000..f712c255ed --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/Commands/KubernetesVerifyResourcesCommandFixture.cs @@ -0,0 +1,89 @@ +using System; +using Calamari.Common.Commands; +using Calamari.Common.Features.Processes; +using Calamari.Common.Plumbing.Variables; +using Calamari.Kubernetes; +using Calamari.Kubernetes.Commands; +using Calamari.Kubernetes.Integration; +using Calamari.Kubernetes.ResourceStatus; +using Calamari.Testing.Helpers; +using Calamari.Tests.Fixtures.Integration.FileSystem; +using FluentAssertions; +using NSubstitute; +using NUnit.Framework; + +namespace Calamari.Tests.KubernetesFixtures.Commands +{ + [TestFixture] + public class KubernetesVerifyResourcesCommandFixture + { + [Test] + public void WhenNoAppliedResourcesVariableIsSet_ShouldThrowCommandException() + { + var variables = new CalamariVariables(); + var statusReporter = Substitute.For(); + var command = CreateCommand(variables, statusReporter, new InMemoryLog()); + + Action execute = () => command.Execute(new string[] { }); + + execute.Should() + .Throw() + .WithMessage("The applied resources variable was not found. This variable is required to verify the deployed resources."); + statusReporter.ReceivedCalls().Should().BeEmpty(); + } + + [Test] + public void WhenAppliedResourcesIsAnEmptyList_ShouldDoNothingAndSucceed() + { + var variables = new CalamariVariables + { + [SpecialVariables.AppliedResources] = "[]" + }; + var statusReporter = Substitute.For(); + var log = new InMemoryLog(); + var command = CreateCommand(variables, statusReporter, log); + + var result = command.Execute(new string[] { }); + + result.Should().Be(0); + statusReporter.ReceivedCalls().Should().BeEmpty(); + log.MessagesInfoFormatted.Should().Contain("Applied resources list is empty; nothing to verify."); + } + + [Test] + public void WhenAppliedResourcesIsNotValidJson_ShouldThrowCommandException() + { + var variables = new CalamariVariables + { + [SpecialVariables.AppliedResources] = "this is not json" + }; + var statusReporter = Substitute.For(); + var command = CreateCommand(variables, statusReporter, new InMemoryLog()); + + Action execute = () => command.Execute(new string[] { }); + + execute.Should() + .Throw() + .WithMessage("Could not parse applied resources output variable:*"); + statusReporter.ReceivedCalls().Should().BeEmpty(); + } + + static KubernetesVerifyResourcesCommand CreateCommand( + IVariables variables, + IResourceStatusReportExecutor statusReporter, + InMemoryLog log) + { + var fileSystem = new TestCalamariPhysicalFileSystem(); + var commandLineRunner = Substitute.For(); + var kubectl = new Kubectl(variables, log, commandLineRunner); + + return new KubernetesVerifyResourcesCommand( + log, + variables, + fileSystem, + commandLineRunner, + kubectl, + statusReporter); + } + } +} diff --git a/source/Calamari/Kubernetes/Commands/KubernetesApplyRawYamlCommand.cs b/source/Calamari/Kubernetes/Commands/KubernetesApplyRawYamlCommand.cs index efe027a246..653006719e 100644 --- a/source/Calamari/Kubernetes/Commands/KubernetesApplyRawYamlCommand.cs +++ b/source/Calamari/Kubernetes/Commands/KubernetesApplyRawYamlCommand.cs @@ -4,6 +4,7 @@ using Calamari.Common.Features.Packages; using Calamari.Common.Features.StructuredVariables; using Calamari.Common.Features.Substitutions; +using Calamari.Common.FeatureToggles; using Calamari.Common.Plumbing.Deployment.Journal; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; @@ -44,14 +45,17 @@ public KubernetesApplyRawYamlCommand( protected override async Task ExecuteCommand(RunningDeployment runningDeployment) { - if (!variables.GetFlag(SpecialVariables.ResourceStatusCheck)) + //When ArgoRollouts support is enabled, status checking is performed by a separate verification action + var argoRolloutsEnabled = OctopusFeatureToggles.ArgoRolloutsSupportFeatureToggle.IsEnabled(variables); + + if (argoRolloutsEnabled || !variables.GetFlag(SpecialVariables.ResourceStatusCheck)) { return await kubernetesApplyExecutor.Execute(runningDeployment); } var timeoutSeconds = variables.GetInt32(SpecialVariables.Timeout) ?? 0; var waitForJobs = variables.GetFlag(SpecialVariables.WaitForJobs); - + var statusCheck = statusReporter.Start(timeoutSeconds, waitForJobs); return await kubernetesApplyExecutor.Execute(runningDeployment, (newResources) => statusCheck.AddResources(newResources)) && diff --git a/source/Calamari/Kubernetes/Commands/KubernetesKustomizeCommand.cs b/source/Calamari/Kubernetes/Commands/KubernetesKustomizeCommand.cs index 94d04b69af..28dfefbca7 100644 --- a/source/Calamari/Kubernetes/Commands/KubernetesKustomizeCommand.cs +++ b/source/Calamari/Kubernetes/Commands/KubernetesKustomizeCommand.cs @@ -5,6 +5,7 @@ using Calamari.Common.Features.Packages; using Calamari.Common.Features.StructuredVariables; using Calamari.Common.Features.Substitutions; +using Calamari.Common.FeatureToggles; using Calamari.Common.Plumbing.Deployment.Journal; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; @@ -52,7 +53,10 @@ public KubernetesKustomizeCommand( protected override async Task ExecuteCommand(RunningDeployment runningDeployment) { - if (!variables.GetFlag(SpecialVariables.ResourceStatusCheck)) + //When ArgoRollouts support is enabled, status checking is performed by a separate verification action + var argoRolloutsEnabled = OctopusFeatureToggles.ArgoRolloutsSupportFeatureToggle.IsEnabled(variables); + + if (argoRolloutsEnabled || !variables.GetFlag(SpecialVariables.ResourceStatusCheck)) { return await kubernetesApplyExecutor.Execute(runningDeployment); } diff --git a/source/Calamari/Kubernetes/Commands/KubernetesVerifyResourcesCommand.cs b/source/Calamari/Kubernetes/Commands/KubernetesVerifyResourcesCommand.cs new file mode 100644 index 0000000000..3a7f37bd0a --- /dev/null +++ b/source/Calamari/Kubernetes/Commands/KubernetesVerifyResourcesCommand.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Calamari.Aws.Integration; +using Calamari.Commands.Support; +using Calamari.Common.Commands; +using Calamari.Common.Features.Processes; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Logging; +using Calamari.Common.Plumbing.Variables; +using Calamari.Deployment; +using Calamari.Deployment.Conventions; +using Calamari.Kubernetes.Conventions; +using Calamari.Kubernetes.Integration; +using Calamari.Kubernetes.ResourceStatus; +using Calamari.Kubernetes.ResourceStatus.Resources; +using Newtonsoft.Json; + +namespace Calamari.Kubernetes.Commands +{ + [Command(Name, Description = "Verifies that resources applied by a Kubernetes deploy step have reached their desired state")] + public class KubernetesVerifyResourcesCommand : Command + { + public const string Name = "kubernetes-verify-resources"; + + readonly ILog log; + readonly IVariables variables; + readonly ICalamariFileSystem fileSystem; + readonly ICommandLineRunner commandLineRunner; + readonly Kubectl kubectl; + readonly IResourceStatusReportExecutor statusReporter; + + public KubernetesVerifyResourcesCommand( + ILog log, + IVariables variables, + ICalamariFileSystem fileSystem, + ICommandLineRunner commandLineRunner, + Kubectl kubectl, + IResourceStatusReportExecutor statusReporter) + { + this.log = log; + this.variables = variables; + this.fileSystem = fileSystem; + this.commandLineRunner = commandLineRunner; + this.kubectl = kubectl; + this.statusReporter = statusReporter; + } + + public override int Execute(string[] commandLineArguments) + { + Options.Parse(commandLineArguments); + + var json = variables.Get(SpecialVariables.AppliedResources); + if (string.IsNullOrWhiteSpace(json)) + { + throw new CommandException($"The applied resources variable was not found. This variable is required to verify the deployed resources."); + } + + List resources; + try + { + resources = DeserializeResources(json); + } + catch (JsonException ex) + { + throw new CommandException($"Could not parse applied resources output variable: {ex.Message}"); + } + + if (resources.Count == 0) + { + log.Info("Applied resources list is empty; nothing to verify."); + return 0; + } + + WarnForUnverifiableResources(resources); + + AuthenticateKubectl(); + + var timeoutSeconds = variables.GetInt32(SpecialVariables.Timeout) ?? 0; + var waitForJobs = variables.GetFlag(SpecialVariables.WaitForJobs); + + var statusCheck = statusReporter.Start(timeoutSeconds, waitForJobs, resources); + var success = statusCheck.WaitForCompletionOrTimeout(CancellationToken.None).GetAwaiter().GetResult(); + + if (!success) + { + throw new CommandException("Resource verification failed. Check verbose logs for more details."); + } + + return 0; + } + + static List DeserializeResources(string json) + { + var raw = JsonConvert.DeserializeObject>(json) ?? new List(); + return raw + .Select(r => new ResourceIdentifier( + new ResourceGroupVersionKind(r.Group ?? string.Empty, r.Version, r.Kind), + r.Name, + r.Namespace)) + .ToList(); + } + + void WarnForUnverifiableResources(IEnumerable resources) + { + foreach (var resource in resources) + { + var gvk = resource.GroupVersionKind; + if (ResourceFactory.IsVerifiable(gvk)) + continue; + + var name = string.IsNullOrEmpty(resource.Namespace) ? resource.Name : $"{resource.Namespace}/{resource.Name}"; + log.Warn($"Unable to fully verify resource '{gvk}' '{name}'. Calamari does not know the readiness criteria for this resource type; only its existence will be confirmed."); + } + } + + void AuthenticateKubectl() + { + var deployment = new RunningDeployment(variables); + kubectl.SetWorkingDirectory(deployment.CurrentDirectory); + kubectl.SetEnvironmentVariables(deployment.EnvironmentVariables); + + var conventions = new List(); + if (variables.Get(Deployment.SpecialVariables.Account.AccountType) == "AmazonWebServicesAccount") + { + conventions.Add(new AwsAuthConvention(log, variables)); + } + conventions.Add(new KubernetesAuthContextConvention(log, commandLineRunner, kubectl, fileSystem)); + + new ConventionProcessor(deployment, conventions, log).RunConventions(); + } + + class AppliedResourceDto + { + public string Group { get; set; } + public string Version { get; set; } + public string Kind { get; set; } + public string Name { get; set; } + public string Namespace { get; set; } + } + } +} diff --git a/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs b/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs index 0550dc2c3e..dbceb58374 100644 --- a/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs +++ b/source/Calamari/Kubernetes/Conventions/Helm/HelmUpgradeExecutor.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -24,18 +24,21 @@ public class HelmUpgradeExecutor readonly HelmTemplateValueSourcesParser templateValueSourcesParser; readonly HelmCli helmCli; readonly IKubernetesManifestNamespaceResolver namespaceResolver; + readonly IManifestReporter manifestReporter; public HelmUpgradeExecutor(ILog log, ICalamariFileSystem fileSystem, HelmTemplateValueSourcesParser templateValueSourcesParser, HelmCli helmCli, - IKubernetesManifestNamespaceResolver namespaceResolver) + IKubernetesManifestNamespaceResolver namespaceResolver, + IManifestReporter manifestReporter = null) { this.log = log; this.fileSystem = fileSystem; this.templateValueSourcesParser = templateValueSourcesParser; this.helmCli = helmCli; this.namespaceResolver = namespaceResolver; + this.manifestReporter = manifestReporter; } public void ExecuteHelmUpgrade(RunningDeployment deployment, @@ -64,15 +67,15 @@ public void ExecuteHelmUpgrade(RunningDeployment deployment, if (OctopusFeatureToggles.ArgoRolloutsSupportFeatureToggle.IsEnabled(deployment.Variables)) { - SetAppliedResourcesOutputVariable(deployment, releaseName, newRevisionNumber); + ReportManifestAndSetAppliedResources(deployment, releaseName, newRevisionNumber); } installCompletedCts.Cancel(); } - void SetAppliedResourcesOutputVariable(RunningDeployment deployment, string releaseName, int revisionNumber) + void ReportManifestAndSetAppliedResources(RunningDeployment deployment, string releaseName, int revisionNumber) { - string manifest = null; + string manifest; try { manifest = helmCli.GetManifest(releaseName, revisionNumber); @@ -89,6 +92,10 @@ void SetAppliedResourcesOutputVariable(RunningDeployment deployment, string rele return; } + //Manifest reporting normally happens inside HelmManifestAndStatusReporter, which is + //skipped on this path; emit it inline so the UI still gets the applied manifest. + manifestReporter?.ReportManifestApplied(manifest); + var resources = ManifestParser.GetResourcesFromManifest(manifest, namespaceResolver, deployment.Variables, log); AppliedResourcesOutputHelper.SetAppliedResourcesOutputVariable(log, deployment, resources); } @@ -104,8 +111,10 @@ List GetUpgradeCommandArgs(RunningDeployment deployment) SetValuesParameters(deployment, args); var hasAdditionalArgs = SetAdditionalArguments(deployment, args); - //Adjust args based on KOS - if (deployment.Variables.GetFlag(SpecialVariables.ResourceStatusCheck)) + //Adjust args based on KOS. When ArgoRollouts support is enabled, status checking moves + //to a separate verification action, so we don't force --wait on the deploy step. + if (deployment.Variables.GetFlag(SpecialVariables.ResourceStatusCheck) + && !OctopusFeatureToggles.ArgoRolloutsSupportFeatureToggle.IsEnabled(deployment.Variables)) { AddKOSArgs(deployment.Variables, hasAdditionalArgs, args); } diff --git a/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs b/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs index 7ff9ee13ad..a2f41e8ccf 100644 --- a/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs +++ b/source/Calamari/Kubernetes/Conventions/HelmUpgradeWithKOSConvention.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Calamari.Common.Commands; using Calamari.Common.Features.Processes; +using Calamari.Common.FeatureToggles; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Logging; using Calamari.Common.Plumbing.Variables; @@ -57,10 +58,20 @@ public void Install(RunningDeployment deployment) var newRevisionNumber = (currentRevisionNumber ?? 0) + 1; + //When ArgoRollouts support is enabled, the parallel manifest + KOS reporter is replaced + //by a separate verification action that runs after the deploy step. Manifest reporting + //and AppliedResources emission are performed inline by HelmUpgradeExecutor instead. + if (OctopusFeatureToggles.ArgoRolloutsSupportFeatureToggle.IsEnabled(deployment.Variables)) + { + var executor = new HelmUpgradeExecutor(log, fileSystem, valueSourcesParser, helmCli, namespaceResolver, manifestReporter); + executor.ExecuteHelmUpgrade(deployment, releaseName, newRevisionNumber, new CancellationTokenSource(), new CancellationTokenSource()); + return; + } + //This is used to cancel KOS when the helm upgrade has completed //It does not cancel the get manifest var helmInstallCompletedCts = new CancellationTokenSource(); - + //This is used to cancel the get manifest when the helm install fails (and we are still trying to retrieve the manifest) var helmInstallErrorCts = new CancellationTokenSource(); @@ -71,7 +82,7 @@ public void Install(RunningDeployment deployment) valueSourcesParser, helmCli, namespaceResolver); - + executor.ExecuteHelmUpgrade(deployment, releaseName, newRevisionNumber, helmInstallCompletedCts, helmInstallErrorCts); }); @@ -82,7 +93,7 @@ public void Install(RunningDeployment deployment) await runner.StartBackgroundMonitoringAndReporting(deployment, releaseName, newRevisionNumber, - helmInstallCompletedCts.Token, + helmInstallCompletedCts.Token, helmInstallErrorCts.Token); }, helmInstallCompletedCts.Token); diff --git a/source/Calamari/Kubernetes/ResourceStatus/Resources/ResourceFactory.cs b/source/Calamari/Kubernetes/ResourceStatus/Resources/ResourceFactory.cs index 68058675be..166b36e597 100644 --- a/source/Calamari/Kubernetes/ResourceStatus/Resources/ResourceFactory.cs +++ b/source/Calamari/Kubernetes/ResourceStatus/Resources/ResourceFactory.cs @@ -25,6 +25,8 @@ public static class ResourceFactory { SupportedResourceGroupVersionKinds.PersistentVolumeV1, (d, o) => new PersistentVolume(d, o) } }; + public static bool IsVerifiable(ResourceGroupVersionKind gvk) => resourceFactories.ContainsKey(gvk); + public static Resource FromJson(string json, Options options) => FromJObject(JObject.Parse(json), options); public static IEnumerable FromListJson(string json, Options options)