Move resource verification into a separate step if argo rollouts support feature is enabled #2002
+275
−13
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| using System; | ||
| using Calamari.Common.Commands; | ||
| using Calamari.Common.Features.Processes; | ||
| using Calamari.Common.Plumbing.Variables; | ||
| using Calamari.Kubernetes; | ||
| using Calamari.Kubernetes.Commands; | ||
| using Calamari.Kubernetes.Integration; | ||
| using Calamari.Kubernetes.ResourceStatus; | ||
| using Calamari.Testing.Helpers; | ||
| using Calamari.Tests.Fixtures.Integration.FileSystem; | ||
| using FluentAssertions; | ||
| using NSubstitute; | ||
| using NUnit.Framework; | ||
|
|
||
| namespace Calamari.Tests.KubernetesFixtures.Commands | ||
| { | ||
| [TestFixture] | ||
| public class KubernetesVerifyResourcesCommandFixture | ||
| { | ||
| [Test] | ||
| public void WhenNoAppliedResourcesVariableIsSet_ShouldThrowCommandException() | ||
| { | ||
| var variables = new CalamariVariables(); | ||
| var statusReporter = Substitute.For<IResourceStatusReportExecutor>(); | ||
| var command = CreateCommand(variables, statusReporter, new InMemoryLog()); | ||
|
|
||
| Action execute = () => command.Execute(new string[] { }); | ||
|
|
||
| execute.Should() | ||
| .Throw<CommandException>() | ||
| .WithMessage("The applied resources variable was not found. This variable is required to verify the deployed resources."); | ||
| statusReporter.ReceivedCalls().Should().BeEmpty(); | ||
| } | ||
|
|
||
| [Test] | ||
| public void WhenAppliedResourcesIsAnEmptyList_ShouldDoNothingAndSucceed() | ||
| { | ||
| var variables = new CalamariVariables | ||
| { | ||
| [SpecialVariables.AppliedResources] = "[]" | ||
| }; | ||
| var statusReporter = Substitute.For<IResourceStatusReportExecutor>(); | ||
| var log = new InMemoryLog(); | ||
| var command = CreateCommand(variables, statusReporter, log); | ||
|
|
||
| var result = command.Execute(new string[] { }); | ||
|
|
||
| result.Should().Be(0); | ||
| statusReporter.ReceivedCalls().Should().BeEmpty(); | ||
| log.MessagesInfoFormatted.Should().Contain("Applied resources list is empty; nothing to verify."); | ||
| } | ||
|
|
||
| [Test] | ||
| public void WhenAppliedResourcesIsNotValidJson_ShouldThrowCommandException() | ||
| { | ||
| var variables = new CalamariVariables | ||
| { | ||
| [SpecialVariables.AppliedResources] = "this is not json" | ||
| }; | ||
| var statusReporter = Substitute.For<IResourceStatusReportExecutor>(); | ||
| var command = CreateCommand(variables, statusReporter, new InMemoryLog()); | ||
|
|
||
| Action execute = () => command.Execute(new string[] { }); | ||
|
|
||
| execute.Should() | ||
| .Throw<CommandException>() | ||
| .WithMessage("Could not parse applied resources output variable:*"); | ||
| statusReporter.ReceivedCalls().Should().BeEmpty(); | ||
| } | ||
|
|
||
| static KubernetesVerifyResourcesCommand CreateCommand( | ||
| IVariables variables, | ||
| IResourceStatusReportExecutor statusReporter, | ||
| InMemoryLog log) | ||
| { | ||
| var fileSystem = new TestCalamariPhysicalFileSystem(); | ||
| var commandLineRunner = Substitute.For<ICommandLineRunner>(); | ||
| var kubectl = new Kubectl(variables, log, commandLineRunner); | ||
|
|
||
| return new KubernetesVerifyResourcesCommand( | ||
| log, | ||
| variables, | ||
| fileSystem, | ||
| commandLineRunner, | ||
| kubectl, | ||
| statusReporter); | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,143 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Threading; | ||
| using Calamari.Aws.Integration; | ||
| using Calamari.Commands.Support; | ||
| using Calamari.Common.Commands; | ||
| using Calamari.Common.Features.Processes; | ||
| using Calamari.Common.Plumbing.FileSystem; | ||
| using Calamari.Common.Plumbing.Logging; | ||
| using Calamari.Common.Plumbing.Variables; | ||
| using Calamari.Deployment; | ||
| using Calamari.Deployment.Conventions; | ||
| using Calamari.Kubernetes.Conventions; | ||
| using Calamari.Kubernetes.Integration; | ||
| using Calamari.Kubernetes.ResourceStatus; | ||
| using Calamari.Kubernetes.ResourceStatus.Resources; | ||
| using Newtonsoft.Json; | ||
|
|
||
| namespace Calamari.Kubernetes.Commands | ||
| { | ||
| [Command(Name, Description = "Verifies that resources applied by a Kubernetes deploy step have reached their desired state")] | ||
| public class KubernetesVerifyResourcesCommand : Command | ||
| { | ||
| public const string Name = "kubernetes-verify-resources"; | ||
|
|
||
| readonly ILog log; | ||
| readonly IVariables variables; | ||
| readonly ICalamariFileSystem fileSystem; | ||
| readonly ICommandLineRunner commandLineRunner; | ||
| readonly Kubectl kubectl; | ||
| readonly IResourceStatusReportExecutor statusReporter; | ||
|
|
||
| public KubernetesVerifyResourcesCommand( | ||
| ILog log, | ||
| IVariables variables, | ||
| ICalamariFileSystem fileSystem, | ||
| ICommandLineRunner commandLineRunner, | ||
| Kubectl kubectl, | ||
| IResourceStatusReportExecutor statusReporter) | ||
| { | ||
| this.log = log; | ||
| this.variables = variables; | ||
| this.fileSystem = fileSystem; | ||
| this.commandLineRunner = commandLineRunner; | ||
| this.kubectl = kubectl; | ||
| this.statusReporter = statusReporter; | ||
| } | ||
|
|
||
| public override int Execute(string[] commandLineArguments) | ||
| { | ||
| Options.Parse(commandLineArguments); | ||
|
|
||
| var json = variables.Get(SpecialVariables.AppliedResources); | ||
| if (string.IsNullOrWhiteSpace(json)) | ||
| { | ||
| throw new CommandException($"The applied resources variable was not found. This variable is required to verify the deployed resources."); | ||
| } | ||
|
|
||
| List<ResourceIdentifier> resources; | ||
| try | ||
| { | ||
| resources = DeserializeResources(json); | ||
| } | ||
| catch (JsonException ex) | ||
| { | ||
| throw new CommandException($"Could not parse applied resources output variable: {ex.Message}"); | ||
| } | ||
|
|
||
| if (resources.Count == 0) | ||
| { | ||
| log.Info("Applied resources list is empty; nothing to verify."); | ||
| return 0; | ||
| } | ||
|
|
||
| WarnForUnverifiableResources(resources); | ||
|
|
||
| AuthenticateKubectl(); | ||
|
|
||
| var timeoutSeconds = variables.GetInt32(SpecialVariables.Timeout) ?? 0; | ||
| var waitForJobs = variables.GetFlag(SpecialVariables.WaitForJobs); | ||
|
|
||
| var statusCheck = statusReporter.Start(timeoutSeconds, waitForJobs, resources); | ||
| var success = statusCheck.WaitForCompletionOrTimeout(CancellationToken.None).GetAwaiter().GetResult(); | ||
|
Collaborator
There was a problem hiding this comment. Sucks that |
||
|
|
||
| if (!success) | ||
| { | ||
| throw new CommandException("Resource verification failed. Check verbose logs for more details."); | ||
| } | ||
|
|
||
| return 0; | ||
| } | ||
|
|
||
| static List<ResourceIdentifier> DeserializeResources(string json) | ||
| { | ||
| var raw = JsonConvert.DeserializeObject<List<AppliedResourceDto>>(json) ?? new List<AppliedResourceDto>(); | ||
| return raw | ||
| .Select(r => new ResourceIdentifier( | ||
| new ResourceGroupVersionKind(r.Group ?? string.Empty, r.Version, r.Kind), | ||
| r.Name, | ||
| r.Namespace)) | ||
| .ToList(); | ||
| } | ||
|
|
||
| void WarnForUnverifiableResources(IEnumerable<ResourceIdentifier> resources) | ||
| { | ||
| foreach (var resource in resources) | ||
| { | ||
| var gvk = resource.GroupVersionKind; | ||
| if (ResourceFactory.IsVerifiable(gvk)) | ||
| continue; | ||
|
|
||
| var name = string.IsNullOrEmpty(resource.Namespace) ? resource.Name : $"{resource.Namespace}/{resource.Name}"; | ||
| log.Warn($"Unable to fully verify resource '{gvk}' '{name}'. Calamari does not know the readiness criteria for this resource type; only its existence will be confirmed."); | ||
|
Collaborator
There was a problem hiding this comment. Maybe one for Idan, but do we want to indicate that using the Agent with kubernetes monitor may unlock verification for their type. Possibly a link to a public doc which will indicate which types are supported for verification? It should be pretty expansive once https://linear.app/octopus/issue/SIE-143/add-argo-rollout-support-to-kubernetes-monitor is completed. |
||
| } | ||
| } | ||
|
|
||
| void AuthenticateKubectl() | ||
| { | ||
| var deployment = new RunningDeployment(variables); | ||
| kubectl.SetWorkingDirectory(deployment.CurrentDirectory); | ||
| kubectl.SetEnvironmentVariables(deployment.EnvironmentVariables); | ||
|
|
||
| var conventions = new List<IConvention>(); | ||
| if (variables.Get(Deployment.SpecialVariables.Account.AccountType) == "AmazonWebServicesAccount") | ||
| { | ||
| conventions.Add(new AwsAuthConvention(log, variables)); | ||
| } | ||
| conventions.Add(new KubernetesAuthContextConvention(log, commandLineRunner, kubectl, fileSystem)); | ||
|
|
||
| new ConventionProcessor(deployment, conventions, log).RunConventions(); | ||
|
Collaborator
There was a problem hiding this comment. Is there no shared code we can leverage for auth stuff? |
||
| } | ||
|
|
||
| class AppliedResourceDto | ||
| { | ||
| public string Group { get; set; } | ||
| public string Version { get; set; } | ||
| public string Kind { get; set; } | ||
| public string Name { get; set; } | ||
| public string Namespace { get; set; } | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If the variable doesn't exist we should fail, I think. Otherwise people might think verification is working correctly when there is actually some issue with it.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If the variable exists and it's an empty array
[], then the info log is good.