From 3c846f7eac39c843f64da6a3f3e94dd1fad34111 Mon Sep 17 00:00:00 2001 From: Khushboo Date: Mon, 22 Jun 2026 17:51:43 -0700 Subject: [PATCH] Add chart upgrade test Signed-off-by: Khushboo --- .../keywords/rancher.resource | 310 ++++++++++++++++-- .../keywords/variables.resource | 2 + .../libs/keywords/rancher_keywords.py | 114 ++++++- harvester_robot_tests/libs/rancher/base.py | 33 +- harvester_robot_tests/libs/rancher/crd.py | 246 +++++++++++++- harvester_robot_tests/libs/rancher/rancher.py | 42 ++- harvester_robot_tests/libs/rancher/rest.py | 214 +++++++++++- .../test_rancher_integration.robot | 56 +++- 8 files changed, 947 insertions(+), 70 deletions(-) rename harvester_robot_tests/tests/regression/{ => rancher}/test_rancher_integration.robot (75%) diff --git a/harvester_robot_tests/keywords/rancher.resource b/harvester_robot_tests/keywords/rancher.resource index 1e758e9c1..711cc7331 100644 --- a/harvester_robot_tests/keywords/rancher.resource +++ b/harvester_robot_tests/keywords/rancher.resource @@ -769,6 +769,21 @@ Get Chart Versions ${versions}= rancher_keywords.get_chart_versions ${repo_name} ${chart_name} ${cluster_id} RETURN ${versions} +Get Latest Chart Version + [Arguments] ${repo_name} ${chart_name} ${cluster_id}=${None} + [Documentation] Get the latest available version for a chart from a Rancher + ... chart repo. Fails with a clear error if no versions are found. + ... Args: + ... repo_name: Chart repository name + ... chart_name: Chart name + ... cluster_id: Optional guest cluster ID to query via proxy + ... Returns: + ... str: Latest chart version + ${versions}= Get Chart Versions ${repo_name} ${chart_name} ${cluster_id} + Should Not Be Empty ${versions} No versions found for chart ${chart_name} in repo ${repo_name} + ${latest}= Get From List ${versions} 0 + RETURN ${latest} + Create Cluster Repo [Arguments] ${cluster_id} ${repo_name} ${git_url} ${git_branch} [Documentation] Create a ClusterRepo on a guest cluster. @@ -783,13 +798,16 @@ Create Cluster Repo Wait For Cluster Repo Ready [Arguments] ${cluster_id} ${repo_name} ${timeout}=${RANCHER_WAIT_TIMEOUT} + ... ${expected_git_branch}=${NONE} [Documentation] Wait for a ClusterRepo to finish downloading on a guest cluster. ... Args: ... cluster_id: Rancher management cluster ID ... repo_name: ClusterRepo name ... timeout: Timeout in seconds + ... expected_git_branch: When set, verify spec.gitBranch matches Log Waiting for ClusterRepo ${repo_name} to be ready - ${result}= rancher_keywords.wait_for_cluster_repo_ready ${cluster_id} ${repo_name} ${timeout} + ${result}= rancher_keywords.wait_for_cluster_repo_ready ${cluster_id} ${repo_name} + ... ${timeout} ${expected_git_branch} RETURN ${result} Create Cloud Config Secret @@ -806,16 +824,18 @@ Create Cloud Config Secret RETURN ${result} Wait For Chart App Ready - [Arguments] ${cluster_id} ${release_name} ${namespace} ${timeout}=${RANCHER_WAIT_TIMEOUT} + [Arguments] ${cluster_id} ${release_name} ${namespace} + ... ${timeout}=${RANCHER_WAIT_TIMEOUT} ${expected_version}=${NONE} [Documentation] Wait for a chart app to be deployed and ready. ... Args: ... cluster_id: Rancher management cluster ID ... release_name: Helm release name ... namespace: Namespace where chart was installed ... timeout: Timeout in seconds + ... expected_version: When set, also wait for this chart version Log Waiting for chart app ${release_name} to be ready ${result}= rancher_keywords.wait_for_chart_app_ready ${cluster_id} ${release_name} - ... ${namespace} ${timeout} + ... ${namespace} ${timeout} ${expected_version} RETURN ${result} Get Deployed Chart Version @@ -830,6 +850,72 @@ Get Deployed Chart Version ${version}= rancher_keywords.get_deployed_chart_version ${cluster_id} ${release_name} ${namespace} RETURN ${version} +Upgrade Chart + [Arguments] ${cluster_id} ${repo_name} ${chart_name} ${version} + ... ${release_name} ${namespace} ${values}=${None} + [Documentation] Upgrade an installed Helm chart on a guest cluster. + ... Args: + ... cluster_id: Rancher management cluster ID + ... repo_name: Chart repository name + ... chart_name: Chart name + ... version: Target chart version (empty string for latest) + ... release_name: Helm release name + ... namespace: Target namespace + ... values: Optional dict of chart values overrides + Log Upgrading chart ${chart_name} to v${version} on cluster ${cluster_id} + ${result}= rancher_keywords.upgrade_chart ${cluster_id} ${repo_name} ${chart_name} + ... ${version} ${release_name} ${namespace} ${values} + RETURN ${result} + +Uninstall Chart + [Arguments] ${cluster_id} ${release_name} ${namespace} + [Documentation] Uninstall a Helm chart (app) from a guest cluster. + ... Args: + ... cluster_id: Rancher management cluster ID + ... release_name: Helm release name + ... namespace: Namespace where the chart is installed + Log Uninstalling chart ${release_name} from cluster ${cluster_id} + ${result}= rancher_keywords.uninstall_chart ${cluster_id} ${release_name} ${namespace} + RETURN ${result} + +Wait For Chart App Deleted + [Arguments] ${cluster_id} ${release_name} ${namespace} ${timeout}=${RANCHER_WAIT_TIMEOUT} + [Documentation] Wait for a chart app to be fully removed from a guest cluster. + ... Args: + ... cluster_id: Rancher management cluster ID + ... release_name: Helm release name + ... namespace: Namespace where chart was installed + ... timeout: Timeout in seconds + Log Waiting for chart app ${release_name} to be deleted + ${result}= rancher_keywords.wait_for_chart_app_deleted ${cluster_id} ${release_name} + ... ${namespace} ${timeout} + RETURN ${result} + +Scale Deployment + [Arguments] ${cluster_id} ${namespace} ${name} ${replicas} + [Documentation] Scale a deployment in a guest cluster to the given replica count. + ... Args: + ... cluster_id: Cluster ID + ... namespace: Namespace + ... name: Deployment name + ... replicas: Desired replica count + Log Scaling deployment ${name} to ${replicas} replicas + rancher_keywords.scale_deployment ${cluster_id} ${namespace} ${name} ${replicas} + +Wait For Deployment Scaled + [Arguments] ${cluster_id} ${namespace} ${name} ${replicas} ${timeout}=${WAIT_TIMEOUT} + [Documentation] Wait for a deployment to reach the given ready replica count. + ... Args: + ... cluster_id: Cluster ID + ... namespace: Namespace + ... name: Deployment name + ... replicas: Expected ready replica count + ... timeout: Timeout in seconds + Log Waiting for deployment ${name} to scale to ${replicas} + ${result}= rancher_keywords.wait_for_deployment_scaled ${cluster_id} ${namespace} ${name} + ... ${replicas} ${timeout} + RETURN ${result} + # RWX Volume / StorageClass / StatefulSet Operations Create RWX StorageClass On Host [Arguments] ${name}=longhorn-rwx @@ -1283,12 +1369,12 @@ Multi node cluster is upgraded to next version Upgrade RKE2 Cluster ${MULTI_CLUSTER_NAME} ${NEXT_RKE2_VERSION} New CSI workload is deployed - [Arguments] ${suffix} + [Arguments] ${suffix} ${cluster_id}=${MULTI_CLUSTER_ID} ${cluster_name}=${MULTI_CLUSTER_NAME} [Documentation] Create a new PVC and nginx deployment for post-operation verification. - ${pvc_name}= Set Variable pvc-${suffix}-${MULTI_CLUSTER_NAME} - ${deploy_name}= Set Variable csi-${suffix}-${MULTI_CLUSTER_NAME} - Create PVC ${MULTI_CLUSTER_ID} ${pvc_name} 1Gi - Create Deployment ${MULTI_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${deploy_name} nginx:1.27 ${pvc_name} + ${pvc_name}= Set Variable pvc-${suffix}-${cluster_name} + ${deploy_name}= Set Variable csi-${suffix}-${cluster_name} + Create PVC ${cluster_id} ${pvc_name} 1Gi + Create Deployment ${cluster_id} ${DEFAULT_NAMESPACE} ${deploy_name} nginx:1.27 ${pvc_name} All test workloads are removed from cluster [Documentation] Delete all test deployments, PVCs, and LB services from the multi-node cluster. @@ -1341,26 +1427,172 @@ CSI workload is deployed on single node import cluster Create Deployment ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${csi_deploy} nginx:1.27 ${pvc_name} Cloud provider workloads are deployed on single node import cluster + [Arguments] ${suffix}=${EMPTY} [Documentation] Deploy a Whoami workload with DHCP and IP Pool LoadBalancer ... services on the import cluster to exercise the Harvester cloud ... provider. + ${base}= Set Variable If '${suffix}' == '${EMPTY}' ${IMPORT_CLUSTER_NAME} ${suffix}-${IMPORT_CLUSTER_NAME} # Whoami workload - ${whoami_deploy}= Set Variable whoami-${IMPORT_CLUSTER_NAME} + ${whoami_deploy}= Set Variable whoami-${base} Create Deployment ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${whoami_deploy} traefik/whoami:v1.11.0 # LB DHCP - ${lb_dhcp}= Set Variable lb-${IMPORT_CLUSTER_NAME}-dhcp + ${lb_dhcp}= Set Variable lb-${base}-dhcp Create LB Service ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${lb_dhcp} ${whoami_deploy} ipam=dhcp Wait For LB Service Ready ${IMPORT_CLUSTER_ID} ${lb_dhcp} # LB Pool - ${lb_pool}= Set Variable lb-${IMPORT_CLUSTER_NAME}-pool + ${lb_pool}= Set Variable lb-${base}-pool Create LB Service ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${lb_pool} ${whoami_deploy} ipam=pool Wait For LB Service Ready ${IMPORT_CLUSTER_ID} ${lb_pool} Harvester CSI driver chart is installed on single node import cluster - Install Harvester CSI Driver Chart On Import Cluster ${IMPORT_CLUSTER_ID} ${IMPORT_CLUSTER_NAME} + [Arguments] ${version}=${CSI_CHART_VERSION} + [Documentation] Install the Harvester CSI Driver chart on the import cluster. + ... When version is empty the latest available version is used. + Install Harvester CSI Driver Chart On Import Cluster ${IMPORT_CLUSTER_ID} ${IMPORT_CLUSTER_NAME} ${version} + Harvester CSI driver should be ready ${IMPORT_CLUSTER_ID} Harvester cloud provider chart is installed on single node import cluster - Install Harvester Cloud Provider Chart On Import Cluster ${IMPORT_CLUSTER_ID} ${IMPORT_CLUSTER_NAME} + [Arguments] ${version}=${CP_CHART_VERSION} + [Documentation] Install the Harvester Cloud Provider chart on the import cluster. + ... When version is empty the latest available version is used. + Install Harvester Cloud Provider Chart On Import Cluster ${IMPORT_CLUSTER_ID} ${IMPORT_CLUSTER_NAME} ${version} + Harvester cloud provider should be ready ${IMPORT_CLUSTER_ID} + +Multiple Harvester CSI driver chart versions are available + [Documentation] Ensure the rancher-charts repo offers at least two CSI driver + ... chart versions so an upgrade can be tested. + ... If CSI_CHART_VERSION is set, it is used as the target (latest) + Ensure Harvester Chart Prerequisites On Import Cluster ${IMPORT_CLUSTER_ID} ${IMPORT_CLUSTER_NAME} + ${versions}= Get Chart Versions rancher-charts harvester-csi-driver ${IMPORT_CLUSTER_ID} + ${count}= Get Length ${versions} + IF '${CSI_CHART_VERSION}' != '${EMPTY}' + ${idx}= Get Index From List ${versions} ${CSI_CHART_VERSION} + Should Not Be Equal As Integers ${idx} -1 + ... CSI chart version ${CSI_CHART_VERSION} not found in available versions: ${versions} + ${prev_idx}= Evaluate ${idx} + 1 + Skip If ${prev_idx} >= ${count} + ... No older version available before ${CSI_CHART_VERSION}; cannot test upgrade + ${latest}= Set Variable ${CSI_CHART_VERSION} + ${previous}= Get From List ${versions} ${prev_idx} + ELSE + Skip If ${count} < 2 + ... Only ${count} harvester-csi-driver chart version(s) available; need at least 2 to test upgrade + ${latest}= Get From List ${versions} 0 + ${previous}= Get From List ${versions} 1 + END + Set Suite Variable ${CSI_LATEST_VERSION} ${latest} + Set Suite Variable ${CSI_PREV_VERSION} ${previous} + Log CSI chart upgrade path: ${previous} -> ${latest} console=yes + +Harvester CSI driver chart is upgraded to latest on import cluster + [Documentation] Upgrade the Harvester CSI Driver chart to the latest version + ... and verify the deployed chart version matches. + ${csi_values}= Evaluate {"cloudConfig": {"hostPath": "", "secretName": "harvester-cloud-provider-config"}} + Upgrade Chart + ... ${IMPORT_CLUSTER_ID} rancher-charts harvester-csi-driver + ... ${CSI_LATEST_VERSION} harvester-csi-driver kube-system ${csi_values} + Wait For Chart App Ready + ... ${IMPORT_CLUSTER_ID} harvester-csi-driver kube-system + ... expected_version=${CSI_LATEST_VERSION} + Log Upgraded harvester-csi-driver to ${CSI_LATEST_VERSION} console=yes + +CSI upgrade workload is restarted on import cluster + [Documentation] Scale the CSI workload down to 0 and back up to 1 to confirm + ... the upgraded CSI driver can re-attach the volume. + ${csi_deploy}= Set Variable csi-${IMPORT_CLUSTER_NAME} + Scale Deployment ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${csi_deploy} 0 + Wait For Deployment Scaled ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${csi_deploy} 0 + Scale Deployment ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${csi_deploy} 1 + Wait For Deployment Ready ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${csi_deploy} + Log CSI workload restarted on import cluster console=yes + +Cleanup CSI upgrade test resources on import cluster + [Arguments] ${deploy_names}=csi-${IMPORT_CLUSTER_NAME}, csi-csiup-${IMPORT_CLUSTER_NAME} + ... ${pvc_names}=pvc-${IMPORT_CLUSTER_NAME}, pvc-csiup-${IMPORT_CLUSTER_NAME} + [Documentation] Delete CSI workloads and PVCs and uninstall the CSI driver + ... chart so the subsequent install test starts clean. + ... Args: + ... deploy_names: Comma-separated list of deployment names to delete + ... pvc_names: Comma-separated list of PVC names to delete + @{deployments}= Split String ${deploy_names} , + FOR ${deployment} IN @{deployments} + Run Keyword And Ignore Error Delete Deployment ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${deployment.strip()} + END + @{pvcs}= Split String ${pvc_names} , + FOR ${pvc} IN @{pvcs} + Run Keyword And Ignore Error Delete PVC ${IMPORT_CLUSTER_ID} ${pvc.strip()} + END + Run Keyword And Ignore Error Uninstall Chart ${IMPORT_CLUSTER_ID} harvester-csi-driver kube-system + Run Keyword And Ignore Error Wait For Chart App Deleted ${IMPORT_CLUSTER_ID} harvester-csi-driver kube-system + +Multiple Harvester cloud provider chart versions are available + [Documentation] Ensure the rancher-charts repo offers at least two cloud provider + ... chart versions so an upgrade can be tested. + ... If CP_CHART_VERSION is set, it is used as the target (latest) version + Ensure Harvester Chart Prerequisites On Import Cluster ${IMPORT_CLUSTER_ID} ${IMPORT_CLUSTER_NAME} + ${versions}= Get Chart Versions rancher-charts harvester-cloud-provider ${IMPORT_CLUSTER_ID} + ${count}= Get Length ${versions} + IF '${CP_CHART_VERSION}' != '${EMPTY}' + ${idx}= Get Index From List ${versions} ${CP_CHART_VERSION} + Should Not Be Equal As Integers ${idx} -1 + ... Cloud provider chart version ${CP_CHART_VERSION} not found in available versions: ${versions} + ${prev_idx}= Evaluate ${idx} + 1 + Skip If ${prev_idx} >= ${count} + ... No older version available before ${CP_CHART_VERSION}; cannot test upgrade + ${latest}= Set Variable ${CP_CHART_VERSION} + ${previous}= Get From List ${versions} ${prev_idx} + ELSE + Skip If ${count} < 2 + ... Only ${count} harvester-cloud-provider chart version(s) available; need at least 2 to test upgrade + ${latest}= Get From List ${versions} 0 + ${previous}= Get From List ${versions} 1 + END + Set Suite Variable ${CP_LATEST_VERSION} ${latest} + Set Suite Variable ${CP_PREV_VERSION} ${previous} + Log Cloud provider chart upgrade path: ${previous} -> ${latest} console=yes + +Harvester cloud provider chart is upgraded to latest on import cluster + [Documentation] Upgrade the Harvester Cloud Provider chart to the latest version + ... and verify the deployed chart version matches. + ${cp_values}= Evaluate {"cloudConfigPath": "/var/lib/rancher/rke2/etc/config-files/cloud-provider-config", "global": {"cattle": {"clusterName": "${IMPORT_CLUSTER_ID}"}}} + Upgrade Chart + ... ${IMPORT_CLUSTER_ID} rancher-charts harvester-cloud-provider + ... ${CP_LATEST_VERSION} harvester-cloud-provider kube-system ${cp_values} + Wait For Chart App Ready + ... ${IMPORT_CLUSTER_ID} harvester-cloud-provider kube-system + ... expected_version=${CP_LATEST_VERSION} + Log Upgraded harvester-cloud-provider to ${CP_LATEST_VERSION} console=yes + +Cloud provider upgrade workload is restarted on import cluster + [Documentation] Scale the Whoami workload down to 0 and back up to 1 to confirm + ... the upgraded cloud provider keeps serving the LoadBalancer. + ${whoami_deploy}= Set Variable whoami-${IMPORT_CLUSTER_NAME} + ${lb_dhcp}= Set Variable lb-${IMPORT_CLUSTER_NAME}-dhcp + Scale Deployment ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${whoami_deploy} 0 + Wait For Deployment Scaled ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${whoami_deploy} 0 + Scale Deployment ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${whoami_deploy} 1 + Wait For Deployment Ready ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${whoami_deploy} + Verify LB Via Cluster ${IMPORT_CLUSTER_ID} ${lb_dhcp} Hostname: + Log Cloud provider workload restarted on import cluster console=yes + +Cleanup cloud provider upgrade test resources on import cluster + [Arguments] ${lb_names}=lb-${IMPORT_CLUSTER_NAME}-dhcp, lb-${IMPORT_CLUSTER_NAME}-pool, lb-cpup2-${IMPORT_CLUSTER_NAME}-dhcp, lb-cpup2-${IMPORT_CLUSTER_NAME}-pool + ... ${deploy_names}=whoami-${IMPORT_CLUSTER_NAME}, whoami-cpup2-${IMPORT_CLUSTER_NAME} + [Documentation] Delete cloud provider workloads and LoadBalancers and uninstall + ... the cloud provider chart so the subsequent install test starts clean. + ... Args: + ... lb_names: Comma-separated list of LoadBalancer service names to delete + ... deploy_names: Comma-separated list of deployment names to delete + @{lbs}= Split String ${lb_names} , + FOR ${lb} IN @{lbs} + Run Keyword And Ignore Error Delete LB Service ${IMPORT_CLUSTER_ID} ${lb.strip()} + END + @{deployments}= Split String ${deploy_names} , + FOR ${deployment} IN @{deployments} + Run Keyword And Ignore Error Delete Deployment ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${deployment.strip()} + END + Run Keyword And Ignore Error Uninstall Chart ${IMPORT_CLUSTER_ID} harvester-cloud-provider kube-system + Run Keyword And Ignore Error Wait For Chart App Deleted ${IMPORT_CLUSTER_ID} harvester-cloud-provider kube-system Multi node RKE2 cluster is deleted [Documentation] Delete the multi-node RKE2 cluster and wait for removal. @@ -1429,11 +1661,18 @@ CSI workload should be active on single node import cluster Log CSI workload verified on import cluster console=yes Cloud provider workloads should be active on single node import cluster + [Arguments] ${suffix}=${EMPTY} [Documentation] Verify the Whoami deployment and both LB services (DHCP and IP - ... Pool) are functional on the import cluster. - ${whoami_deploy}= Set Variable whoami-${IMPORT_CLUSTER_NAME} - ${lb_dhcp}= Set Variable lb-${IMPORT_CLUSTER_NAME}-dhcp - ${lb_pool}= Set Variable lb-${IMPORT_CLUSTER_NAME}-pool + ... Pool) are functional on the import cluster. An optional suffix + ... selects the matching workload set. + IF '${suffix}' == '${EMPTY}' + ${base}= Set Variable ${IMPORT_CLUSTER_NAME} + ELSE + ${base}= Set Variable ${suffix}-${IMPORT_CLUSTER_NAME} + END + ${whoami_deploy}= Set Variable whoami-${base} + ${lb_dhcp}= Set Variable lb-${base}-dhcp + ${lb_pool}= Set Variable lb-${base}-pool Wait For Deployment Ready ${IMPORT_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${whoami_deploy} Verify LB Via Cluster ${IMPORT_CLUSTER_ID} ${lb_dhcp} Hostname: Verify LB Via Cluster ${IMPORT_CLUSTER_ID} ${lb_pool} Hostname: @@ -1561,12 +1800,12 @@ All existing workloads should be active Log All existing workloads verified active console=yes New CSI workload should be active - [Arguments] ${suffix} + [Arguments] ${suffix} ${cluster_id}=${MULTI_CLUSTER_ID} ${cluster_name}=${MULTI_CLUSTER_NAME} [Documentation] Verify the new CSI PVC and deployment are active. - ${pvc_name}= Set Variable pvc-${suffix}-${MULTI_CLUSTER_NAME} - ${deploy_name}= Set Variable csi-${suffix}-${MULTI_CLUSTER_NAME} - Wait For PVC Bound ${MULTI_CLUSTER_ID} ${pvc_name} - Wait For Deployment Ready ${MULTI_CLUSTER_ID} ${DEFAULT_NAMESPACE} ${deploy_name} + ${pvc_name}= Set Variable pvc-${suffix}-${cluster_name} + ${deploy_name}= Set Variable csi-${suffix}-${cluster_name} + Wait For PVC Bound ${cluster_id} ${pvc_name} + Wait For Deployment Ready ${cluster_id} ${DEFAULT_NAMESPACE} ${deploy_name} Log New CSI deployment ready: ${deploy_name} console=yes Workloads should be cleaned up @@ -1759,6 +1998,7 @@ Ensure Harvester Chart Prerequisites On Import Cluster Create Cluster Repo ... ${cluster_id} rancher-charts ${CHART_REPO_URL} ${chart_repo_branch} Wait For Cluster Repo Ready ${cluster_id} rancher-charts + ... expected_git_branch=${chart_repo_branch} Log ClusterRepo rancher-charts (branch=${chart_repo_branch}) is ready console=yes # Create cloud-provider-config secret on the guest cluster @@ -1768,27 +2008,28 @@ Ensure Harvester Chart Prerequisites On Import Cluster Set Suite Variable ${IMPORT_CHART_PREREQS_DONE} ${True} Install Harvester CSI Driver Chart On Import Cluster - [Arguments] ${cluster_id} ${cluster_name} + [Arguments] ${cluster_id} ${cluster_name} ${version}=${EMPTY} [Documentation] Install the Harvester CSI Driver chart from Rancher Apps on an ... imported cluster. Uses the cloud-config secret (instead of a ... hostPath) and waits for the chart app to be deployed. Ensure Harvester Chart Prerequisites On Import Cluster ${cluster_id} ${cluster_name} - # Get latest CSI driver chart version from the guest cluster's repo - ${csi_versions}= Get Chart Versions rancher-charts harvester-csi-driver ${cluster_id} - ${csi_version}= Get From List ${csi_versions} 0 - Log harvester-csi-driver chart version: ${csi_version} console=yes + # Resolve CSI driver chart version (latest when not specified) + IF '${version}' == '${EMPTY}' + ${version}= Get Latest Chart Version rancher-charts harvester-csi-driver ${cluster_id} + END + Log harvester-csi-driver chart version: ${version} console=yes # Install Harvester CSI Driver — use secret instead of hostPath ${csi_values}= Evaluate {"cloudConfig": {"hostPath": "", "secretName": "harvester-cloud-provider-config"}} Install Chart ... ${cluster_id} rancher-charts harvester-csi-driver - ... ${csi_version} harvester-csi-driver kube-system ${csi_values} + ... ${version} harvester-csi-driver kube-system ${csi_values} Wait For Chart App Ready ${cluster_id} harvester-csi-driver kube-system Log Harvester CSI Driver chart installed on import cluster console=yes Install Harvester Cloud Provider Chart On Import Cluster - [Arguments] ${cluster_id} ${cluster_name} + [Arguments] ${cluster_id} ${cluster_name} ${version}=${EMPTY} [Documentation] Install the Harvester Cloud Provider chart from Rancher Apps on ... an imported cluster. Writes the cloud-config to the node hostPath ... required by the chart and waits for the chart app to be deployed. @@ -1799,16 +2040,17 @@ Install Harvester Cloud Provider Chart On Import Cluster Write Cloud Config To Nodes ... ${cluster_id} harvester-cloud-provider-config kube-system - # Get latest cloud provider chart version from the guest cluster's repo - ${cp_versions}= Get Chart Versions rancher-charts harvester-cloud-provider ${cluster_id} - ${cp_version}= Get From List ${cp_versions} 0 - Log harvester-cloud-provider chart version: ${cp_version} console=yes + # Resolve cloud provider chart version (latest when not specified) + IF '${version}' == '${EMPTY}' + ${version}= Get Latest Chart Version rancher-charts harvester-cloud-provider ${cluster_id} + END + Log harvester-cloud-provider chart version: ${version} console=yes # Install Harvester Cloud Provider ${cp_values}= Evaluate {"cloudConfigPath": "/var/lib/rancher/rke2/etc/config-files/cloud-provider-config", "global": {"cattle": {"clusterName": "${cluster_id}"}}} Install Chart ... ${cluster_id} rancher-charts harvester-cloud-provider - ... ${cp_version} harvester-cloud-provider kube-system ${cp_values} + ... ${version} harvester-cloud-provider kube-system ${cp_values} Wait For Chart App Ready ${cluster_id} harvester-cloud-provider kube-system Log Harvester Cloud Provider chart installed on import cluster console=yes diff --git a/harvester_robot_tests/keywords/variables.resource b/harvester_robot_tests/keywords/variables.resource index 39c01c00b..23b1cbd38 100644 --- a/harvester_robot_tests/keywords/variables.resource +++ b/harvester_robot_tests/keywords/variables.resource @@ -66,6 +66,8 @@ ${INGRESS_CONTROLLER} %{INGRESS_CONTROLLER=traefik} # Chart Repository for import clusters ${CHART_REPO_URL} %{CHART_REPO_URL=https://github.com/rancher/charts} ${CHART_REPO_BRANCH} %{CHART_REPO_BRANCH=release-v2.14} +${CSI_CHART_VERSION} %{CSI_CHART_VERSION=} +${CP_CHART_VERSION} %{CP_CHART_VERSION=} # IP Pool for LoadBalancer tests ${IP_POOL_SUBNET} %{IP_POOL_SUBNET=192.168.1.0/24} diff --git a/harvester_robot_tests/libs/keywords/rancher_keywords.py b/harvester_robot_tests/libs/keywords/rancher_keywords.py index af09dabb6..e69dec0bd 100644 --- a/harvester_robot_tests/libs/keywords/rancher_keywords.py +++ b/harvester_robot_tests/libs/keywords/rancher_keywords.py @@ -644,6 +644,41 @@ def wait_for_deployment_deleted(self, cluster_id, namespace, name, timeout=DEFAU cluster_id, namespace, name, int(timeout) ) + def scale_deployment(self, cluster_id, namespace, name, replicas): + """ + Scale a deployment in a guest cluster to the given replica count + + Args: + cluster_id: Cluster ID + namespace: Namespace + name: Deployment name + replicas: Desired replica count + """ + logging(f"Scaling deployment {name} to {replicas} replicas") + return self.rancher.scale_deployment( + cluster_id, namespace, name, int(replicas) + ) + + def wait_for_deployment_scaled(self, cluster_id, namespace, name, + replicas, timeout=DEFAULT_TIMEOUT): + """ + Wait for a deployment to reach the given ready replica count + + Args: + cluster_id: Cluster ID + namespace: Namespace + name: Deployment name + replicas: Expected ready replica count + timeout: Timeout in seconds + + Returns: + dict: Deployment data when scaled + """ + logging(f"Waiting for deployment {name} to scale to {replicas}") + return self.rancher.wait_for_deployment_scaled( + cluster_id, namespace, name, int(replicas), int(timeout) + ) + # PVC Operations def create_pvc(self, cluster_id, name, size="1Gi", storage_class=None): """ @@ -971,6 +1006,72 @@ def install_chart(self, cluster_id, repo_name, chart_name, version, release_name, namespace, values ) + def upgrade_chart(self, cluster_id, repo_name, chart_name, version, + release_name, namespace, values=None): + """ + Upgrade an installed Helm chart on a guest cluster. + + Args: + cluster_id: Rancher management cluster ID + repo_name: Chart repository name + chart_name: Chart name + version: Target chart version (empty string for latest) + release_name: Helm release name + namespace: Target namespace + values: Optional dict of chart values overrides + """ + if not version: + versions = self.rancher.get_chart_versions( + repo_name, chart_name, cluster_id + ) + if not versions: + raise Exception( + f"No versions found for chart {chart_name} " + f"in {repo_name}" + ) + version = versions[0] + logging(f"Using latest chart version: {version}") + + logging(f"Upgrading chart {chart_name} to v{version} as " + f"{release_name} in {namespace}") + return self.rancher.upgrade_chart( + cluster_id, repo_name, chart_name, version, + release_name, namespace, values + ) + + def uninstall_chart(self, cluster_id, release_name, namespace): + """ + Uninstall a Helm chart (app) from a guest cluster. + + Args: + cluster_id: Rancher management cluster ID + release_name: Helm release name + namespace: Namespace where the chart is installed + """ + logging(f"Uninstalling chart {release_name} from {namespace}") + return self.rancher.uninstall_chart( + cluster_id, release_name, namespace + ) + + def wait_for_chart_app_deleted(self, cluster_id, release_name, namespace, + timeout=DEFAULT_TIMEOUT): + """ + Wait for a chart app to be fully removed from a guest cluster. + + Args: + cluster_id: Rancher management cluster ID + release_name: Helm release name + namespace: Namespace where the chart was installed + timeout: Timeout in seconds + + Returns: + bool: True when deleted + """ + logging(f"Waiting for chart app {release_name} to be deleted") + return self.rancher.wait_for_chart_app_deleted( + cluster_id, release_name, namespace, int(timeout) + ) + def get_chart_versions(self, repo_name, chart_name, cluster_id=None): """ Get available versions for a chart from a Rancher chart repository. @@ -1020,7 +1121,8 @@ def create_cluster_repo(self, cluster_id, repo_name, git_url, git_branch): ) def wait_for_cluster_repo_ready(self, cluster_id, repo_name, - timeout=DEFAULT_TIMEOUT): + timeout=DEFAULT_TIMEOUT, + expected_git_branch=None): """ Wait for a ClusterRepo to finish downloading on a guest cluster. @@ -1028,10 +1130,12 @@ def wait_for_cluster_repo_ready(self, cluster_id, repo_name, cluster_id: Rancher management cluster ID repo_name: ClusterRepo name timeout: Maximum wait time in seconds + expected_git_branch: When set, verify spec.gitBranch matches and + re-apply if Rancher has reconciled it back """ logging(f"Waiting for ClusterRepo {repo_name} to be ready") return self.rancher.wait_for_cluster_repo_ready( - cluster_id, repo_name, timeout + cluster_id, repo_name, timeout, expected_git_branch ) def create_cloud_config_secret(self, cluster_id, secret_name, @@ -1066,7 +1170,8 @@ def write_cloud_config_to_nodes(self, cluster_id, secret_name, namespace): ) def wait_for_chart_app_ready(self, cluster_id, release_name, - namespace, timeout=DEFAULT_TIMEOUT): + namespace, timeout=DEFAULT_TIMEOUT, + expected_version=None): """ Wait for a chart app to be deployed and ready. @@ -1075,10 +1180,11 @@ def wait_for_chart_app_ready(self, cluster_id, release_name, release_name: Helm release name namespace: Namespace where chart was installed timeout: Timeout in seconds + expected_version: If set, also wait for this exact chart version """ logging(f"Waiting for chart app {release_name} to be ready") return self.rancher.wait_for_chart_app_ready( - cluster_id, release_name, namespace, int(timeout) + cluster_id, release_name, namespace, int(timeout), expected_version ) # RWX Volume / StorageClass / StatefulSet Operations diff --git a/harvester_robot_tests/libs/rancher/base.py b/harvester_robot_tests/libs/rancher/base.py index 11cdba50c..f061d8a15 100644 --- a/harvester_robot_tests/libs/rancher/base.py +++ b/harvester_robot_tests/libs/rancher/base.py @@ -207,6 +207,17 @@ def wait_for_deployment_deleted(self, cluster_id, namespace, name, timeout): """Wait for deployment to be deleted""" pass + @abstractmethod + def scale_deployment(self, cluster_id, namespace, name, replicas): + """Scale a deployment in a guest cluster to the given replica count""" + pass + + @abstractmethod + def wait_for_deployment_scaled(self, cluster_id, namespace, name, + replicas, timeout): + """Wait for a deployment to reach the given ready replica count""" + pass + # PVC Operations @abstractmethod def create_pvc(self, cluster_id, name, size="1Gi", storage_class=None): @@ -385,13 +396,31 @@ def install_chart(self, cluster_id, repo_name, chart_name, version, """Install a Helm chart on a guest cluster via Rancher catalog API.""" pass + @abstractmethod + def upgrade_chart(self, cluster_id, repo_name, chart_name, version, + release_name, namespace, values=None): + """Upgrade an installed Helm chart on a guest cluster.""" + pass + + @abstractmethod + def uninstall_chart(self, cluster_id, release_name, namespace): + """Uninstall a Helm chart (app) from a guest cluster.""" + pass + + @abstractmethod + def wait_for_chart_app_deleted(self, cluster_id, release_name, + namespace, timeout): + """Wait for a chart app to be fully removed from a guest cluster.""" + pass + @abstractmethod def create_cluster_repo(self, cluster_id, repo_name, git_url, git_branch): """Create a ClusterRepo on a guest cluster.""" pass @abstractmethod - def wait_for_cluster_repo_ready(self, cluster_id, repo_name, timeout): + def wait_for_cluster_repo_ready(self, cluster_id, repo_name, timeout, + expected_git_branch=None): """Wait for a ClusterRepo to finish downloading on a guest cluster.""" pass @@ -418,7 +447,7 @@ def write_cloud_config_to_nodes(self, cluster_id, secret_name, namespace): @abstractmethod def wait_for_chart_app_ready(self, cluster_id, release_name, - namespace, timeout): + namespace, timeout, expected_version=None): """Wait for a chart app to be deployed and ready.""" pass diff --git a/harvester_robot_tests/libs/rancher/crd.py b/harvester_robot_tests/libs/rancher/crd.py index 511654266..9de1f3ff0 100644 --- a/harvester_robot_tests/libs/rancher/crd.py +++ b/harvester_robot_tests/libs/rancher/crd.py @@ -1380,6 +1380,49 @@ def wait_for_deployment_deleted(self, cluster_id, namespace, name, timeout=DEFAU raise Exception(f"Timeout waiting for deployment {name} to be deleted") + def scale_deployment(self, cluster_id, namespace, name, replicas): + """Scale a deployment in a guest cluster to the given replica count""" + logging(f"Scaling deployment {name} to {replicas} replicas " + f"in cluster {cluster_id}") + + rc, stdout, stderr = self._run_kubectl_guest( + cluster_id, + ["scale", "deployment", name, "-n", namespace, + f"--replicas={replicas}"] + ) + + if rc != 0: + raise Exception(f"Failed to scale deployment {name}: {stderr}") + + logging(f"Scaled deployment {name} to {replicas} replicas") + + def wait_for_deployment_scaled(self, cluster_id, namespace, name, + replicas, timeout=DEFAULT_TIMEOUT): + """Wait for a deployment to reach the given ready replica count""" + logging(f"Waiting for deployment {name} to scale to {replicas}") + retry_count, retry_interval = get_retry_count_and_interval() + replicas = int(replicas) + + end_time = time.time() + int(timeout) + while time.time() < end_time: + try: + deployment = self.get_deployment(cluster_id, namespace, name) + if deployment: + status = deployment.get("status", {}) + ready = status.get("readyReplicas", 0) + if ready == replicas: + logging(f"Deployment {name} scaled to {replicas}") + return deployment + except Exception as e: + logging(f"Error checking deployment scale: {e}", + level="WARNING") + + time.sleep(retry_interval) + + raise Exception( + f"Timeout waiting for deployment {name} to scale to {replicas}" + ) + def create_pvc(self, cluster_id, name, size="1Gi", storage_class=None): """Create PVC in guest cluster via kubectl""" logging(f"Creating PVC {name} in cluster {cluster_id}") @@ -2184,7 +2227,7 @@ def install_chart(self, cluster_id, repo_name, chart_name, version, namespace: Target namespace for the chart values: Optional dict of chart values overrides """ - logging(f"Installing chart {chart_name} v{version} as {release_name} " + logging(f"Installing chart {chart_name} {version} as {release_name} " f"in {namespace} on cluster {cluster_id}") payload = { @@ -2216,9 +2259,121 @@ def install_chart(self, cluster_id, repo_name, chart_name, version, f"Failed to install chart {chart_name}: {stderr[:500]}" ) - logging(f"Chart install initiated: {chart_name} v{version}") + logging(f"Chart install initiated: {chart_name} {version}") return json.loads(stdout) if stdout.strip() else {} + def upgrade_chart(self, cluster_id, repo_name, chart_name, version, + release_name, namespace, values=None): + """Upgrade an installed Helm chart on a guest cluster via Rancher + catalog API. + + Args: + cluster_id: Rancher management cluster ID (e.g. c-m-xxxxx) + repo_name: Chart repository name (e.g. rancher-charts) + chart_name: Chart name (e.g. harvester-csi-driver) + version: Target chart version + release_name: Helm release name + namespace: Namespace where the chart is installed + values: Optional dict of chart values overrides + """ + logging(f"Upgrading chart {chart_name} to {version} as " + f"{release_name} in {namespace} on cluster {cluster_id}") + + payload = { + "charts": [ + { + "chartName": chart_name, + "version": version, + "releaseName": release_name, + "annotations": { + "catalog.cattle.io/ui-source-repo": repo_name, + "catalog.cattle.io/ui-source-repo-type": "cluster" + }, + "values": values or {} + } + ], + "namespace": namespace + } + + rc, stdout, stderr = self._run_kubectl_raw_guest( + cluster_id, + f"/v1/catalog.cattle.io.clusterrepos/{repo_name}" + f"?action=upgrade", + method="create", + input_data=json.dumps(payload) + ) + + if rc != 0: + raise Exception( + f"Failed to upgrade chart {chart_name}: {stderr[:500]}" + ) + + logging(f"Chart upgrade initiated: {chart_name} v{version}") + return json.loads(stdout) if stdout.strip() else {} + + def uninstall_chart(self, cluster_id, release_name, namespace): + """Uninstall a Helm chart (app) from a guest cluster via Rancher + catalog API. + + Args: + cluster_id: Rancher management cluster ID + release_name: Helm release name + namespace: Namespace where the chart is installed + """ + logging(f"Uninstalling chart {release_name} from {namespace} " + f"on cluster {cluster_id}") + + rc, stdout, stderr = self._run_kubectl_raw_guest( + cluster_id, + f"/v1/catalog.cattle.io.apps/{namespace}/{release_name}" + f"?action=uninstall", + method="create", + input_data=json.dumps({}) + ) + + if rc != 0: + if "NotFound" in stderr or "not found" in stderr.lower(): + logging(f"Chart app {release_name} not found; " + f"nothing to uninstall") + return {} + raise Exception( + f"Failed to uninstall chart {release_name}: {stderr[:500]}" + ) + + logging(f"Chart uninstall initiated: {release_name}") + return json.loads(stdout) if stdout.strip() else {} + + def wait_for_chart_app_deleted(self, cluster_id, release_name, namespace, + timeout=DEFAULT_TIMEOUT): + """Wait for a chart app to be fully removed from a guest cluster. + + Args: + cluster_id: Rancher management cluster ID + release_name: Helm release name + namespace: Namespace where the chart was installed + timeout: Timeout in seconds + """ + logging(f"Waiting for chart app {release_name} to be deleted " + f"from {namespace}") + retry_count, retry_interval = get_retry_count_and_interval() + + end_time = time.time() + int(timeout) + while time.time() < end_time: + rc, stdout, stderr = self._run_kubectl_guest( + cluster_id, + ["get", "apps.catalog.cattle.io", release_name, + "-n", namespace, "-o", "json"] + ) + if rc != 0 and ("NotFound" in stderr or + "not found" in stderr.lower()): + logging(f"Chart app {release_name} has been deleted") + return True + time.sleep(retry_interval) + + raise Exception( + f"Timeout waiting for chart app {release_name} to be deleted" + ) + def create_cluster_repo(self, cluster_id, repo_name, git_url, git_branch): """Create a ClusterRepo on a guest cluster via kubectl. @@ -2261,7 +2416,8 @@ def create_cluster_repo(self, cluster_id, repo_name, git_url, git_branch): return repo_spec def wait_for_cluster_repo_ready(self, cluster_id, repo_name, - timeout=DEFAULT_TIMEOUT): + timeout=DEFAULT_TIMEOUT, + expected_git_branch=None): """Wait for a ClusterRepo to finish downloading on a guest cluster. Requires Downloaded=True on two consecutive polls to avoid the @@ -2272,10 +2428,15 @@ def wait_for_cluster_repo_ready(self, cluster_id, repo_name, cluster_id: Rancher management cluster ID repo_name: ClusterRepo name timeout: Maximum wait time in seconds + expected_git_branch: When set, verify spec.gitBranch matches """ - logging(f"Waiting for ClusterRepo {repo_name} to be ready") + logging(f"Waiting for ClusterRepo {repo_name} to be ready" + + (f" (branch={expected_git_branch})" + if expected_git_branch else "")) end_time = time.time() + int(timeout) consecutive_ready = 0 + reapply_count = 0 + max_reapply = 3 while time.time() < end_time: rc, stdout, stderr = self._run_kubectl_guest( @@ -2284,6 +2445,7 @@ def wait_for_cluster_repo_ready(self, cluster_id, repo_name, "-o", "json"] ) downloaded = False + data = {} if rc == 0 and stdout.strip(): data = json.loads(stdout) conditions = data.get("status", {}).get("conditions", []) @@ -2296,8 +2458,51 @@ def wait_for_cluster_repo_ready(self, cluster_id, repo_name, if downloaded: consecutive_ready += 1 if consecutive_ready >= 2: + # Verify branch matches expected value if provided + if expected_git_branch: + spec = data.get("spec", {}) + current_branch = spec.get("gitBranch", "") + current_url = spec.get("gitRepo", "") + + if current_branch != expected_git_branch: + if reapply_count >= max_reapply: + raise Exception( + f"ClusterRepo {repo_name} branch mismatch " + f"after {max_reapply} re-apply attempts. " + f"Expected branch={expected_git_branch}, " + f"got branch={current_branch}. " + f"Rancher is reconciling the repo back to " + f"its built-in defaults." + ) + logging( + f"ClusterRepo {repo_name} branch mismatch " + f"(got={current_branch}, " + f"expected={expected_git_branch}), " + f"re-applying " + f"(attempt {reapply_count + 1}/{max_reapply})", + level="WARNING" + ) + repo_spec = { + "apiVersion": "catalog.cattle.io/v1", + "kind": "ClusterRepo", + "metadata": {"name": repo_name}, + "spec": { + "gitRepo": current_url, + "gitBranch": expected_git_branch + } + } + self._run_kubectl_guest( + cluster_id, + ["apply", "-f", "-"], + input_data=json.dumps(repo_spec) + ) + reapply_count += 1 + consecutive_ready = 0 + time.sleep(5) + continue + logging(f"ClusterRepo {repo_name} is ready " - f"(stable after {consecutive_ready} checks)") + f"(stable, spec verified)") return data else: consecutive_ready = 0 @@ -2562,7 +2767,8 @@ def write_cloud_config_to_nodes(self, cluster_id, secret_name, namespace): logging(f"Deleted DaemonSet {ds_name}") def wait_for_chart_app_ready(self, cluster_id, release_name, - namespace, timeout=DEFAULT_TIMEOUT): + namespace, timeout=DEFAULT_TIMEOUT, + expected_version=None): """Wait for a chart app to be deployed and ready. Args: @@ -2572,9 +2778,15 @@ def wait_for_chart_app_ready(self, cluster_id, release_name, timeout: Timeout in seconds """ logging(f"Waiting for chart app {release_name} to be ready " - f"in {namespace}") + f"in {namespace}" + + (f" at version {expected_version}" if expected_version else "")) retry_count, retry_interval = get_retry_count_and_interval() + # Give the async helm operation pod time to start so the status + # transitions away from the stale pre-upgrade 'deployed' state. + if expected_version: + time.sleep(5) + end_time = time.time() + int(timeout) iteration = 0 while time.time() < end_time: @@ -2588,13 +2800,24 @@ def wait_for_chart_app_ready(self, cluster_id, release_name, data = json.loads(stdout) info = data.get("spec", {}).get("info", {}) status = info.get("status", "") - if status == "deployed": - logging(f"Chart app {release_name} is deployed") - return data + version = (data.get("spec", {}) + .get("chart", {}) + .get("metadata", {}) + .get("version", "")) + + if expected_version: + if status == "deployed" and version == expected_version: + logging(f"Chart app {release_name} deployed " + f"at {expected_version}") + return data + else: + if status == "deployed": + logging(f"Chart app {release_name} is deployed") + return data if iteration % 10 == 0: logging(f"Chart app {release_name} status: " - f"{status}") + f"{status}, version: {version}") elif iteration % 10 == 0: logging(f"Chart app {release_name} not found yet: " f"{stderr.strip()[:200]}", level="WARNING") @@ -2608,6 +2831,7 @@ def wait_for_chart_app_ready(self, cluster_id, release_name, raise Exception( f"Timeout waiting for chart app {release_name} to be ready" + + (f" at version {expected_version}" if expected_version else "") ) # RWX Volume / StorageClass / StatefulSet Operations diff --git a/harvester_robot_tests/libs/rancher/rancher.py b/harvester_robot_tests/libs/rancher/rancher.py index 3427b3125..91b16dcb1 100644 --- a/harvester_robot_tests/libs/rancher/rancher.py +++ b/harvester_robot_tests/libs/rancher/rancher.py @@ -171,6 +171,19 @@ def wait_for_deployment_deleted(self, cluster_id, namespace, name, timeout): """Wait for deployment to be deleted""" return self.rancher.wait_for_deployment_deleted(cluster_id, namespace, name, timeout) + def scale_deployment(self, cluster_id, namespace, name, replicas): + """Scale a deployment to the given replica count""" + return self.rancher.scale_deployment( + cluster_id, namespace, name, replicas + ) + + def wait_for_deployment_scaled(self, cluster_id, namespace, name, + replicas, timeout): + """Wait for a deployment to reach the given ready replica count""" + return self.rancher.wait_for_deployment_scaled( + cluster_id, namespace, name, replicas, timeout + ) + # PVC Operations def create_pvc(self, cluster_id, name, size="1Gi", storage_class=None): """Create PVC in guest cluster""" @@ -285,6 +298,27 @@ def install_chart(self, cluster_id, repo_name, chart_name, version, release_name, namespace, values ) + def upgrade_chart(self, cluster_id, repo_name, chart_name, version, + release_name, namespace, values=None): + """Upgrade an installed Helm chart on a guest cluster""" + return self.rancher.upgrade_chart( + cluster_id, repo_name, chart_name, version, + release_name, namespace, values + ) + + def uninstall_chart(self, cluster_id, release_name, namespace): + """Uninstall a Helm chart from a guest cluster""" + return self.rancher.uninstall_chart( + cluster_id, release_name, namespace + ) + + def wait_for_chart_app_deleted(self, cluster_id, release_name, + namespace, timeout): + """Wait for a chart app to be fully removed""" + return self.rancher.wait_for_chart_app_deleted( + cluster_id, release_name, namespace, timeout + ) + def create_cluster_repo(self, cluster_id, repo_name, git_url, git_branch): """Create a ClusterRepo on a guest cluster""" return self.rancher.create_cluster_repo( @@ -292,10 +326,10 @@ def create_cluster_repo(self, cluster_id, repo_name, git_url, git_branch): ) def wait_for_cluster_repo_ready(self, cluster_id, repo_name, - timeout=600): + timeout=600, expected_git_branch=None): """Wait for a ClusterRepo to finish downloading""" return self.rancher.wait_for_cluster_repo_ready( - cluster_id, repo_name, timeout + cluster_id, repo_name, timeout, expected_git_branch ) def get_chart_versions(self, repo_name, chart_name, cluster_id=None): @@ -324,10 +358,10 @@ def write_cloud_config_to_nodes(self, cluster_id, secret_name, namespace): ) def wait_for_chart_app_ready(self, cluster_id, release_name, - namespace, timeout): + namespace, timeout, expected_version=None): """Wait for chart app to be deployed""" return self.rancher.wait_for_chart_app_ready( - cluster_id, release_name, namespace, timeout + cluster_id, release_name, namespace, timeout, expected_version ) # RWX Volume / StorageClass / StatefulSet Operations diff --git a/harvester_robot_tests/libs/rancher/rest.py b/harvester_robot_tests/libs/rancher/rest.py index 9f8563dad..0ffb18086 100644 --- a/harvester_robot_tests/libs/rancher/rest.py +++ b/harvester_robot_tests/libs/rancher/rest.py @@ -1034,6 +1034,55 @@ def wait_for_deployment_deleted(self, cluster_id, namespace, name, timeout=DEFAU raise Exception(f"Timeout waiting for deployment {name} to be deleted") + def scale_deployment(self, cluster_id, namespace, name, replicas): + """Scale a deployment in a guest cluster to the given replica count""" + logging(f"Scaling deployment {name} to {replicas} replicas " + f"in cluster {cluster_id}") + + deployment = self.get_deployment(cluster_id, namespace, name) + if deployment is None: + raise Exception(f"Deployment {name} not found") + + deployment.setdefault("spec", {})["replicas"] = int(replicas) + + code, data = self._rancher_request( + "PUT", + f"k8s/clusters/{cluster_id}/v1/apps.deployments/{namespace}/{name}", + deployment + ) + + if code not in [200, 201]: + raise Exception(f"Failed to scale deployment {name}: {code}, {data}") + + logging(f"Scaled deployment {name} to {replicas} replicas") + + def wait_for_deployment_scaled(self, cluster_id, namespace, name, + replicas, timeout=DEFAULT_TIMEOUT): + """Wait for a deployment to reach the given ready replica count""" + logging(f"Waiting for deployment {name} to scale to {replicas}") + retry_count, retry_interval = get_retry_count_and_interval() + replicas = int(replicas) + + end_time = time.time() + int(timeout) + while time.time() < end_time: + try: + deployment = self.get_deployment(cluster_id, namespace, name) + if deployment: + status = deployment.get("status", {}) + ready = status.get("readyReplicas", 0) + if ready == replicas: + logging(f"Deployment {name} scaled to {replicas}") + return deployment + except Exception as e: + logging(f"Error checking deployment scale: {e}", + level="WARNING") + + time.sleep(retry_interval) + + raise Exception( + f"Timeout waiting for deployment {name} to scale to {replicas}" + ) + def create_pvc(self, cluster_id, name, size="1Gi", storage_class=None): """Create PVC in guest cluster""" logging(f"Creating PVC {name} in cluster {cluster_id}") @@ -1836,6 +1885,95 @@ def install_chart(self, cluster_id, repo_name, chart_name, version, logging(f"Chart install initiated: {chart_name} v{version}") return data + def upgrade_chart(self, cluster_id, repo_name, chart_name, version, + release_name, namespace, values=None): + """Upgrade an installed Helm chart on a guest cluster via Rancher + catalog API using the 'upgrade' action.""" + logging(f"Upgrading chart {chart_name} to v{version} as " + f"{release_name} in {namespace} on cluster {cluster_id}") + + payload = { + "charts": [ + { + "chartName": chart_name, + "version": version, + "releaseName": release_name, + "annotations": { + "catalog.cattle.io/ui-source-repo": repo_name, + "catalog.cattle.io/ui-source-repo-type": "cluster" + }, + "values": values or {} + } + ], + "namespace": namespace + } + + code, data = self._rancher_request( + "POST", + f"k8s/clusters/{cluster_id}" + f"/v1/catalog.cattle.io.clusterrepos/{repo_name}" + f"?action=upgrade", + data=payload + ) + + if code not in [200, 201]: + raise Exception( + f"Failed to upgrade chart {chart_name}: " + f"{code}, {str(data)[:500]}" + ) + + logging(f"Chart upgrade initiated: {chart_name} v{version}") + return data + + def uninstall_chart(self, cluster_id, release_name, namespace): + """Uninstall a Helm chart (app) from a guest cluster via Rancher + catalog API.""" + logging(f"Uninstalling chart {release_name} from {namespace} " + f"on cluster {cluster_id}") + + code, data = self._rancher_request( + "POST", + f"k8s/clusters/{cluster_id}" + f"/v1/catalog.cattle.io.apps/{namespace}/{release_name}" + f"?action=uninstall", + data={} + ) + + if code == 404: + logging(f"Chart app {release_name} not found; " + f"nothing to uninstall") + return {} + if code not in [200, 201]: + raise Exception( + f"Failed to uninstall chart {release_name}: " + f"{code}, {str(data)[:500]}" + ) + + logging(f"Chart uninstall initiated: {release_name}") + return data + + def wait_for_chart_app_deleted(self, cluster_id, release_name, namespace, + timeout=DEFAULT_TIMEOUT): + """Wait for a chart app to be fully removed from a guest cluster.""" + logging(f"Waiting for chart app {release_name} to be deleted " + f"from {namespace}") + retry_count, retry_interval = get_retry_count_and_interval() + + end_time = time.time() + int(timeout) + while time.time() < end_time: + code, data = self._rancher_proxy_request( + "GET", cluster_id, + f"v1/catalog.cattle.io.apps/{namespace}/{release_name}" + ) + if code == 404: + logging(f"Chart app {release_name} has been deleted") + return True + time.sleep(retry_interval) + + raise Exception( + f"Timeout waiting for chart app {release_name} to be deleted" + ) + def create_cluster_repo(self, cluster_id, repo_name, git_url, git_branch): """Create a ClusterRepo on a guest cluster via Rancher proxy.""" logging(f"Creating ClusterRepo {repo_name} on cluster {cluster_id} " @@ -1871,15 +2009,21 @@ def create_cluster_repo(self, cluster_id, repo_name, git_url, git_branch): return data def wait_for_cluster_repo_ready(self, cluster_id, repo_name, - timeout=DEFAULT_TIMEOUT): + timeout=DEFAULT_TIMEOUT, + expected_git_branch=None): """Wait for a ClusterRepo to finish downloading on a guest cluster. Requires Downloaded=True on two consecutive polls to avoid the false-positive that occurs briefly after initial creation. + """ - logging(f"Waiting for ClusterRepo {repo_name} to be ready") + logging(f"Waiting for ClusterRepo {repo_name} to be ready" + + (f" (branch={expected_git_branch})" + if expected_git_branch else "")) end_time = time.time() + int(timeout) consecutive_ready = 0 + reapply_count = 0 + max_reapply = 3 while time.time() < end_time: code, data = self._rancher_proxy_request( @@ -1898,8 +2042,43 @@ def wait_for_cluster_repo_ready(self, cluster_id, repo_name, if downloaded: consecutive_ready += 1 if consecutive_ready >= 2: + if expected_git_branch: + spec = data.get("spec", {}) + current_branch = spec.get("gitBranch", "") + + if current_branch != expected_git_branch: + if reapply_count >= max_reapply: + raise Exception( + f"ClusterRepo {repo_name} branch mismatch " + f"after {max_reapply} re-apply attempts. " + f"Expected branch={expected_git_branch}, " + f"got branch={current_branch}. " + f"Rancher is reconciling the repo back to " + f"its built-in defaults." + ) + logging( + f"ClusterRepo {repo_name} branch mismatch " + f"(got={current_branch}, " + f"expected={expected_git_branch}), " + f"re-applying via PUT " + f"(attempt {reapply_count + 1}/{max_reapply})", + level="WARNING" + ) + updated = dict(data) + updated.setdefault("spec", {}) + updated["spec"]["gitBranch"] = expected_git_branch + self._rancher_proxy_request( + "PUT", cluster_id, + f"v1/catalog.cattle.io.clusterrepos/{repo_name}", + updated + ) + reapply_count += 1 + consecutive_ready = 0 + time.sleep(5) + continue + logging(f"ClusterRepo {repo_name} is ready " - f"(stable after {consecutive_ready} checks)") + f"(stable, spec verified)") return data else: consecutive_ready = 0 @@ -2146,12 +2325,17 @@ def write_cloud_config_to_nodes(self, cluster_id, secret_name, namespace): logging(f"Deleted DaemonSet {ds_name}") def wait_for_chart_app_ready(self, cluster_id, release_name, - namespace, timeout=DEFAULT_TIMEOUT): + namespace, timeout=DEFAULT_TIMEOUT, + expected_version=None): """Wait for a chart app to be deployed and ready.""" logging(f"Waiting for chart app {release_name} to be ready " - f"in {namespace}") + f"in {namespace}" + + (f" at version {expected_version}" if expected_version else "")) retry_count, retry_interval = get_retry_count_and_interval() + if expected_version: + time.sleep(5) + end_time = time.time() + int(timeout) iteration = 0 while time.time() < end_time: @@ -2163,13 +2347,24 @@ def wait_for_chart_app_ready(self, cluster_id, release_name, if code == 200 and data: info = data.get("spec", {}).get("info", {}) status = info.get("status", "") - if status == "deployed": - logging(f"Chart app {release_name} is deployed") - return data + version = (data.get("spec", {}) + .get("chart", {}) + .get("metadata", {}) + .get("version", "")) + + if expected_version: + if status == "deployed" and version == expected_version: + logging(f"Chart app {release_name} deployed " + f"at {expected_version}") + return data + else: + if status == "deployed": + logging(f"Chart app {release_name} is deployed") + return data if iteration % 10 == 0: logging(f"Chart app {release_name} status: " - f"{status}") + f"{status}, version: {version}") except Exception as e: if iteration % 10 == 0: logging(f"Error checking chart app: {e}", @@ -2180,6 +2375,7 @@ def wait_for_chart_app_ready(self, cluster_id, release_name, raise Exception( f"Timeout waiting for chart app {release_name} to be ready" + + (f" at version {expected_version}" if expected_version else "") ) # RWX Volume / StorageClass / StatefulSet Operations diff --git a/harvester_robot_tests/tests/regression/test_rancher_integration.robot b/harvester_robot_tests/tests/regression/rancher/test_rancher_integration.robot similarity index 75% rename from harvester_robot_tests/tests/regression/test_rancher_integration.robot rename to harvester_robot_tests/tests/regression/rancher/test_rancher_integration.robot index e64ffca58..8b93f0953 100644 --- a/harvester_robot_tests/tests/regression/test_rancher_integration.robot +++ b/harvester_robot_tests/tests/regression/rancher/test_rancher_integration.robot @@ -38,20 +38,20 @@ Test RWX Volume On Single Node Cluster # [Teardown] Cleanup RWX test resources on single node cluster Test Create Multi Node RKE2 Cluster - [Tags] p0 + [Tags] p0 cloudprovider csi [Documentation] Create and verify a 3-node RKE2 cluster. When Multi node RKE2 cluster is created Then Harvester deployments should be ready ${MULTI_CLUSTER_ID} Test CSI Deployment - [Tags] p0 + [Tags] p0 csi [Documentation] Deploy a CSI workload with PVC on the multi-node cluster. Given Multi node cluster is available When CSI workload with PVC is deployed Then CSI deployment and PVC should be active Test Load Balancer DHCP Mode - [Tags] p0 + [Tags] p0 cloudprovider [Documentation] Deploy a Whoami workload and create a LoadBalancer service in DHCP mode. Given Multi node cluster is available And Whoami workload is deployed dhcp @@ -60,7 +60,7 @@ Test Load Balancer DHCP Mode Then DHCP load balancer should be serving traffic Test Load Balancer Pool Mode - [Tags] p0 + [Tags] p0 cloudprovider [Documentation] Deploy a Whoami workload and create a LoadBalancer service in IP Pool mode. Given Multi node cluster is available And Whoami workload is deployed pool @@ -146,19 +146,63 @@ Test Import Existing RKE2 Cluster When Single node import RKE2 cluster is created Then Import cluster should be ready and running +Test Upgrade Harvester CSI Driver Chart On Imported RKE2 Cluster + [Tags] import chart csi upgrade p2 + [Documentation] Test CSI chart upgrade from Rancher Apps from n-1 to latest + + Given Import cluster should be ready + And Multiple Harvester CSI driver chart versions are available + + When Harvester CSI driver chart is installed on single node import cluster ${CSI_PREV_VERSION} + And CSI workload is deployed on single node import cluster + Then CSI workload should be active on single node import cluster + + When Harvester CSI driver chart is upgraded to latest on import cluster + Then Harvester CSI driver should be ready ${IMPORT_CLUSTER_ID} + + When CSI upgrade workload is restarted on import cluster + Then CSI workload should be active on single node import cluster + + When New CSI workload is deployed csiup ${IMPORT_CLUSTER_ID} ${IMPORT_CLUSTER_NAME} + Then New CSI workload should be active csiup ${IMPORT_CLUSTER_ID} ${IMPORT_CLUSTER_NAME} + + [Teardown] Cleanup CSI upgrade test resources on import cluster + Test Harvester CSI Driver Chart On Imported RKE2 Cluster [Tags] import chart csi p2 [Documentation] Tests the Harvester CSI Driver chart from Rancher Apps - [Setup] Import cluster should be ready + Given Import cluster should be ready When Harvester CSI driver chart is installed on single node import cluster Then Harvester CSI driver should be ready ${IMPORT_CLUSTER_ID} When CSI workload is deployed on single node import cluster Then CSI workload should be active on single node import cluster +Test Upgrade Harvester Cloud Provider Chart On Imported RKE2 Cluster + [Tags] import chart cloudprovider upgrade p2 + [Documentation] Test cloud provider chart upgrade from Rancher Apps from n-1 to latest + + Given Import cluster should be ready + And Multiple Harvester cloud provider chart versions are available + + When Harvester cloud provider chart is installed on single node import cluster ${CP_PREV_VERSION} + And Cloud provider workloads are deployed on single node import cluster + Then Cloud provider workloads should be active on single node import cluster + + When Harvester cloud provider chart is upgraded to latest on import cluster + Then Harvester cloud provider should be ready ${IMPORT_CLUSTER_ID} + + When Cloud provider upgrade workload is restarted on import cluster + Then Cloud provider workloads should be active on single node import cluster + + When Cloud provider workloads are deployed on single node import cluster cpup2 + Then Cloud provider workloads should be active on single node import cluster cpup2 + + [Teardown] Cleanup cloud provider upgrade test resources on import cluster + Test Harvester Cloud Provider Chart On Imported RKE2 Cluster [Tags] import chart cloudprovider p2 [Documentation] Tests the Harvester Cloud Provider chart from Rancher Apps - [Setup] Import cluster should be ready + Given Import cluster should be ready When Harvester cloud provider chart is installed on single node import cluster Then Harvester cloud provider should be ready ${IMPORT_CLUSTER_ID} When Cloud provider workloads are deployed on single node import cluster