diff --git a/harvester_e2e_tests/integrations/test_0_storage_network.py b/harvester_e2e_tests/integrations/test_0_storage_network.py index 328f19717..829b621e9 100644 --- a/harvester_e2e_tests/integrations/test_0_storage_network.py +++ b/harvester_e2e_tests/integrations/test_0_storage_network.py @@ -74,6 +74,7 @@ def cluster_network(request, api_client, unique_name): assert 200 == code, (code, data) +@pytest.mark.robot_ported @pytest.mark.p0 @pytest.mark.smoke @pytest.mark.settings @@ -147,6 +148,7 @@ def test_enable_storage_network( assert snet_enabled, (code, data) +@pytest.mark.robot_ported @pytest.mark.p0 @pytest.mark.smoke @pytest.mark.settings diff --git a/harvester_robot_tests/keywords/common.resource b/harvester_robot_tests/keywords/common.resource index 2bcddf93a..8231f9fb0 100644 --- a/harvester_robot_tests/keywords/common.resource +++ b/harvester_robot_tests/keywords/common.resource @@ -12,12 +12,14 @@ Set up test environment common_keywords.init k8s api client Generate Unique Name - [Arguments] ${prefix1}=${EMPTY} ${prefix2}=${EMPTY} - [Documentation] Generate a unique name with optional prefixes and yymmdd-HHMMSS-ffffqa timestamp suffix - ... Generate Unique Name -> '260603-145834-1273qa' - ... Generate Unique Name foo -> 'foo-260603-145834-1273qa' - ... Generate Unique Name foo bar -> 'foo-bar-2260603-145834-1273qa' - ${name}= common_keywords.generate_name_with_suffix ${prefix1} ${prefix2} + [Arguments] ${prefix1}=${EMPTY} ${prefix2}=${EMPTY} ${precise}=${FALSE} + [Documentation] Generate a name with timestamp suffix and optional prefixes + ... The timestamp format is `%m%d%H%M` by default and `%m%d%H%M%S%f` when precise=${TRUE} + ... Examples: + ... Generate Unique Name -> '06181015' + ... Generate Unique Name foo -> 'foo-06181015' + ... Generate Unique Name foo bar precise=${TRUE}-> 'foo-bar-0618101519681156' + ${name}= common_keywords.generate_name_with_suffix ${prefix1} ${prefix2} ${precise} RETURN ${name} Cleanup test resources diff --git a/harvester_robot_tests/keywords/rancher.resource b/harvester_robot_tests/keywords/rancher.resource index 1e758e9c1..3ce31c90d 100644 --- a/harvester_robot_tests/keywords/rancher.resource +++ b/harvester_robot_tests/keywords/rancher.resource @@ -29,13 +29,6 @@ Clear State [Documentation] Clear state - either specific key or all rancher_keywords.clear_state ${key} -Generate Unique Name - [Arguments] ${prefix}=test - [Documentation] Generate unique name with timestamp - ${name}= rancher_keywords.generate_unique_name ${prefix} - Log Generated unique name: ${name} - RETURN ${name} - # Harvester Management Cluster Operations Create Harvester Mgmt Cluster [Arguments] ${cluster_name} @@ -974,13 +967,6 @@ Get Storage Network Status ${status}= storage_network.Get Status RETURN ${status} -Wait For Storage Network Ready - [Arguments] ${timeout}=${RANCHER_WAIT_TIMEOUT} - [Documentation] Wait for storage-network to be applied and completed. - ... Delegates to Storage Network module. - ${data}= storage_network.Wait For Ready ${timeout} - RETURN ${data} - Get Vlan Network CIDR [Arguments] ${vlan_id} ${cluster_network} [Documentation] Get the CIDR for a VLAN network by creating a temp network. @@ -1172,7 +1158,7 @@ Storage network is configured for RWX ${cluster_network}= Set Variable %{CLUSTER_NETWORK=mgmt} ${cidr}= Get Vlan Network CIDR ${VLAN_ID} ${cluster_network} Enable Storage Network ${VLAN_ID} ${cluster_network} ${cidr} share_rwx=${TRUE} - Wait For Storage Network Ready ${RANCHER_WAIT_TIMEOUT} + storage_network.Wait For Enabled ${cidr} ${RANCHER_WAIT_TIMEOUT} Log Storage network configured for RWX console=yes Storage network is enabled for RWX @@ -1197,7 +1183,7 @@ Storage network is enabled for RWX ${cluster_network}= Set Variable %{CLUSTER_NETWORK=mgmt} ${cidr}= Get Vlan Network CIDR ${VLAN_ID} ${cluster_network} Enable Storage Network ${VLAN_ID} ${cluster_network} ${cidr} share_rwx=${TRUE} - Wait For Storage Network Ready ${RANCHER_WAIT_TIMEOUT} + storage_network.Wait For Enabled ${cidr} ${RANCHER_WAIT_TIMEOUT} Log Storage network enabled console=yes # Start VMs back up diff --git a/harvester_robot_tests/keywords/storage_network.resource b/harvester_robot_tests/keywords/storage_network.resource index 7723a6078..677739f70 100644 --- a/harvester_robot_tests/keywords/storage_network.resource +++ b/harvester_robot_tests/keywords/storage_network.resource @@ -27,12 +27,22 @@ Get Status ${status}= storage_network_keywords.get_storage_network_status RETURN ${status} -Wait For Ready +Is Enabled + [Arguments] ${snet_cidr} + ${enabled}= storage_network_keywords.is_storage_network_enabled_in_config + Should Be True ${enabled} Storage network config (Harvester) is NOT enabled + ${enabled}= storage_network_keywords.is_storage_network_enabled_on_pods ${snet_cidr} + Should Be True ${enabled} Storage network service (Longhorn) is NOT enabled + +Wait For Enabled + [Arguments] ${snet_cidr} ${timeout}=${WAIT_TIMEOUT} + storage_network_keywords.wait_storage_network_enabled_in_config ${timeout} + storage_network_keywords.wait_storage_network_enabled_on_pods ${snet_cidr} ${timeout} + +Wait For Disabled [Arguments] ${timeout}=${WAIT_TIMEOUT} - [Documentation] Wait for storage-network to be applied and completed. - Log Waiting for storage network to be ready - ${data}= storage_network_keywords.wait_for_storage_network_ready ${timeout} - RETURN ${data} + storage_network_keywords.wait_storage_network_disabled_in_config ${timeout} + storage_network_keywords.wait_storage_network_disabled_on_pods ${timeout} Get Vlan Network CIDR [Arguments] ${vlan_id} ${cluster_network} diff --git a/harvester_robot_tests/libs/crd.py b/harvester_robot_tests/libs/crd.py index cbe1580d6..ebfe06b60 100644 --- a/harvester_robot_tests/libs/crd.py +++ b/harvester_robot_tests/libs/crd.py @@ -22,6 +22,19 @@ def get_cr(group, version, namespace, plural, name): ) +def get_cluster_cr(group, version, plural, name): + """ + Get a cluster-scoped Custom Resource + """ + obj_api = client.CustomObjectsApi() + return obj_api.get_cluster_custom_object( + group=group, + version=version, + plural=plural, + name=name + ) + + def create_cr(group, version, namespace, plural, body): """ Create a Custom Resource @@ -36,6 +49,19 @@ def create_cr(group, version, namespace, plural, body): ) +def create_cluster_cr(group, version, plural, body): + """ + Create a cluster-scoped Custom Resource + """ + obj_api = client.CustomObjectsApi() + return obj_api.create_cluster_custom_object( + group=group, + version=version, + plural=plural, + body=body + ) + + def delete_cr(group, version, namespace, plural, name): """ Delete a Custom Resource @@ -50,6 +76,19 @@ def delete_cr(group, version, namespace, plural, name): ) +def delete_cluster_cr(group, version, plural, name): + """ + Delete a cluster-scoped Custom Resource + """ + obj_api = client.CustomObjectsApi() + return obj_api.delete_cluster_custom_object( + group=group, + version=version, + plural=plural, + name=name + ) + + def list_cr(group, version, namespace, plural, label_selector=None): """ List Custom Resources @@ -64,6 +103,19 @@ def list_cr(group, version, namespace, plural, label_selector=None): ) +def list_cluster_cr(group, version, plural, label_selector=None): + """ + List cluster-scoped Custom Resources + """ + obj_api = client.CustomObjectsApi() + return obj_api.list_cluster_custom_object( + group=group, + version=version, + plural=plural, + label_selector=label_selector + ) + + def patch_cr(group, version, namespace, plural, name, body): """ Patch a Custom Resource @@ -79,6 +131,20 @@ def patch_cr(group, version, namespace, plural, name, body): ) +def patch_cluster_cr(group, version, plural, name, body): + """ + Patch a cluster-scoped Custom Resource + """ + obj_api = client.CustomObjectsApi() + return obj_api.patch_cluster_custom_object( + group=group, + version=version, + plural=plural, + name=name, + body=body + ) + + def replace_cr(group, version, namespace, plural, name, body): """ Replace a Custom Resource @@ -94,6 +160,20 @@ def replace_cr(group, version, namespace, plural, name, body): ) +def replace_cluster_cr(group, version, plural, name, body): + """ + Replace a cluster-scoped Custom Resource + """ + obj_api = client.CustomObjectsApi() + return obj_api.replace_cluster_custom_object( + group=group, + version=version, + plural=plural, + name=name, + body=body + ) + + def wait_for_cr_deleted(group, version, namespace, plural, name, timeout=DEFAULT_TIMEOUT_SHORT): """ Wait for a Custom Resource to be deleted @@ -116,6 +196,34 @@ def wait_for_cr_deleted(group, version, namespace, plural, name, timeout=DEFAULT raise AssertionError(f"Expected {plural}/{name} to be deleted but it still exists") +def wait_for_cluster_cr_deleted(group, version, plural, name, timeout=DEFAULT_TIMEOUT_SHORT): + """ + Wait for a cluster-scoped Custom Resource to be deleted + """ + obj_api = client.CustomObjectsApi() + retry_count, retry_interval = get_retry_count_and_interval() + + for i in range(retry_count): + logging(f"Waiting for {plural}/{name} to be deleted... ({i})") + try: + obj_api.get_cluster_custom_object( + group=group, + version=version, + plural=plural, + name=name + ) + except ApiException as e: + if e.reason == 'Not Found': + logging(f"Successfully deleted {plural}/{name}") + return True + except Exception as e: + logging(f"Error checking {plural}/{name} deletion: {e}") + + time.sleep(retry_interval) + + raise AssertionError(f"Expected {plural}/{name} to be deleted but it still exists") + + def wait_for_cr_status(group, version, namespace, plural, name, status_field, expected_value, timeout=DEFAULT_TIMEOUT_SHORT): # NOQA """ Wait for a CR status field to reach expected value @@ -143,6 +251,39 @@ def wait_for_cr_status(group, version, namespace, plural, name, status_field, ex ) +def wait_for_cluster_cr_status( + group, version, plural, name, status_field, + expected_value, timeout=DEFAULT_TIMEOUT_SHORT +): + """ + Wait for a cluster-scoped CR status field to reach expected value + """ + retry_count, retry_interval = get_retry_count_and_interval() + + cr = None + for i in range(retry_count): + logging( + f"Waiting for {plural}/{name} " + f"{status_field}={expected_value} ({i})..." + ) + try: + cr = get_cluster_cr(group, version, plural, name) + actual_value = cr.get("status", {}).get(status_field) + if actual_value == expected_value: + logging(f"{plural}/{name} {status_field}={expected_value}") + return True + except Exception as e: + logging(f"Error getting {plural}/{name} status: {e}") + + time.sleep(retry_interval) + + actual_value = cr.get("status", {}).get(status_field) if cr else None + raise AssertionError( + f"Expected {plural}/{name} {status_field}={expected_value}, " + f"but got {actual_value}" + ) + + def wait_for_cr_condition( group, version, namespace, plural, name, condition_type, condition_status, timeout=DEFAULT_TIMEOUT_SHORT @@ -177,6 +318,50 @@ def wait_for_cr_condition( ) +def wait_for_cluster_cr_condition( + group, version, plural, name, + condition_type, condition_status, timeout=DEFAULT_TIMEOUT_SHORT +): + """ + Wait for a cluster-scoped CR condition to reach expected status + """ + retry_count, retry_interval = get_retry_count_and_interval() + + cr = None + for i in range(retry_count): + logging( + f"Waiting for {plural}/{name} condition " + f"{condition_type}={condition_status} ({i})..." + ) + try: + cr = get_cluster_cr(group, version, plural, name) + conditions = cr.get("status", {}).get("conditions", []) + for condition in conditions: + type_match = ( + condition.get("type", "").lower() + == condition_type.lower() + ) + status_match = ( + condition.get("status", "").lower() + == condition_status.lower() + ) + if type_match and status_match: + logging( + f"{plural}/{name} condition " + f"{condition_type}={condition_status}" + ) + return True + except Exception as e: + logging(f"Error getting {plural}/{name} conditions: {e}") + + time.sleep(retry_interval) + + raise AssertionError( + f"Failed to wait for {plural}/{name} condition " + f"{condition_type}={condition_status}" + ) + + def set_cr_annotation(group, version, namespace, plural, name, annotation_key, annotation_value): """ Set an annotation on a Custom Resource @@ -209,6 +394,45 @@ def set_cr_annotation(group, version, namespace, plural, name, annotation_key, a ) +def set_cluster_cr_annotation( + group, version, plural, name, annotation_key, annotation_value +): + """ + Set an annotation on a cluster-scoped Custom Resource + """ + retry_count, retry_interval = get_retry_count_and_interval() + + for i in range(retry_count): + try: + cr = get_cluster_cr(group, version, plural, name) + annotations = cr.get('metadata', {}).get('annotations', {}) + annotations[annotation_key] = annotation_value + if 'metadata' not in cr: + cr['metadata'] = {} + cr['metadata']['annotations'] = annotations + + replace_cluster_cr(group, version, plural, name, cr) + logging( + f"Set annotation {annotation_key}={annotation_value} " + f"on {plural}/{name}" + ) + return + except ApiException as e: + if e.status == 409: + logging( + f"Conflict error when setting annotation, retry ({i})..." + ) + else: + raise e + + time.sleep(retry_interval) + + raise AssertionError( + f"Failed to set annotation on {plural}/{name} " + f"after {retry_count} retries" + ) + + def get_cr_annotation_value(group, version, namespace, plural, name, annotation_key): """ Get an annotation value from a Custom Resource @@ -217,6 +441,16 @@ def get_cr_annotation_value(group, version, namespace, plural, name, annotation_ return cr.get('metadata', {}).get('annotations', {}).get(annotation_key) +def get_cluster_cr_annotation_value( + group, version, plural, name, annotation_key +): + """ + Get an annotation value from a cluster-scoped Custom Resource + """ + cr = get_cluster_cr(group, version, plural, name) + return cr.get('metadata', {}).get('annotations', {}).get(annotation_key) + + def convert_size_to_bytes(size_str): """ Convert size string (like '10Gi', '5Mi') to bytes diff --git a/harvester_robot_tests/libs/keywords/common_keywords.py b/harvester_robot_tests/libs/keywords/common_keywords.py index fcf2b2783..d1874cfef 100644 --- a/harvester_robot_tests/libs/keywords/common_keywords.py +++ b/harvester_robot_tests/libs/keywords/common_keywords.py @@ -12,6 +12,7 @@ from utility.utility import init_harvester_api_client # noqa E402 from utility.utility import init_k8s_api_client # noqa E402 from utility.utility import logging # noqa E402 +from constant import LONGHORN_NAMESPACE # noqa E402 class common_keywords: @@ -28,9 +29,9 @@ def init_k8s_api_client(self): """Initialize Kubernetes API client""" return init_k8s_api_client() - def generate_name_with_suffix(self, kind, suffix): + def generate_name_with_suffix(self, kind, suffix, precise=False): """Generate unique name with timestamp""" - return generate_name_with_suffix(kind, suffix) + return generate_name_with_suffix(kind, suffix, precise) def cleanup_vms(self): """Cleanup VMs""" @@ -54,17 +55,33 @@ def cleanup_storageclasses(self): def cleanup_networks(self): """Cleanup networks""" - logging('Cleanup networks requested') + from network import Network + net = Network() + net.cleanup_vlan_networks() + net.cleanup_vlan_configs() + net.cleanup_cluster_networks() def cleanup_backups(self): """Cleanup backups""" logging('Cleanup backups requested') def list_pods_by_label(self, namespace, label_selector, status=None): - """List pods by label""" + """List pods by label + Returns: list of V1Pod objects + """ pods = get_pods_by_label(namespace, label_selector) if status: pods = [pod for pod in pods if pod.status.phase == status] logging(f"Found {len(pods)} pods by label {label_selector} in namespace {namespace}: \ {[pod.metadata.name for pod in pods]}") return pods + + def list_running_im_pods(self): + """List running instance-manager pods + Returns: list of V1Pod objects + """ + return self.list_pods_by_label( + namespace=LONGHORN_NAMESPACE, + label_selector='longhorn.io/data-engine=v1,longhorn.io/component=instance-manager', + status='Running' + ) diff --git a/harvester_robot_tests/libs/keywords/storage_network_keywords.py b/harvester_robot_tests/libs/keywords/storage_network_keywords.py index f13293928..e3427b794 100644 --- a/harvester_robot_tests/libs/keywords/storage_network_keywords.py +++ b/harvester_robot_tests/libs/keywords/storage_network_keywords.py @@ -2,21 +2,30 @@ Storage Network Keywords - creates StorageNetwork() instance and delegates Layer 3: Keyword wrappers for Robot Framework """ +import json import os import sys +import time +from ipaddress import ip_address, ip_network sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))) # noqa E402 from utility.utility import logging # noqa E402 from storage_network import StorageNetwork # noqa E402 -from constant import DEFAULT_TIMEOUT # noqa E402 +from common_keywords import common_keywords # noqa E402 +from host_keywords import host_keywords # noqa E402 +from constant import DEFAULT_RETRY_INTERVAL, DEFAULT_TIMEOUT # noqa E402 + +NETS_ANNO = "k8s.v1.cni.cncf.io/networks" +NET_STATUS_ANNO = "k8s.v1.cni.cncf.io/network-status" class storage_network_keywords: """Storage Network keyword wrapper - creates StorageNetwork component and delegates""" - def __init__(self): """Initialize storage network keywords with lazy loading""" self._storage_network = None + self._common = None + self._host = None @property def storage_network(self): @@ -25,6 +34,18 @@ def storage_network(self): self._storage_network = StorageNetwork() return self._storage_network + @property + def common(self): + if self._common is None: + self._common = common_keywords() + return self._common + + @property + def host(self): + if self._host is None: + self._host = host_keywords() + return self._host + def enable_storage_network(self, vlan_id, cluster_network, ip_range, share_rwx=False): """ @@ -61,20 +82,149 @@ def get_storage_network_status(self): logging("Getting storage-network status") return self.storage_network.get_storage_network_status() - def wait_for_storage_network_ready(self, timeout=DEFAULT_TIMEOUT): + def _is_storage_network_configured(self, snet_data): + """ Check storage-network is configured + snet_data: return value of get_storage_network_status() """ - Wait for storage-network setting to be applied and completed. + conditions = snet_data.get("status", {}).get("conditions", []) + if not conditions: + return False - Args: - timeout: Timeout in seconds + last = conditions[-1] + return last.get("reason") == "Completed" and last.get("status") == "True" - Returns: - dict: Storage network data when ready + def is_storage_network_enabled_in_config(self): + """ Check storage-network is enabled from config perspective """ - logging("Waiting for storage-network to be ready") - return self.storage_network.wait_for_storage_network_ready( - int(timeout) - ) + snet_data = self.get_storage_network_status() + return self._is_storage_network_configured(snet_data) and snet_data.get("value") + + def wait_storage_network_enabled_in_config(self, timeout=DEFAULT_TIMEOUT): + end_time = time.time() + int(timeout) + while time.time() < end_time: + if self.is_storage_network_enabled_in_config(): + return + time.sleep(DEFAULT_RETRY_INTERVAL) + + raise Exception("Timeout waiting for storage-network to be enabled on harvester setting") + + def is_storage_network_disabled_in_config(self): + """ Check storage-network is disabled from config perspective + """ + snet_data = self.get_storage_network_status() + return self._is_storage_network_configured(snet_data) and not snet_data.get("value") + + def wait_storage_network_disabled_in_config(self, timeout=DEFAULT_TIMEOUT): + end_time = time.time() + int(timeout) + while time.time() < end_time: + if self.is_storage_network_disabled_in_config(): + return + time.sleep(DEFAULT_RETRY_INTERVAL) + + raise Exception("Timeout waiting for storage-network to be disabled on harvester setting") + + def is_storage_network_enabled_on_pods(self, snet_cidr): + """ Check storage-network is enabled from service perspective + """ + standard_nodes = self.host.get_standard_nodes() + imgrs = self.common.list_running_im_pods() + + # Check IM pods are Running + if not len(imgrs) == len(standard_nodes): + logging( + "IM pods counts do not match standard nodes: " + f"\nimgrs: {[imgr.metadata.name for imgr in imgrs]}" + f"\nstandard_nodes: {standard_nodes}" + ) + return False + + # Check IM pods have proper annotations with correct CIDR + for imgr in imgrs: + metadata = getattr(imgr, 'metadata', None) + annotations = getattr(metadata, 'annotations', {}) or {} + name = getattr(metadata, 'name', '') + for target_anno in [NETS_ANNO, NET_STATUS_ANNO]: + if target_anno not in annotations: + logging(f"Pod {name} has no annotation {target_anno}") + return False + + # Check k8s.v1.cni.cncf.io/networks + target_anno = NETS_ANNO + try: + nets = json.loads(annotations[target_anno]) + snet = next(n for n in nets if 'lhnet1' == n.get('interface')) + except StopIteration: + logging(f"Annotation {target_anno} has no interface 'lhnet1' in pod {name}") + return False + + # Check k8s.v1.cni.cncf.io/network-status + target_anno = NET_STATUS_ANNO + try: + net_statuses = json.loads(annotations[target_anno]) + snet_status = next(s for s in net_statuses if 'lhnet1' == s.get('interface')) + except StopIteration: + logging(f"Annotation {target_anno} has no interface 'lhnet1' in pod {name}") + return False + + snet_ips = snet_status.get('ips', ['::1']) + if not all(ip_address(sip) in ip_network(snet_cidr) for sip in snet_ips): + logging(f"Dedicated IPs {snet_ips} does NOT fit {snet_cidr} in pod {name}") + return False + + # Check network name identical in both annotations + if f"{snet.get('namespace')}/{snet.get('name')}" != snet_status.get('name'): + logging( + f"Network name is not identical between annotations " + f"k8s.v1.cni.cncf.io/networks and k8s.v1.cni.cncf.io/network-status " + f"in pod {name}" + ) + return False + + return True + + def wait_storage_network_enabled_on_pods(self, snet_cidr, timeout=DEFAULT_TIMEOUT): + end_time = time.time() + int(timeout) + while time.time() < end_time: + if self.is_storage_network_enabled_on_pods(snet_cidr): + return + time.sleep(DEFAULT_RETRY_INTERVAL) + + raise Exception("Timeout waiting for storage-network to be enabled on longhorn pods") + + def is_storage_network_disabled_on_pods(self): + """ Check storage-network is disabled from service perspective + """ + standard_nodes = self.host.get_standard_nodes() + imgrs = self.common.list_running_im_pods() + + # Check IM pods are Running + if not len(imgrs) == len(standard_nodes): + logging( + "IM pods counts do not match standard nodes: " + f"\nimgrs: {[imgr.metadata.name for imgr in imgrs]}" + f"\nstandard_nodes: {standard_nodes}" + ) + return False + + # IM pods should not have 'k8s.v1.cni.cncf.io/networks' annotation + for imgr in imgrs: + metadata = getattr(imgr, 'metadata', None) + annotations = getattr(metadata, 'annotations', {}) or {} + name = getattr(metadata, 'name', '') + if NETS_ANNO in annotations: + logging(f"Pod {name} should not have annotation {NETS_ANNO}") + return False + + return True + + def wait_storage_network_disabled_on_pods(self, timeout=DEFAULT_TIMEOUT): + end_time = time.time() + int(timeout) + while time.time() < end_time: + if self.is_storage_network_disabled_on_pods(): + return + time.sleep(DEFAULT_RETRY_INTERVAL) + + raise Exception("Timeout waiting for storage-network to be disabled on longhorn pods") def get_vlan_network_cidr(self, vlan_id, cluster_network): """ diff --git a/harvester_robot_tests/libs/network/base.py b/harvester_robot_tests/libs/network/base.py index 7daf43306..a2d1e9d17 100644 --- a/harvester_robot_tests/libs/network/base.py +++ b/harvester_robot_tests/libs/network/base.py @@ -30,6 +30,10 @@ def delete_cluster_network(self, name): """ pass + @abstractmethod + def cleanup_cluster_networks(self): + pass + @abstractmethod def create_vlan_config(self, name, cluster_network, nic): """Create VLAN config to bind NIC to cluster network. @@ -53,6 +57,10 @@ def delete_vlan_config(self, name): """ pass + @abstractmethod + def cleanup_vlan_configs(self): + pass + @abstractmethod def wait_for_cluster_network_ready(self, name, timeout): """Wait for cluster network to become ready. @@ -90,6 +98,10 @@ def delete_vlan_network(self, name): """ pass + @abstractmethod + def cleanup_vlan_networks(self): + pass + # IP Pool Operations @abstractmethod def get_ip_pool(self, name): diff --git a/harvester_robot_tests/libs/network/crd.py b/harvester_robot_tests/libs/network/crd.py index 00b936c95..a3888028b 100644 --- a/harvester_robot_tests/libs/network/crd.py +++ b/harvester_robot_tests/libs/network/crd.py @@ -3,11 +3,17 @@ Layer 4: Makes actual kubectl/K8s API calls for network operations """ -import time import json -from kubernetes import client from kubernetes.client.rest import ApiException -from constant import DEFAULT_NAMESPACE +from crd import ( + create_cr, create_cluster_cr, + delete_cr, delete_cluster_cr, + list_cr, list_cluster_cr, + wait_for_cr_deleted, wait_for_cluster_cr_deleted, + get_cluster_cr, + wait_for_cluster_cr_condition, +) +from constant import DEFAULT_NAMESPACE, LABEL_TEST, LABEL_TEST_VALUE, DEFAULT_TIMEOUT_SHORT from utility.utility import logging from network.base import Base @@ -17,10 +23,6 @@ class CRD(Base): CRD implementation for Network operations using Kubernetes API """ - def __init__(self): - """Initialize Kubernetes clients""" - self.custom_api = client.CustomObjectsApi() - # Cluster Network Operations def create_cluster_network(self, name): """Create cluster network""" @@ -30,12 +32,15 @@ def create_cluster_network(self, name): "apiVersion": "network.harvesterhci.io/v1beta1", "kind": "ClusterNetwork", "metadata": { - "name": name + "name": name, + "labels": { + LABEL_TEST: LABEL_TEST_VALUE + }, } } try: - self.custom_api.create_cluster_custom_object( + create_cluster_cr( group="network.harvesterhci.io", version="v1beta1", plural="clusternetworks", @@ -51,16 +56,24 @@ def create_cluster_network(self, name): f"Failed to create cluster network: {e}" ) - def delete_cluster_network(self, name): + def delete_cluster_network(self, name, wait=False): """Delete cluster network""" logging(f"Deleting cluster network: {name}") try: - self.custom_api.delete_cluster_custom_object( + delete_cluster_cr( group="network.harvesterhci.io", version="v1beta1", plural="clusternetworks", name=name ) + if wait: + wait_for_cluster_cr_deleted( + group="network.harvesterhci.io", + version="v1beta1", + plural="clusternetworks", + name=name, + timeout=DEFAULT_TIMEOUT_SHORT + ) logging(f"Deleted cluster network: {name}") except ApiException as e: if e.status != 404: @@ -68,6 +81,28 @@ def delete_cluster_network(self, name): f"Failed to delete cluster network: {e}" ) + def list_cluster_networks(self, label_selector=None): + return list_cluster_cr( + group="network.harvesterhci.io", + version="v1beta1", + plural="clusternetworks", + label_selector=label_selector + ).get("items", []) + + def cleanup_cluster_networks(self): + logging('Cleaning up test cluster networks') + try: + cnets = self.list_cluster_networks(label_selector=f"{LABEL_TEST}={LABEL_TEST_VALUE}") + for cnet in cnets: + name = cnet['metadata']['name'] + try: + logging(f'Deleting test cluster networks: {name}') + self.delete_cluster_network(name, wait=True) + except Exception as e: + logging(f'Error deleting cluster networks {name}: {e}', 'WARNING') + except Exception as e: + logging(f'Error during cluster networks cleanup: {e}', 'WARNING') + def create_vlan_config(self, name, cluster_network, nic): """Create VLAN config to bind NIC to cluster network""" logging(f"Creating VLAN config: {name} " @@ -79,8 +114,8 @@ def create_vlan_config(self, name, cluster_network, nic): "metadata": { "name": name, "labels": { - "network.harvesterhci.io/clusternetwork": - cluster_network + "network.harvesterhci.io/clusternetwork": cluster_network, + LABEL_TEST: LABEL_TEST_VALUE } }, "spec": { @@ -99,7 +134,7 @@ def create_vlan_config(self, name, cluster_network, nic): } try: - self.custom_api.create_cluster_custom_object( + create_cluster_cr( group="network.harvesterhci.io", version="v1beta1", plural="vlanconfigs", @@ -115,16 +150,24 @@ def create_vlan_config(self, name, cluster_network, nic): f"Failed to create VLAN config: {e}" ) - def delete_vlan_config(self, name): + def delete_vlan_config(self, name, wait=False): """Delete VLAN config""" logging(f"Deleting VLAN config: {name}") try: - self.custom_api.delete_cluster_custom_object( + delete_cluster_cr( group="network.harvesterhci.io", version="v1beta1", plural="vlanconfigs", name=name ) + if wait: + wait_for_cluster_cr_deleted( + group="network.harvesterhci.io", + version="v1beta1", + plural="vlanconfigs", + name=name, + timeout=DEFAULT_TIMEOUT_SHORT + ) logging(f"Deleted VLAN config: {name}") except ApiException as e: if e.status != 404: @@ -132,35 +175,48 @@ def delete_vlan_config(self, name): f"Failed to delete VLAN config: {e}" ) + def list_vlan_configs(self, label_selector=None): + return list_cluster_cr( + group="network.harvesterhci.io", + version="v1beta1", + plural="vlanconfigs", + label_selector=label_selector + ).get("items", []) + + def cleanup_vlan_configs(self): + logging('Cleaning up test vlan configs') + try: + vlan_configs = self.list_vlan_configs( + label_selector=f"{LABEL_TEST}={LABEL_TEST_VALUE}" + ) + for vlan_config in vlan_configs: + name = vlan_config['metadata']['name'] + try: + logging(f'Deleting test vlan configs: {name}') + self.delete_vlan_config(name, wait=True) + except Exception as e: + logging(f'Error deleting vlan configs {name}: {e}', 'WARNING') + except Exception as e: + logging(f'Error during vlan configs cleanup: {e}', 'WARNING') + def wait_for_cluster_network_ready(self, name, timeout=120): """Wait for cluster network to become ready""" logging(f"Waiting for cluster network {name} to be ready") - deadline = time.time() + int(timeout) - - while time.time() < deadline: - try: - cn = self.custom_api.get_cluster_custom_object( - group="network.harvesterhci.io", - version="v1beta1", - plural="clusternetworks", - name=name - ) - conditions = cn.get("status", {}).get( - "conditions", [] - ) - for cond in conditions: - if (cond.get("type") == "ready" - and cond.get("status") == "True"): - logging( - f"Cluster network {name} is ready" - ) - return cn - except ApiException: - pass - time.sleep(5) - - raise Exception( - f"Cluster network {name} not ready within {timeout}s" + wait_for_cluster_cr_condition( + group="network.harvesterhci.io", + version="v1beta1", + plural="clusternetworks", + name=name, + condition_type="ready", + condition_status="True", + timeout=timeout + ) + logging(f"Cluster network {name} is ready") + return get_cluster_cr( + group="network.harvesterhci.io", + version="v1beta1", + plural="clusternetworks", + name=name ) # VLAN Network Operations @@ -179,9 +235,9 @@ def create_vlan_network(self, name, vlan_id, cluster_network): '{"mode":"auto"}' }, "labels": { - "network.harvesterhci.io/clusternetwork": - cluster_network, - "network.harvesterhci.io/type": "L2VlanNetwork" + "network.harvesterhci.io/clusternetwork": cluster_network, + "network.harvesterhci.io/type": "L2VlanNetwork", + LABEL_TEST: LABEL_TEST_VALUE } }, "spec": { @@ -198,7 +254,7 @@ def create_vlan_network(self, name, vlan_id, cluster_network): } try: - self.custom_api.create_namespaced_custom_object( + create_cr( group="k8s.cni.cncf.io", version="v1", namespace=DEFAULT_NAMESPACE, @@ -215,17 +271,26 @@ def create_vlan_network(self, name, vlan_id, cluster_network): f"Failed to create VLAN network: {e}" ) - def delete_vlan_network(self, name): + def delete_vlan_network(self, name, wait=False): """Delete VLAN network""" logging(f"Deleting VLAN network: {name}") try: - self.custom_api.delete_namespaced_custom_object( + delete_cr( group="k8s.cni.cncf.io", version="v1", namespace=DEFAULT_NAMESPACE, plural="network-attachment-definitions", name=name ) + if wait: + wait_for_cr_deleted( + group="k8s.cni.cncf.io", + version="v1", + namespace=DEFAULT_NAMESPACE, + plural="network-attachment-definitions", + name=name, + timeout=DEFAULT_TIMEOUT_SHORT + ) logging(f"Deleted VLAN network: {name}") except ApiException as e: if e.status != 404: @@ -233,13 +298,36 @@ def delete_vlan_network(self, name): f"Failed to delete VLAN network: {e}" ) + def list_vlan_networks(self, label_selector=None): + return list_cr( + group="k8s.cni.cncf.io", + version="v1", + namespace=DEFAULT_NAMESPACE, + plural="network-attachment-definitions", + label_selector=label_selector + ).get("items", []) + + def cleanup_vlan_networks(self): + logging('Cleaning up test vlan networks') + try: + vnets = self.list_vlan_networks(label_selector=f"{LABEL_TEST}={LABEL_TEST_VALUE}") + for vnet in vnets: + name = vnet['metadata']['name'] + try: + logging(f'Deleting test vlan network: {name}') + self.delete_vlan_network(name, wait=True) + except Exception as e: + logging(f'Error deleting vlan network {name}: {e}', 'WARNING') + except Exception as e: + logging(f'Error during vlan network cleanup: {e}', 'WARNING') + # IP Pool Operations def get_ip_pool(self, name): """Get IP pool by name, returns None if not found""" lb_group = "loadbalancer.harvesterhci.io" lb_version = "v1beta1" try: - return self.custom_api.get_cluster_custom_object( + return get_cluster_cr( group=lb_group, version=lb_version, plural="ippools", @@ -290,7 +378,7 @@ def create_ip_pool(self, name, subnet, start_ip, end_ip, network_id, } try: - self.custom_api.create_cluster_custom_object( + create_cluster_cr( group=lb_group, version=lb_version, plural="ippools", @@ -314,7 +402,7 @@ def delete_ip_pool(self, name): lb_version = "v1beta1" try: - self.custom_api.delete_cluster_custom_object( + delete_cluster_cr( group=lb_group, version=lb_version, plural="ippools", diff --git a/harvester_robot_tests/libs/network/network.py b/harvester_robot_tests/libs/network/network.py index aa415c5f1..fa03134f7 100644 --- a/harvester_robot_tests/libs/network/network.py +++ b/harvester_robot_tests/libs/network/network.py @@ -46,6 +46,9 @@ def delete_cluster_network(self, name): """Delete cluster network""" return self.network.delete_cluster_network(name) + def cleanup_cluster_networks(self): + return self.network.cleanup_cluster_networks() + def create_vlan_config(self, name, cluster_network, nic): """Create VLAN config to bind NIC to cluster network""" return self.network.create_vlan_config( @@ -56,6 +59,9 @@ def delete_vlan_config(self, name): """Delete VLAN config""" return self.network.delete_vlan_config(name) + def cleanup_vlan_configs(self): + return self.network.cleanup_vlan_configs() + def wait_for_cluster_network_ready(self, name, timeout=120): """Wait for cluster network to become ready""" return self.network.wait_for_cluster_network_ready(name, timeout) @@ -71,6 +77,9 @@ def delete_vlan_network(self, name): """Delete VLAN network""" return self.network.delete_vlan_network(name) + def cleanup_vlan_networks(self): + return self.network.cleanup_vlan_networks() + # IP Pool Operations def get_ip_pool(self, name): """Get IP pool by name""" diff --git a/harvester_robot_tests/libs/network/rest.py b/harvester_robot_tests/libs/network/rest.py index 71cf94420..cb14fec7d 100644 --- a/harvester_robot_tests/libs/network/rest.py +++ b/harvester_robot_tests/libs/network/rest.py @@ -59,6 +59,9 @@ def delete_cluster_network(self, name): logging(f"Deleted cluster network: {name}") + def cleanup_cluster_networks(self): + pass + def create_vlan_config(self, name, cluster_network, nic): """Create VLAN config to bind NIC to cluster network""" logging(f"Creating VLAN config: {name} " @@ -116,6 +119,9 @@ def delete_vlan_config(self, name): logging(f"Deleted VLAN config: {name}") + def cleanup_vlan_configs(self): + pass + def wait_for_cluster_network_ready(self, name, timeout=120): """Wait for cluster network to become ready""" logging(f"Waiting for cluster network {name} to be ready") @@ -209,6 +215,9 @@ def delete_vlan_network(self, name): logging(f"Deleted VLAN network: {name}") + def cleanup_vlan_networks(self): + pass + # IP Pool Operations def get_ip_pool(self, name): """Get IP pool by name, returns None if not found""" diff --git a/harvester_robot_tests/libs/storage_network/base.py b/harvester_robot_tests/libs/storage_network/base.py index 5a8cbabeb..8a531ac70 100644 --- a/harvester_robot_tests/libs/storage_network/base.py +++ b/harvester_robot_tests/libs/storage_network/base.py @@ -37,15 +37,6 @@ def get_storage_network_status(self): """ pass - @abstractmethod - def wait_for_storage_network_ready(self, timeout): - """Wait for storage-network setting to be applied and completed. - - Args: - timeout: Timeout in seconds - """ - pass - @abstractmethod def get_vlan_network_cidr(self, vlan_id, cluster_network): """Get the CIDR for a VLAN network by creating a temporary network. diff --git a/harvester_robot_tests/libs/storage_network/crd.py b/harvester_robot_tests/libs/storage_network/crd.py index c3cd8aec2..3f87847a9 100644 --- a/harvester_robot_tests/libs/storage_network/crd.py +++ b/harvester_robot_tests/libs/storage_network/crd.py @@ -8,7 +8,7 @@ import json from kubernetes import client from kubernetes.client.rest import ApiException -from constant import DEFAULT_TIMEOUT, DEFAULT_NAMESPACE +from constant import DEFAULT_NAMESPACE from utility.utility import logging from storage_network.base import Base @@ -109,36 +109,6 @@ def get_storage_network_status(self): return json.loads(stdout) - def wait_for_storage_network_ready(self, timeout=DEFAULT_TIMEOUT): - """Wait for storage-network to be applied and completed""" - logging("Waiting for storage-network to be ready") - - end_time = time.time() + int(timeout) - while time.time() < end_time: - try: - data = self.get_storage_network_status() - conditions = (data.get("status", {}) - .get("conditions", [])) - if conditions: - last = conditions[-1] - if (last.get("status") == "True" - and last.get("reason") == "Completed" - and data.get("value")): - logging("Storage network is ready (Completed)") - return data - logging(f"Storage network status: " - f"reason={last.get('reason')}, " - f"status={last.get('status')}") - except Exception as e: - logging(f"Error checking storage-network: {e}", - level="WARNING") - - time.sleep(10) - - raise Exception( - "Timeout waiting for storage-network to be ready" - ) - def get_vlan_network_cidr(self, vlan_id, cluster_network): """Get CIDR by creating a temp VLAN network and reading its route""" logging(f"Getting VLAN network CIDR for vlan={vlan_id}") diff --git a/harvester_robot_tests/libs/storage_network/rest.py b/harvester_robot_tests/libs/storage_network/rest.py index a2d703670..03bbe4020 100644 --- a/harvester_robot_tests/libs/storage_network/rest.py +++ b/harvester_robot_tests/libs/storage_network/rest.py @@ -6,7 +6,7 @@ import time import json from utility.utility import logging, get_harvester_api_client -from constant import DEFAULT_TIMEOUT, DEFAULT_NAMESPACE +from constant import DEFAULT_NAMESPACE from storage_network.base import Base @@ -98,36 +98,6 @@ def get_storage_network_status(self): return data - def wait_for_storage_network_ready(self, timeout=DEFAULT_TIMEOUT): - """Wait for storage-network to be applied and completed""" - logging("Waiting for storage-network to be ready") - - end_time = time.time() + int(timeout) - while time.time() < end_time: - try: - data = self.get_storage_network_status() - conditions = (data.get("status", {}) - .get("conditions", [])) - if conditions: - last = conditions[-1] - if (last.get("status") == "True" - and last.get("reason") == "Completed" - and data.get("value")): - logging("Storage network is ready (Completed)") - return data - logging(f"Storage network status: " - f"reason={last.get('reason')}, " - f"status={last.get('status')}") - except Exception as e: - logging(f"Error checking storage-network: {e}", - level="WARNING") - - time.sleep(10) - - raise Exception( - "Timeout waiting for storage-network to be ready" - ) - def get_vlan_network_cidr(self, vlan_id, cluster_network): """Get CIDR by creating a temp VLAN network and reading its route""" logging(f"Getting VLAN network CIDR for vlan={vlan_id}") diff --git a/harvester_robot_tests/libs/storage_network/storage_network.py b/harvester_robot_tests/libs/storage_network/storage_network.py index a23d3455d..bf0d2f311 100644 --- a/harvester_robot_tests/libs/storage_network/storage_network.py +++ b/harvester_robot_tests/libs/storage_network/storage_network.py @@ -52,10 +52,6 @@ def get_storage_network_status(self): """Get the current storage-network setting status""" return self.storage_network.get_storage_network_status() - def wait_for_storage_network_ready(self, timeout): - """Wait for storage-network to be applied and completed""" - return self.storage_network.wait_for_storage_network_ready(timeout) - def get_vlan_network_cidr(self, vlan_id, cluster_network): """Get CIDR for a VLAN network""" return self.storage_network.get_vlan_network_cidr( diff --git a/harvester_robot_tests/libs/utility/utility.py b/harvester_robot_tests/libs/utility/utility.py index 79a13a18d..0e6115520 100644 --- a/harvester_robot_tests/libs/utility/utility.py +++ b/harvester_robot_tests/libs/utility/utility.py @@ -56,11 +56,15 @@ def get_timestamp(): return datetime.now().strftime("%Y%m%d%H%M%S") -def generate_name_with_suffix(kind, suffix): - """Generate unique name with timestamp suffix""" - timestamp = datetime.now().strftime("%y%m%d-%H%M%S-%f")[:-2] - name = "-".join([s for s in [kind, suffix, timestamp] if s]) - return name+"qa" # Add 'qa' suffix to all generated names for easier identification +def generate_name_with_suffix(kind, suffix, precise=False): + """Generate unique name with timestamp suffix + e.g. + * %m%d%H%M -> 06181015 (general, covers resources with limited length like cluster network) + * %m%d%H%M%S%f -> 0618101519681156 (precise, for highly concurrent tests to avoid collisions) + """ + format_str = "%m%d%H%M%S%f" if precise else "%m%d%H%M" + timestamp = datetime.now().strftime(format_str) + return "-".join([s for s in [kind, suffix, timestamp] if s]) def get_retry_count_and_interval(): diff --git a/harvester_robot_tests/tests/regression/test_0_storage_network.robot b/harvester_robot_tests/tests/regression/test_0_storage_network.robot new file mode 100644 index 000000000..b81e7d1e2 --- /dev/null +++ b/harvester_robot_tests/tests/regression/test_0_storage_network.robot @@ -0,0 +1,43 @@ +*** Settings *** +Documentation Storage Network Test Cases +Test Tags storage network setting + +Resource ../../keywords/common.resource +Resource ../../keywords/network.resource +Resource ../../keywords/storage_network.resource + +Suite Setup Local Suite Setup +Suite Teardown Common Suite Teardown +Test Teardown Common Test Teardown + + +*** Variables *** +${CNET_NAME} ${EMPTY} +${VNET_CIDR} ${EMPTY} + + +*** Test Cases *** +Test Enable Storage Network + [Tags] p0 smoke + Given Create Cluster Network ${CNET_NAME} + And Create VLAN Config ${CNET_NAME} ${CNET_NAME} ${VLAN_NIC} + And Get VLAN Network CIDR + When storage_network.Enable ${VLAN_ID} ${CNET_NAME} ${VNET_CIDR} + Then storage_network.Wait For Enabled ${VNET_CIDR} + +Test Disable Storage Network + [Tags] p0 smoke + Given storage_network.Is Enabled ${VNET_CIDR} + When storage_network.Disable + Then storage_network.Wait For Disabled + + +*** Keywords *** +Local Suite Setup + Set Up Test Environment + ${name}= Generate Unique Name qa + Set Suite Variable ${CNET_NAME} ${name} + +Get VLAN Network CIDR + ${cidr}= storage_network.Get VLAN Network CIDR ${VLAN_ID} ${CNET_NAME} + Set Suite Variable ${VNET_CIDR} ${cidr} \ No newline at end of file