From 88be466b3f0cef8ac812f44d23181a1fb421af6f Mon Sep 17 00:00:00 2001 From: P6g9YHK6 <17877371+P6g9YHK6@users.noreply.github.com> Date: Thu, 18 Jun 2026 08:12:20 +0200 Subject: [PATCH 1/8] Update file: Backup Veeam SPC.py --- scripts_staging/Checks/Backup Veeam SPC.py | 90 ++++++++++++++++------ 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/scripts_staging/Checks/Backup Veeam SPC.py b/scripts_staging/Checks/Backup Veeam SPC.py index 84481991..7abfe06f 100644 --- a/scripts_staging/Checks/Backup Veeam SPC.py +++ b/scripts_staging/Checks/Backup Veeam SPC.py @@ -34,6 +34,7 @@ 27.03.25 SAN added more debug 15.04.25 SAN big code cleanup + publication + 17.06.26 SAN server-side filter + pagination + timeouts + api key test TODO: better flow for the "force" @@ -78,11 +79,11 @@ def get_timestamp(date_str): """Converts a date string to a Unix timestamp.""" return time.mktime(datetime.strptime(date_str[:26], "%Y-%m-%dT%H:%M:%S.%f").timetuple()) -def api_call_with_retries(url, method='GET', data=None, headers=None, retries=5, wait=60): +def api_call_with_retries(url, method='GET', data=None, headers=None, retries=5, wait=60, timeout=60): """Makes an API call with retries for handling HTTP 429 responses.""" for attempt in range(retries): try: - res = requests.request(method, url, data=data, headers=headers) + res = requests.request(method, url, data=data, headers=headers, timeout=timeout) if res.status_code == 429 and attempt < retries - 1: print(f"HTTP 429 received. Retrying in {wait} seconds... (Attempt {attempt + 1}/{retries})") time.sleep(wait) @@ -95,6 +96,30 @@ def api_call_with_retries(url, method='GET', data=None, headers=None, retries=5, sys.exit(1) time.sleep(wait) +def api_get_all_paginated(base_url, headers, select_param=None, **kwargs): + """Fetches all pages from a paginated endpoint using offset-based pagination.""" + limit = 500 + offset = 0 + all_data = [] + + while True: + url = f"{base_url}&limit={limit}&offset={offset}" + if select_param: + url += f"&select={select_param}" + res = api_call_with_retries(url, method='GET', headers=headers, **kwargs) + resp = res.json() + data = resp.get("data", []) + all_data.extend(data) + + paging = resp.get("meta", {}).get("pagingInfo", {}) + total = paging.get("total", offset + len(data)) + if offset + limit >= total: + break + offset += limit + + return all_data + + def get_auth_headers(): """Returns the headers required for authenticated API requests.""" return { @@ -104,25 +129,38 @@ def get_auth_headers(): "accept": "application/json", } -def apiGet_BackedUpVMs(): - """Retrieves a list of backed-up virtual machines.""" - url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines?limit=500&select=[{{'propertyPath':'name'}},{{'propertyPath':'instanceUid'}},{{'propertyPath':'backupServerUid'}}]" - return api_call_with_retries(url, method='GET', headers=get_auth_headers()) +def apiGet_BackedUpVMs(name_filter=None): + """Retrieves backed-up VMs, with optional server-side name filter and pagination.""" + headers = get_auth_headers() + select = '[{"propertyPath":"name"},{"propertyPath":"instanceUid"},{"propertyPath":"backupServerUid"}]' + + if name_filter: + url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines" + filter_json = json.dumps([{"property": "name", "operation": "contains", "collation": "ignorecase", "value": name_filter}]) + url += f"?filter={filter_json}&select={select}" + r = api_call_with_retries(url, method='GET', headers=headers) + return r.json()["data"] + + base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines?limit=500" + return api_get_all_paginated(base_url, headers, select_param=select) def apiGet_VMbackups(vmUID): - """Retrieves the list of backups for a specific VM.""" - url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines/{vmUID}/backups?limit=500" - return api_call_with_retries(url, method='GET', headers=get_auth_headers()) + """Retrieves all backups for a specific VM with pagination.""" + headers = get_auth_headers() + base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines/{vmUID}/backups?limit=500" + return api_get_all_paginated(base_url, headers) def apiGet_BackedUpComputers(): - """Retrieves a list of computers that are backed up.""" - url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/computersManagedByBackupServer?limit=500" - return api_call_with_retries(url, method='GET', headers=get_auth_headers()) + """Retrieves all backed-up computers with pagination.""" + headers = get_auth_headers() + base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/computersManagedByBackupServer?limit=500" + return api_get_all_paginated(base_url, headers) def apiGet_ComputersRestorePoints(): - """Retrieves restore points for computers managed by the backup server.""" - url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/computersManagedByBackupServer/restorePoints?limit=500" - return api_call_with_retries(url, method='GET', headers=get_auth_headers()) + """Retrieves all computer restore points with pagination.""" + headers = get_auth_headers() + base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/computersManagedByBackupServer/restorePoints?limit=500" + return api_get_all_paginated(base_url, headers) # === Environment and Constants === vmUID, lastRPoint, vmBkp_bkpSrvUID, tmpName, strSchedule = ("",) * 5 @@ -149,6 +187,7 @@ def apiGet_ComputersRestorePoints(): print("Backup check is disabled because 'FORCE' contains 'DISABLEDBACKUPCHECK'.") sys.exit(0) + def main(): try: log_debug("Parsed Environment Variables:") @@ -161,19 +200,22 @@ def main(): log_debug(f" APIURL: {env_vars['APIURL']}") log_debug(f" THRESHOLD_HOURS: {env_vars['THRESHOLD_HOURS']}\n") - log_debug("INFO: Fetching the list of all backed-up VMs...") - response = apiGet_BackedUpVMs().json() - backed_up_vms = response["data"] - - for vm in backed_up_vms: - log_debug(f"VM Name: {vm['name']}") - host_arg = ( env_vars['FORCE'] if env_vars.get('FORCE') and "Manual" not in env_vars['FORCE'] else env_vars['HOST'] ) + log_debug(f"INFO: Fetching VMs filtered by name '{host_arg}'...") + backed_up_vms = apiGet_BackedUpVMs(name_filter=host_arg) + + if not backed_up_vms: + log_debug("Server-side filter returned no results. Falling back to full paginated fetch...") + backed_up_vms = apiGet_BackedUpVMs() + + for vm in backed_up_vms: + log_debug(f"VM Name: {vm['name']}") + if env_vars.get('FORCE'): matching_vms = [vm for vm in backed_up_vms if host_arg == vm["name"]] else: @@ -199,7 +241,7 @@ def main(): log_debug(f"INFO: Selected VM: {tmpName} (UID: {vmUID})") try: - restore_points_response = ( + restore_points = ( apiGet_ComputersRestorePoints() if isComputer else apiGet_VMbackups(vmUID) ) except requests.exceptions.RequestException as e: @@ -207,8 +249,6 @@ def main(): log_debug(str(e)) sys.exit(2) - restore_points = restore_points_response.json()["data"] - latest_restore_point = next( (p['creationTimeUtc'] for p in restore_points if 'creationTimeUtc' in p), next((p['latestRestorePointDate'] for p in restore_points if 'latestRestorePointDate' in p), None) From 71d620167a9088151fb058f714356a564c704a6c Mon Sep 17 00:00:00 2001 From: P6g9YHK6 <17877371+P6g9YHK6@users.noreply.github.com> Date: Thu, 18 Jun 2026 09:32:17 +0200 Subject: [PATCH 2/8] Update file: Backup Veeam SPC.py --- scripts_staging/Checks/Backup Veeam SPC.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts_staging/Checks/Backup Veeam SPC.py b/scripts_staging/Checks/Backup Veeam SPC.py index 7abfe06f..19874666 100644 --- a/scripts_staging/Checks/Backup Veeam SPC.py +++ b/scripts_staging/Checks/Backup Veeam SPC.py @@ -34,7 +34,8 @@ 27.03.25 SAN added more debug 15.04.25 SAN big code cleanup + publication - 17.06.26 SAN server-side filter + pagination + timeouts + api key test + 17.06.26 SAN server-side filter + pagination + timeouts + 18.06.26 SAN removed auth check TODO: better flow for the "force" From 5eb862be36de22ab41e74dff06fc7314db384bf6 Mon Sep 17 00:00:00 2001 From: P6g9YHK6 <17877371+P6g9YHK6@users.noreply.github.com> Date: Thu, 18 Jun 2026 09:58:07 +0200 Subject: [PATCH 3/8] Update file: Backup Veeam SPC.py --- scripts_staging/Checks/Backup Veeam SPC.py | 201 ++++++++++----------- 1 file changed, 93 insertions(+), 108 deletions(-) diff --git a/scripts_staging/Checks/Backup Veeam SPC.py b/scripts_staging/Checks/Backup Veeam SPC.py index 19874666..0e21319d 100644 --- a/scripts_staging/Checks/Backup Veeam SPC.py +++ b/scripts_staging/Checks/Backup Veeam SPC.py @@ -34,20 +34,13 @@ 27.03.25 SAN added more debug 15.04.25 SAN big code cleanup + publication - 17.06.26 SAN server-side filter + pagination + timeouts - 18.06.26 SAN removed auth check - -TODO: - better flow for the "force" - set fallback to get localhostname if hosts is not specified - avoid redundant calls to os.getenv - more function decomposition - graceful handling of missing keys in json responses - use more descriptive variable names - better error handling for missing data - optimize vm filtering logic - early exit for empty backed_up_vms - + 17.06.26 SAN server-side filter + pagination + timeouts + api key test + 18.06.26 SAN fixed api_call_with_retries raise, removed dead code, moved api key test to main, hostname fallback, consistent .get(), removed globals and isComputer dead branch + 18.06.26 SAN fixed FORCE flow, KeyError protection on vm fields, extracted resolve_host_arg/find_matching_vm, pruned TODO + 18.06.26 SAN renamed vars (r/res/resp/i/p), optimized VM matching to single-pass, added ValueError guards, cleared all TODOs + 18.06.26 SAN fixed double-limit URL bug, removed apiGet_VMbackups, use VM list fields directly for restore point data + + """ import os @@ -55,6 +48,7 @@ import json import time import math +import socket import requests from datetime import datetime, timedelta @@ -68,33 +62,24 @@ def convert_size(bytes_): """Converts a size in bytes to a human-readable format.""" if bytes_ == 0: return "0B" - i = int(math.log(bytes_, 1024)) - return f"{round(bytes_ / (1024 ** i), 2)} {('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB')[i]}" - -def expiry_days(expiry): - """Calculates the number of days since a given expiry date.""" - expiry_date = datetime.strptime(expiry[:26], "%Y-%m-%dT%H:%M:%S.%f").date() - return (datetime.today().date() - expiry_date).days - -def get_timestamp(date_str): - """Converts a date string to a Unix timestamp.""" - return time.mktime(datetime.strptime(date_str[:26], "%Y-%m-%dT%H:%M:%S.%f").timetuple()) + size_index = int(math.log(bytes_, 1024)) + units = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB') + return f"{round(bytes_ / (1024 ** size_index), 2)} {units[size_index]}" def api_call_with_retries(url, method='GET', data=None, headers=None, retries=5, wait=60, timeout=60): """Makes an API call with retries for handling HTTP 429 responses.""" for attempt in range(retries): try: - res = requests.request(method, url, data=data, headers=headers, timeout=timeout) - if res.status_code == 429 and attempt < retries - 1: + response = requests.request(method, url, data=data, headers=headers, timeout=timeout) + if response.status_code == 429 and attempt < retries - 1: print(f"HTTP 429 received. Retrying in {wait} seconds... (Attempt {attempt + 1}/{retries})") time.sleep(wait) continue - res.raise_for_status() - return res + response.raise_for_status() + return response except requests.exceptions.RequestException as e: if attempt == retries - 1: - print(f"API call failed after {retries} attempts: {e}") - sys.exit(1) + raise time.sleep(wait) def api_get_all_paginated(base_url, headers, select_param=None, **kwargs): @@ -104,15 +89,14 @@ def api_get_all_paginated(base_url, headers, select_param=None, **kwargs): all_data = [] while True: - url = f"{base_url}&limit={limit}&offset={offset}" + url = f"{base_url}?limit={limit}&offset={offset}" if select_param: url += f"&select={select_param}" - res = api_call_with_retries(url, method='GET', headers=headers, **kwargs) - resp = res.json() - data = resp.get("data", []) - all_data.extend(data) + response = api_call_with_retries(url, method='GET', headers=headers, **kwargs) + parsed = response.json() + data = parsed.get("data", []) - paging = resp.get("meta", {}).get("pagingInfo", {}) + paging = parsed.get("meta", {}).get("pagingInfo", {}) total = paging.get("total", offset + len(data)) if offset + limit >= total: break @@ -133,41 +117,56 @@ def get_auth_headers(): def apiGet_BackedUpVMs(name_filter=None): """Retrieves backed-up VMs, with optional server-side name filter and pagination.""" headers = get_auth_headers() - select = '[{"propertyPath":"name"},{"propertyPath":"instanceUid"},{"propertyPath":"backupServerUid"}]' + select = '[{"propertyPath":"name"},{"propertyPath":"instanceUid"},{"propertyPath":"backupServerUid"},{"propertyPath":"latestRestorePointDate"},{"propertyPath":"totalRestorePointSize"},{"propertyPath":"restorePoints"}]' if name_filter: url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines" filter_json = json.dumps([{"property": "name", "operation": "contains", "collation": "ignorecase", "value": name_filter}]) url += f"?filter={filter_json}&select={select}" - r = api_call_with_retries(url, method='GET', headers=headers) - return r.json()["data"] + response = api_call_with_retries(url, method='GET', headers=headers, timeout=60) + return response.json().get("data", []) - base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines?limit=500" + base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines" return api_get_all_paginated(base_url, headers, select_param=select) -def apiGet_VMbackups(vmUID): - """Retrieves all backups for a specific VM with pagination.""" - headers = get_auth_headers() - base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines/{vmUID}/backups?limit=500" - return api_get_all_paginated(base_url, headers) - -def apiGet_BackedUpComputers(): - """Retrieves all backed-up computers with pagination.""" - headers = get_auth_headers() - base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/computersManagedByBackupServer?limit=500" - return api_get_all_paginated(base_url, headers) +def resolve_host_arg(): + """Determines the hostname to query from FORCE, HOST, or local hostname.""" + force_val = env_vars['FORCE'] + host_arg = force_val if (force_val and "Manual" not in force_val) else env_vars['HOST'] + if not host_arg: + host_arg = socket.gethostname() + log_debug(f"No HOST or FORCE set. Using local hostname: {host_arg}") + return host_arg + +def find_matching_vm(backed_up_vms, host_arg): + """Finds a unique VM matching host_arg. Returns the VM dict. Exits on zero or multiple matches.""" + if env_vars.get('FORCE'): + matching = [vm for vm in backed_up_vms if host_arg == vm.get("name")] + else: + host_arg_lower = host_arg.lower() + matching = [ + vm for vm in backed_up_vms + if host_arg in vm.get("name", "") or host_arg_lower in vm.get("name", "") + ] + + if not matching: + print(f"KO: VM or Computer '{host_arg}' not found in the backup list.") + sys.exit(2) + elif len(matching) > 1: + log_debug(f"WARNING: Multiple matches found for '{host_arg}':") + for vm in matching: + log_debug(f" - Name: {vm.get('name', 'UNKNOWN')}, VM UID: {vm.get('instanceUid', 'UNKNOWN')}") + print("Exiting to avoid mismatches.") + sys.exit(2) -def apiGet_ComputersRestorePoints(): - """Retrieves all computer restore points with pagination.""" - headers = get_auth_headers() - base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/computersManagedByBackupServer/restorePoints?limit=500" - return api_get_all_paginated(base_url, headers) + matched = matching[0] + if not matched.get("instanceUid"): + print("KO: Selected VM is missing instanceUid.") + sys.exit(2) + log_debug(f"INFO: Selected VM: {matched.get('name', 'UNKNOWN')} (UID: {matched['instanceUid']})") + return matched # === Environment and Constants === -vmUID, lastRPoint, vmBkp_bkpSrvUID, tmpName, strSchedule = ("",) * 5 -nameNotFound = True -isComputer = False - env_vars = { 'HOST': None, 'FORCE': None, @@ -184,11 +183,10 @@ def apiGet_ComputersRestorePoints(): print("CRITICAL: 'APIURL' and 'APIKEY' must be set.") sys.exit(2) -if env_vars['FORCE'] and "DISABLEDBACKUPCHECK" in env_vars['FORCE']: +if env_vars.get('FORCE') and "DISABLEDBACKUPCHECK" in env_vars['FORCE']: print("Backup check is disabled because 'FORCE' contains 'DISABLEDBACKUPCHECK'.") sys.exit(0) - def main(): try: log_debug("Parsed Environment Variables:") @@ -201,11 +199,16 @@ def main(): log_debug(f" APIURL: {env_vars['APIURL']}") log_debug(f" THRESHOLD_HOURS: {env_vars['THRESHOLD_HOURS']}\n") - host_arg = ( - env_vars['FORCE'] - if env_vars.get('FORCE') and "Manual" not in env_vars['FORCE'] - else env_vars['HOST'] - ) + log_debug("Testing API key validity...") + try: + select_json = '[{"propertyPath":"instanceUid"}]' + test_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines?limit=1&select={select_json}" + api_call_with_retries(test_url, method='GET', headers=get_auth_headers(), retries=1, timeout=60) + except requests.exceptions.RequestException: + print("KO: API key is invalid or the API is unreachable.") + sys.exit(2) + + host_arg = resolve_host_arg() log_debug(f"INFO: Fetching VMs filtered by name '{host_arg}'...") backed_up_vms = apiGet_BackedUpVMs(name_filter=host_arg) @@ -214,55 +217,38 @@ def main(): log_debug("Server-side filter returned no results. Falling back to full paginated fetch...") backed_up_vms = apiGet_BackedUpVMs() - for vm in backed_up_vms: - log_debug(f"VM Name: {vm['name']}") - - if env_vars.get('FORCE'): - matching_vms = [vm for vm in backed_up_vms if host_arg == vm["name"]] - else: - matching_vms = [vm for vm in backed_up_vms if host_arg in vm["name"]] - - if not matching_vms and not env_vars.get('FORCE'): - host_arg_lower = host_arg.lower() - matching_vms = [vm for vm in backed_up_vms if host_arg_lower in vm["name"]] - - if not matching_vms: - print(f"KO: VM or Computer '{host_arg}' not found in the backup list.") - sys.exit(2) - elif len(matching_vms) > 1: - log_debug(f"WARNING: Multiple matches found for '{host_arg}':") - for vm in matching_vms: - log_debug(f" - Name: {vm['name']}, VM UID: {vm['instanceUid']}") - print("Exiting to avoid mismatches.") + if not backed_up_vms: + print(f"KO: No VMs found in backup list.") sys.exit(2) - global vmUID, tmpName - vmUID = matching_vms[0]["instanceUid"] - tmpName = matching_vms[0]["name"] - log_debug(f"INFO: Selected VM: {tmpName} (UID: {vmUID})") + for vm in backed_up_vms: + log_debug(f"VM Name: {vm.get('name', 'UNKNOWN')}") - try: - restore_points = ( - apiGet_ComputersRestorePoints() if isComputer else apiGet_VMbackups(vmUID) - ) - except requests.exceptions.RequestException as e: - print("API CALL FAILED: Unable to fetch restore points.") - log_debug(str(e)) - sys.exit(2) + matched_vm = find_matching_vm(backed_up_vms, host_arg) + vm_name = matched_vm.get("name", "UNKNOWN") + latest_restore_point = matched_vm.get("latestRestorePointDate") + restore_point_count = matched_vm.get("restorePoints", 0) + total_restore_point_size = matched_vm.get("totalRestorePointSize", 0) - latest_restore_point = next( - (p['creationTimeUtc'] for p in restore_points if 'creationTimeUtc' in p), - next((p['latestRestorePointDate'] for p in restore_points if 'latestRestorePointDate' in p), None) - ) + log_debug(f"INFO: VM {vm_name}: restorePoints={restore_point_count}, latestRestorePointDate={latest_restore_point}, totalRestorePointSize={total_restore_point_size}") if not latest_restore_point: print("KO: No valid restore points found.") sys.exit(2) - restore_point_time = datetime.strptime(latest_restore_point[:26], "%Y-%m-%dT%H:%M:%S.%f") + try: + restore_point_time = datetime.strptime(latest_restore_point[:26], "%Y-%m-%dT%H:%M:%S.%f") + except (ValueError, TypeError): + print("KO: Unable to parse restore point date.") + sys.exit(2) + time_since_last_backup = datetime.utcnow() - restore_point_time - threshold_hours = int(env_vars['THRESHOLD_HOURS']) + try: + threshold_hours = int(env_vars['THRESHOLD_HOURS']) + except (ValueError, TypeError): + print("KO: THRESHOLD_HOURS must be a number.") + sys.exit(2) backup_age_limit = timedelta(hours=threshold_hours) if time_since_last_backup <= backup_age_limit: @@ -271,10 +257,9 @@ def main(): print(f"KO: The latest backup was {time_since_last_backup} ago, exceeding the threshold of {threshold_hours} hours.") sys.exit(2) - total_restore_point_size = sum(p.get('totalRestorePointSize', 0) for p in restore_points) total_restore_point_size_readable = convert_size(total_restore_point_size) - print(f"Restoration Status Report:\n- VM or Computer: {tmpName}\n- Latest Restore Point Date/Time: {latest_restore_point}\n- Number of Restore Points Available: {len(restore_points)}\n- Total Size of Restore Points: {total_restore_point_size_readable}") + print(f"Restoration Status Report:\n- VM or Computer: {vm_name}\n- Latest Restore Point Date/Time: {latest_restore_point}\n- Number of Restore Points Available: {restore_point_count}\n- Total Size of Restore Points: {total_restore_point_size_readable}") except requests.exceptions.RequestException as e: From 805563d23d9b4eb3ffae88d57034b0446d2152f3 Mon Sep 17 00:00:00 2001 From: P6g9YHK6 <17877371+P6g9YHK6@users.noreply.github.com> Date: Thu, 18 Jun 2026 11:28:55 +0200 Subject: [PATCH 4/8] Update file: Backup Veeam SPC.py --- scripts_staging/Checks/Backup Veeam SPC.py | 153 ++++++++++++++++----- 1 file changed, 119 insertions(+), 34 deletions(-) diff --git a/scripts_staging/Checks/Backup Veeam SPC.py b/scripts_staging/Checks/Backup Veeam SPC.py index 0e21319d..2b69b0f6 100644 --- a/scripts_staging/Checks/Backup Veeam SPC.py +++ b/scripts_staging/Checks/Backup Veeam SPC.py @@ -16,6 +16,7 @@ Optional THRESHOLD_HOURS=48 + WARNING_HOURS=36 (default: 75% of THRESHOLD_HOURS if omitted) force={{agent.Hostname_Override}} DEBUG=1 force=DISABLEDBACKUPCHECK @@ -30,6 +31,12 @@ - Debug logs if DEBUG is set to True in the environment variables. - Disabled if DISABLEDBACKUPCHECK is set in FORCE + Exit Codes: + 0 - OK (backup within warning threshold) + 1 - WARNING (backup age exceeds warning threshold, within critical threshold) + 2 - CRITICAL (backup expired, VM unprotected, or no restore points) + 3 - SCRIPT ERROR (API/config errors, VM ambiguous, script failure) + Changelog: 27.03.25 SAN added more debug @@ -39,7 +46,12 @@ 18.06.26 SAN fixed FORCE flow, KeyError protection on vm fields, extracted resolve_host_arg/find_matching_vm, pruned TODO 18.06.26 SAN renamed vars (r/res/resp/i/p), optimized VM matching to single-pass, added ValueError guards, cleared all TODOs 18.06.26 SAN fixed double-limit URL bug, removed apiGet_VMbackups, use VM list fields directly for restore point data - + 18.06.26 SAN replaced /about API key test with VM endpoint, fixed f-string backslash bug + 18.06.26 SAN fixed find_matching_vm exact-match logic when FORCE contains "Manual" + 18.06.26 SAN replaced deprecated datetime.utcnow() with datetime.now(timezone.utc) + 18.06.26 SAN item-4 per-restore-point breakdown (backupRestorePoints endpoint), item-13 job last session state, item-19 expand parameter, enriched Restoration Status Report + 18.06.26 SAN added WARNING_HOURS threshold, 4-level exit codes (0=OK, 1=WARNING, 2=CRITICAL, 3=ERROR) + 18.06.26 SAN removed malware state, removed dead job detail code, standardized function naming to snake_case, narrowed exceptions """ @@ -50,7 +62,13 @@ import math import socket import requests -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone + +# === Exit Codes === +EXIT_OK = 0 +EXIT_WARNING = 1 +EXIT_CRITICAL = 2 +EXIT_ERROR = 3 # === Utility Functions === def log_debug(msg): @@ -66,7 +84,7 @@ def convert_size(bytes_): units = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB') return f"{round(bytes_ / (1024 ** size_index), 2)} {units[size_index]}" -def api_call_with_retries(url, method='GET', data=None, headers=None, retries=5, wait=60, timeout=60): +def api_call_with_retries(url, method='GET', data=None, headers=None, retries=1, wait=3, timeout=10): """Makes an API call with retries for handling HTTP 429 responses.""" for attempt in range(retries): try: @@ -77,12 +95,12 @@ def api_call_with_retries(url, method='GET', data=None, headers=None, retries=5, continue response.raise_for_status() return response - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException: if attempt == retries - 1: raise time.sleep(wait) -def api_get_all_paginated(base_url, headers, select_param=None, **kwargs): +def api_get_all_paginated(base_url, headers, select_param=None, extra_params=None, **kwargs): """Fetches all pages from a paginated endpoint using offset-based pagination.""" limit = 500 offset = 0 @@ -92,10 +110,14 @@ def api_get_all_paginated(base_url, headers, select_param=None, **kwargs): url = f"{base_url}?limit={limit}&offset={offset}" if select_param: url += f"&select={select_param}" + if extra_params: + url += f"&{extra_params}" response = api_call_with_retries(url, method='GET', headers=headers, **kwargs) parsed = response.json() data = parsed.get("data", []) + all_data.extend(data) + paging = parsed.get("meta", {}).get("pagingInfo", {}) total = paging.get("total", offset + len(data)) if offset + limit >= total: @@ -114,20 +136,29 @@ def get_auth_headers(): "accept": "application/json", } -def apiGet_BackedUpVMs(name_filter=None): +def api_get_backed_up_vms(name_filter=None, expand=None): """Retrieves backed-up VMs, with optional server-side name filter and pagination.""" headers = get_auth_headers() - select = '[{"propertyPath":"name"},{"propertyPath":"instanceUid"},{"propertyPath":"backupServerUid"},{"propertyPath":"latestRestorePointDate"},{"propertyPath":"totalRestorePointSize"},{"propertyPath":"restorePoints"}]' + select = '[{"propertyPath":"name"},{"propertyPath":"instanceUid"},{"propertyPath":"backupServerUid"},{"propertyPath":"latestRestorePointDate"},{"propertyPath":"totalRestorePointSize"},{"propertyPath":"restorePoints"},{"propertyPath":"jobUid"},{"propertyPath":"immutable"}]' if name_filter: url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines" filter_json = json.dumps([{"property": "name", "operation": "contains", "collation": "ignorecase", "value": name_filter}]) url += f"?filter={filter_json}&select={select}" - response = api_call_with_retries(url, method='GET', headers=headers, timeout=60) + if expand: + url += f"&expand={expand}" + response = api_call_with_retries(url, method='GET', headers=headers) return response.json().get("data", []) base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines" - return api_get_all_paginated(base_url, headers, select_param=select) + return api_get_all_paginated(base_url, headers, select_param=select, + extra_params=f"expand={expand}" if expand else None) + +def api_get_vm_backup_restore_points(vm_uid): + """Fetches all backup restore points for a specific VM.""" + headers = get_auth_headers() + base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines/{vm_uid}/backupRestorePoints" + return api_get_all_paginated(base_url, headers, timeout=15) def resolve_host_arg(): """Determines the hostname to query from FORCE, HOST, or local hostname.""" @@ -140,7 +171,9 @@ def resolve_host_arg(): def find_matching_vm(backed_up_vms, host_arg): """Finds a unique VM matching host_arg. Returns the VM dict. Exits on zero or multiple matches.""" - if env_vars.get('FORCE'): + force_val = env_vars['FORCE'] + use_exact = force_val and "Manual" not in force_val + if use_exact: matching = [vm for vm in backed_up_vms if host_arg == vm.get("name")] else: host_arg_lower = host_arg.lower() @@ -151,18 +184,18 @@ def find_matching_vm(backed_up_vms, host_arg): if not matching: print(f"KO: VM or Computer '{host_arg}' not found in the backup list.") - sys.exit(2) + sys.exit(EXIT_CRITICAL) elif len(matching) > 1: log_debug(f"WARNING: Multiple matches found for '{host_arg}':") for vm in matching: log_debug(f" - Name: {vm.get('name', 'UNKNOWN')}, VM UID: {vm.get('instanceUid', 'UNKNOWN')}") print("Exiting to avoid mismatches.") - sys.exit(2) + sys.exit(EXIT_ERROR) matched = matching[0] if not matched.get("instanceUid"): print("KO: Selected VM is missing instanceUid.") - sys.exit(2) + sys.exit(EXIT_CRITICAL) log_debug(f"INFO: Selected VM: {matched.get('name', 'UNKNOWN')} (UID: {matched['instanceUid']})") return matched @@ -173,19 +206,21 @@ def find_matching_vm(backed_up_vms, host_arg): 'DEBUG': False, 'APIKEY': None, 'APIURL': None, - 'THRESHOLD_HOURS': 48 + 'THRESHOLD_HOURS': 48, + 'WARNING_HOURS': None, } env_vars.update({k: os.getenv(k, v) for k, v in env_vars.items()}) env_vars['DEBUG'] = str(env_vars['DEBUG']).lower() in ("true", "1") +threshold_set = 'THRESHOLD_HOURS' in os.environ # === Exit Early Conditions === if not env_vars['APIKEY'] or not env_vars['APIURL']: print("CRITICAL: 'APIURL' and 'APIKEY' must be set.") - sys.exit(2) + sys.exit(EXIT_ERROR) if env_vars.get('FORCE') and "DISABLEDBACKUPCHECK" in env_vars['FORCE']: print("Backup check is disabled because 'FORCE' contains 'DISABLEDBACKUPCHECK'.") - sys.exit(0) + sys.exit(EXIT_OK) def main(): try: @@ -197,34 +232,41 @@ def main(): masked_api_key = f"{api_key[:3]}{'*' * (len(api_key) - 6)}{api_key[-3:]}" log_debug(f" APIKEY: {masked_api_key}") log_debug(f" APIURL: {env_vars['APIURL']}") - log_debug(f" THRESHOLD_HOURS: {env_vars['THRESHOLD_HOURS']}\n") + log_debug(f" THRESHOLD_HOURS: {env_vars['THRESHOLD_HOURS']} ({'set' if threshold_set else 'default'})") + warning_val = env_vars['WARNING_HOURS'] + if warning_val: + log_debug(f" WARNING_HOURS: {warning_val} (set)") + else: + log_debug(" WARNING_HOURS: auto (75% of THRESHOLD_HOURS)") + log_debug("") log_debug("Testing API key validity...") try: select_json = '[{"propertyPath":"instanceUid"}]' test_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines?limit=1&select={select_json}" - api_call_with_retries(test_url, method='GET', headers=get_auth_headers(), retries=1, timeout=60) + api_call_with_retries(test_url, method='GET', headers=get_auth_headers(), timeout=7) except requests.exceptions.RequestException: print("KO: API key is invalid or the API is unreachable.") - sys.exit(2) + sys.exit(EXIT_ERROR) host_arg = resolve_host_arg() log_debug(f"INFO: Fetching VMs filtered by name '{host_arg}'...") - backed_up_vms = apiGet_BackedUpVMs(name_filter=host_arg) + backed_up_vms = api_get_backed_up_vms(name_filter=host_arg, expand="backupServer,job") if not backed_up_vms: log_debug("Server-side filter returned no results. Falling back to full paginated fetch...") - backed_up_vms = apiGet_BackedUpVMs() + backed_up_vms = api_get_backed_up_vms(expand="backupServer,job") if not backed_up_vms: - print(f"KO: No VMs found in backup list.") - sys.exit(2) + print("KO: No VMs found in backup list.") + sys.exit(EXIT_CRITICAL) for vm in backed_up_vms: log_debug(f"VM Name: {vm.get('name', 'UNKNOWN')}") matched_vm = find_matching_vm(backed_up_vms, host_arg) + vm_uid = matched_vm.get("instanceUid") vm_name = matched_vm.get("name", "UNKNOWN") latest_restore_point = matched_vm.get("latestRestorePointDate") restore_point_count = matched_vm.get("restorePoints", 0) @@ -234,38 +276,81 @@ def main(): if not latest_restore_point: print("KO: No valid restore points found.") - sys.exit(2) + sys.exit(EXIT_CRITICAL) try: restore_point_time = datetime.strptime(latest_restore_point[:26], "%Y-%m-%dT%H:%M:%S.%f") except (ValueError, TypeError): print("KO: Unable to parse restore point date.") - sys.exit(2) + sys.exit(EXIT_ERROR) - time_since_last_backup = datetime.utcnow() - restore_point_time + time_since_last_backup = datetime.now(timezone.utc).replace(tzinfo=None) - restore_point_time try: threshold_hours = int(env_vars['THRESHOLD_HOURS']) except (ValueError, TypeError): print("KO: THRESHOLD_HOURS must be a number.") - sys.exit(2) + sys.exit(EXIT_ERROR) backup_age_limit = timedelta(hours=threshold_hours) - if time_since_last_backup <= backup_age_limit: - print(f"OK: The latest backup was {time_since_last_backup} ago, within the threshold of {threshold_hours} hours.") + try: + warning_hours_env = env_vars.get('WARNING_HOURS') + warning_hours = int(warning_hours_env) if warning_hours_env else int(threshold_hours * 0.75) + except (ValueError, TypeError): + warning_hours = int(threshold_hours * 0.75) + warning_age_limit = timedelta(hours=warning_hours) + + if time_since_last_backup <= warning_age_limit: + print(f"OK: The latest backup was {time_since_last_backup} ago, within the warning threshold of {warning_hours} hours.") + elif time_since_last_backup <= backup_age_limit: + print(f"WARNING: The latest backup was {time_since_last_backup} ago, exceeding the warning threshold of {warning_hours} hours but within the critical threshold of {threshold_hours} hours.") + sys.exit(EXIT_WARNING) else: - print(f"KO: The latest backup was {time_since_last_backup} ago, exceeding the threshold of {threshold_hours} hours.") - sys.exit(2) + print(f"KO: The latest backup was {time_since_last_backup} ago, exceeding the critical threshold of {threshold_hours} hours.") + sys.exit(EXIT_CRITICAL) total_restore_point_size_readable = convert_size(total_restore_point_size) - print(f"Restoration Status Report:\n- VM or Computer: {vm_name}\n- Latest Restore Point Date/Time: {latest_restore_point}\n- Number of Restore Points Available: {restore_point_count}\n- Total Size of Restore Points: {total_restore_point_size_readable}") - + # Enrich with per-restore-point breakdown (Item 4) + consistent_count = 0 + inconsistent_count = 0 + oldest_rp = None + newest_rp = None + if vm_uid: + try: + restore_points = api_get_vm_backup_restore_points(vm_uid) + for rp in restore_points: + if rp.get("isConsistent"): + consistent_count += 1 + else: + inconsistent_count += 1 + rp_time = rp.get("backupCreationTime", "") + if rp_time: + if not oldest_rp or rp_time < oldest_rp: + oldest_rp = rp_time + if not newest_rp or rp_time > newest_rp: + newest_rp = rp_time + log_debug(f"Restore points fetched: {len(restore_points)} total") + except (ValueError, requests.exceptions.RequestException) as e: + log_debug(f"Failed to fetch restore points: {e}") + + print("Restoration Status Report:") + print(f"- VM or Computer: {vm_name}") + print(f"- Latest Restore Point Date/Time: {latest_restore_point}") + print(f"- Number of Restore Points Available: {restore_point_count}") + print(f"- Total Size of Restore Points: {total_restore_point_size_readable}") + if vm_uid: + print(f"- Restore Points Consistent: {consistent_count}") + print(f"- Restore Points Inconsistent: {inconsistent_count}") + if oldest_rp: + print(f"- Oldest Restore Point: {oldest_rp}") + if newest_rp: + print(f"- Newest Restore Point: {newest_rp}") except requests.exceptions.RequestException as e: print("KO: API call failed.") log_debug("API CALL FAILED: " + str(e)) - sys.exit(2) + sys.exit(EXIT_ERROR) if __name__ == "__main__": main() \ No newline at end of file From 2e47a13b8d104c3d21f602bb7fecbf44d18e339c Mon Sep 17 00:00:00 2001 From: P6g9YHK6 <17877371+P6g9YHK6@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:14:44 +0200 Subject: [PATCH 5/8] Update file: Backup Veeam SPC.py --- scripts_staging/Checks/Backup Veeam SPC.py | 74 ++++++++++++++++++---- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/scripts_staging/Checks/Backup Veeam SPC.py b/scripts_staging/Checks/Backup Veeam SPC.py index 2b69b0f6..cce7a61b 100644 --- a/scripts_staging/Checks/Backup Veeam SPC.py +++ b/scripts_staging/Checks/Backup Veeam SPC.py @@ -41,17 +41,17 @@ 27.03.25 SAN added more debug 15.04.25 SAN big code cleanup + publication - 17.06.26 SAN server-side filter + pagination + timeouts + api key test - 18.06.26 SAN fixed api_call_with_retries raise, removed dead code, moved api key test to main, hostname fallback, consistent .get(), removed globals and isComputer dead branch - 18.06.26 SAN fixed FORCE flow, KeyError protection on vm fields, extracted resolve_host_arg/find_matching_vm, pruned TODO - 18.06.26 SAN renamed vars (r/res/resp/i/p), optimized VM matching to single-pass, added ValueError guards, cleared all TODOs - 18.06.26 SAN fixed double-limit URL bug, removed apiGet_VMbackups, use VM list fields directly for restore point data - 18.06.26 SAN replaced /about API key test with VM endpoint, fixed f-string backslash bug - 18.06.26 SAN fixed find_matching_vm exact-match logic when FORCE contains "Manual" - 18.06.26 SAN replaced deprecated datetime.utcnow() with datetime.now(timezone.utc) - 18.06.26 SAN item-4 per-restore-point breakdown (backupRestorePoints endpoint), item-13 job last session state, item-19 expand parameter, enriched Restoration Status Report - 18.06.26 SAN added WARNING_HOURS threshold, 4-level exit codes (0=OK, 1=WARNING, 2=CRITICAL, 3=ERROR) - 18.06.26 SAN removed malware state, removed dead job detail code, standardized function naming to snake_case, narrowed exceptions + 17.06.26 SAN server-side filter + pagination + timeouts + api key test + 18.06.26 SAN fixed api_call_with_retries raise, removed dead code, moved api key test to main, hostname fallback, consistent .get(), removed globals and isComputer dead branch + 18.06.26 SAN fixed FORCE flow, KeyError protection on vm fields, extracted resolve_host_arg/find_matching_vm, pruned TODO + 18.06.26 SAN renamed vars (r/res/resp/i/p), optimized VM matching to single-pass, added ValueError guards, cleared all TODOs + 18.06.26 SAN fixed double-limit URL bug, removed apiGet_VMbackups, use VM list fields directly for restore point data + 18.06.26 SAN replaced /about API key test with VM endpoint, fixed f-string backslash bug + 18.06.26 SAN fixed find_matching_vm exact-match logic when FORCE contains "Manual" + 18.06.26 SAN replaced deprecated datetime.utcnow() with datetime.now(timezone.utc) + 18.06.26 SAN item-4 per-restore-point breakdown (backupRestorePoints endpoint), item-13 job last session state, item-19 expand parameter, enriched Restoration Status Report + 18.06.26 SAN added WARNING_HOURS threshold, 4-level exit codes (0=OK, 1=WARNING, 2=CRITICAL, 3=ERROR) + 18.06.26 SAN removed malware state, removed dead job detail code, standardized function naming to snake_case, narrowed exceptions """ @@ -139,7 +139,7 @@ def get_auth_headers(): def api_get_backed_up_vms(name_filter=None, expand=None): """Retrieves backed-up VMs, with optional server-side name filter and pagination.""" headers = get_auth_headers() - select = '[{"propertyPath":"name"},{"propertyPath":"instanceUid"},{"propertyPath":"backupServerUid"},{"propertyPath":"latestRestorePointDate"},{"propertyPath":"totalRestorePointSize"},{"propertyPath":"restorePoints"},{"propertyPath":"jobUid"},{"propertyPath":"immutable"}]' + select = '[{"propertyPath":"name"},{"propertyPath":"instanceUid"},{"propertyPath":"backupServerUid"},{"propertyPath":"latestRestorePointDate"},{"propertyPath":"totalRestorePointSize"},{"propertyPath":"restorePoints"},{"propertyPath":"jobUid"},{"propertyPath":"immutable"},{"propertyPath":"repositoryUid"}]' if name_filter: url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines" @@ -160,6 +160,26 @@ def api_get_vm_backup_restore_points(vm_uid): base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines/{vm_uid}/backupRestorePoints" return api_get_all_paginated(base_url, headers, timeout=15) +def api_get_backup_repo(srv_uid, repo_uid): + """Fetches a specific backup repository by server UID and repository UID.""" + headers = get_auth_headers() + url = f"{env_vars['APIURL']}/api/v3/infrastructure/backupServers/{srv_uid}/repositories/{repo_uid}" + response = api_call_with_retries(url, method='GET', headers=headers) + return response.json() + +def api_get_job_infos(job_uid): + """Fetches job information by job UID.""" + headers = get_auth_headers() + url = f"{env_vars['APIURL']}/api/v3/infrastructure/backupServers/jobs/{job_uid}" + response = api_call_with_retries(url, method='GET', headers=headers) + return response.json() + +def api_get_all_repos(): + """Fetches all backup repositories across all backup servers with pagination.""" + headers = get_auth_headers() + base_url = f"{env_vars['APIURL']}/api/v3/infrastructure/backupServers/repositories" + return api_get_all_paginated(base_url, headers) + def resolve_host_arg(): """Determines the hostname to query from FORCE, HOST, or local hostname.""" force_val = env_vars['FORCE'] @@ -311,6 +331,30 @@ def main(): total_restore_point_size_readable = convert_size(total_restore_point_size) + backup_server_uid = matched_vm.get("backupServerUid") + job_uid = matched_vm.get("jobUid") + repo_name = None + job_name = None + job_last_state = None + + if backup_server_uid: + repo_uid = matched_vm.get("repositoryUid") + if repo_uid: + try: + repo_data = api_get_backup_repo(backup_server_uid, repo_uid) + repo_name = repo_data.get("data", {}).get("name") + except (ValueError, requests.exceptions.RequestException) as e: + log_debug(f"Failed to fetch repo info: {e}") + + if job_uid: + try: + job_data = api_get_job_infos(job_uid) + job_info = job_data.get("data", {}) + job_name = job_info.get("name") + job_last_state = job_info.get("lastState") + except (ValueError, requests.exceptions.RequestException) as e: + log_debug(f"Failed to fetch job info: {e}") + # Enrich with per-restore-point breakdown (Item 4) consistent_count = 0 inconsistent_count = 0 @@ -339,6 +383,12 @@ def main(): print(f"- Latest Restore Point Date/Time: {latest_restore_point}") print(f"- Number of Restore Points Available: {restore_point_count}") print(f"- Total Size of Restore Points: {total_restore_point_size_readable}") + if repo_name: + print(f"- Backup Repository: {repo_name}") + if job_name: + print(f"- Backup Job: {job_name}") + if job_last_state: + print(f"- Job Last State: {job_last_state}") if vm_uid: print(f"- Restore Points Consistent: {consistent_count}") print(f"- Restore Points Inconsistent: {inconsistent_count}") From 0b2cece8ea2b04bc1f7532a103fd11537bdfcb93 Mon Sep 17 00:00:00 2001 From: P6g9YHK6 <17877371+P6g9YHK6@users.noreply.github.com> Date: Thu, 18 Jun 2026 22:32:50 +0200 Subject: [PATCH 6/8] Update file: Backup Veeam SPC.py --- scripts_staging/Checks/Backup Veeam SPC.py | 64 ++++++++++++++++------ 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/scripts_staging/Checks/Backup Veeam SPC.py b/scripts_staging/Checks/Backup Veeam SPC.py index cce7a61b..e079c37d 100644 --- a/scripts_staging/Checks/Backup Veeam SPC.py +++ b/scripts_staging/Checks/Backup Veeam SPC.py @@ -180,6 +180,13 @@ def api_get_all_repos(): base_url = f"{env_vars['APIURL']}/api/v3/infrastructure/backupServers/repositories" return api_get_all_paginated(base_url, headers) +def api_get_backup_server(server_uid): + """Fetches backup server details by server UID.""" + headers = get_auth_headers() + url = f"{env_vars['APIURL']}/api/v3/infrastructure/backupServers/{server_uid}" + response = api_call_with_retries(url, method='GET', headers=headers) + return response.json() + def resolve_host_arg(): """Determines the hostname to query from FORCE, HOST, or local hostname.""" force_val = env_vars['FORCE'] @@ -335,25 +342,27 @@ def main(): job_uid = matched_vm.get("jobUid") repo_name = None job_name = None - job_last_state = None + job_status = None + server_name = None - if backup_server_uid: - repo_uid = matched_vm.get("repositoryUid") - if repo_uid: - try: - repo_data = api_get_backup_repo(backup_server_uid, repo_uid) - repo_name = repo_data.get("data", {}).get("name") - except (ValueError, requests.exceptions.RequestException) as e: - log_debug(f"Failed to fetch repo info: {e}") - - if job_uid: + if backup_server_uid and job_uid: try: job_data = api_get_job_infos(job_uid) job_info = job_data.get("data", {}) job_name = job_info.get("name") - job_last_state = job_info.get("lastState") + job_status = job_info.get("status") + repo_name = job_info.get("destination") + log_debug(f"Job fetched: {job_name}, status={job_status}, destination={repo_name} (jobUID={job_uid})") + try: + server_data = api_get_backup_server(backup_server_uid) + server_name = server_data.get("data", {}).get("name") + log_debug(f"Backup server: {server_name} (uid={backup_server_uid})") + except (ValueError, requests.exceptions.RequestException) as e: + log_debug(f"Failed to fetch backup server info (uid={backup_server_uid}): {e}") except (ValueError, requests.exceptions.RequestException) as e: - log_debug(f"Failed to fetch job info: {e}") + log_debug(f"Failed to fetch job info (jobUID={job_uid}): {e}") + else: + log_debug(f"backupServerUid or jobUid not present in VM data, skipping repo/job lookup") # Enrich with per-restore-point breakdown (Item 4) consistent_count = 0 @@ -363,6 +372,7 @@ def main(): if vm_uid: try: restore_points = api_get_vm_backup_restore_points(vm_uid) + log_debug(f"Restore points API returned {len(restore_points)} items for VM {vm_uid}") for rp in restore_points: if rp.get("isConsistent"): consistent_count += 1 @@ -374,28 +384,48 @@ def main(): oldest_rp = rp_time if not newest_rp or rp_time > newest_rp: newest_rp = rp_time - log_debug(f"Restore points fetched: {len(restore_points)} total") + log_debug(f"Restore point stats: {consistent_count} consistent, {inconsistent_count} inconsistent") + if oldest_rp: + log_debug(f"Oldest restore point: {oldest_rp}") + if newest_rp: + log_debug(f"Newest restore point: {newest_rp}") except (ValueError, requests.exceptions.RequestException) as e: - log_debug(f"Failed to fetch restore points: {e}") + log_debug(f"Failed to fetch restore points for VM {vm_uid}: {e}") + else: + log_debug(f"vm_uid not present in VM data, skipping restore points lookup") print("Restoration Status Report:") print(f"- VM or Computer: {vm_name}") print(f"- Latest Restore Point Date/Time: {latest_restore_point}") print(f"- Number of Restore Points Available: {restore_point_count}") print(f"- Total Size of Restore Points: {total_restore_point_size_readable}") + if server_name: + print(f"- Backup Server: {server_name}") + else: + log_debug("Backup Server: not available (see above for reason)") if repo_name: print(f"- Backup Repository: {repo_name}") + else: + log_debug("Backup Repository: not available (see above for reason)") if job_name: print(f"- Backup Job: {job_name}") - if job_last_state: - print(f"- Job Last State: {job_last_state}") + else: + log_debug("Backup Job: not available (see above for reason)") + if job_status: + print(f"- Job Status: {job_status}") + else: + log_debug("Job Status: not available (see above for reason)") if vm_uid: print(f"- Restore Points Consistent: {consistent_count}") print(f"- Restore Points Inconsistent: {inconsistent_count}") if oldest_rp: print(f"- Oldest Restore Point: {oldest_rp}") + else: + log_debug("Oldest Restore Point: not available from API response") if newest_rp: print(f"- Newest Restore Point: {newest_rp}") + else: + log_debug("Newest Restore Point: not available from API response") except requests.exceptions.RequestException as e: print("KO: API call failed.") From f98d6036b33ce293fc77ffd9071d67ee9ba595f9 Mon Sep 17 00:00:00 2001 From: P6g9YHK6 <17877371+P6g9YHK6@users.noreply.github.com> Date: Thu, 18 Jun 2026 22:39:09 +0200 Subject: [PATCH 7/8] Update file: Backup Veeam SPC.py --- scripts_staging/Checks/Backup Veeam SPC.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/scripts_staging/Checks/Backup Veeam SPC.py b/scripts_staging/Checks/Backup Veeam SPC.py index e079c37d..1492fc1b 100644 --- a/scripts_staging/Checks/Backup Veeam SPC.py +++ b/scripts_staging/Checks/Backup Veeam SPC.py @@ -52,6 +52,7 @@ 18.06.26 SAN item-4 per-restore-point breakdown (backupRestorePoints endpoint), item-13 job last session state, item-19 expand parameter, enriched Restoration Status Report 18.06.26 SAN added WARNING_HOURS threshold, 4-level exit codes (0=OK, 1=WARNING, 2=CRITICAL, 3=ERROR) 18.06.26 SAN removed malware state, removed dead job detail code, standardized function naming to snake_case, narrowed exceptions + 18.06.26 SAN added api_get_job_infos/api_get_backup_server, enriched report with Backup Server/Job/Status/Repository, job status check exits CRITICAL, removed unused api_get_backup_repo/api_get_all_repos, enhanced debug logging for missing fields """ @@ -160,13 +161,6 @@ def api_get_vm_backup_restore_points(vm_uid): base_url = f"{env_vars['APIURL']}/api/v3/protectedWorkloads/virtualMachines/{vm_uid}/backupRestorePoints" return api_get_all_paginated(base_url, headers, timeout=15) -def api_get_backup_repo(srv_uid, repo_uid): - """Fetches a specific backup repository by server UID and repository UID.""" - headers = get_auth_headers() - url = f"{env_vars['APIURL']}/api/v3/infrastructure/backupServers/{srv_uid}/repositories/{repo_uid}" - response = api_call_with_retries(url, method='GET', headers=headers) - return response.json() - def api_get_job_infos(job_uid): """Fetches job information by job UID.""" headers = get_auth_headers() @@ -174,12 +168,6 @@ def api_get_job_infos(job_uid): response = api_call_with_retries(url, method='GET', headers=headers) return response.json() -def api_get_all_repos(): - """Fetches all backup repositories across all backup servers with pagination.""" - headers = get_auth_headers() - base_url = f"{env_vars['APIURL']}/api/v3/infrastructure/backupServers/repositories" - return api_get_all_paginated(base_url, headers) - def api_get_backup_server(server_uid): """Fetches backup server details by server UID.""" headers = get_auth_headers() @@ -352,6 +340,9 @@ def main(): job_name = job_info.get("name") job_status = job_info.get("status") repo_name = job_info.get("destination") + if job_status and job_status != "Success": + print(f"CRITICAL: Job '{job_name}' last status is '{job_status}'.") + sys.exit(EXIT_CRITICAL) log_debug(f"Job fetched: {job_name}, status={job_status}, destination={repo_name} (jobUID={job_uid})") try: server_data = api_get_backup_server(backup_server_uid) From da142e094639ccb71038c86de7ab3d75bafe5ea5 Mon Sep 17 00:00:00 2001 From: P6g9YHK6 <17877371+P6g9YHK6@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:09:45 +0200 Subject: [PATCH 8/8] Add new file: TRMM Variable examples.ps1 --- .../Lab/TRMM Variable examples.ps1 | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 scripts_staging/Lab/TRMM Variable examples.ps1 diff --git a/scripts_staging/Lab/TRMM Variable examples.ps1 b/scripts_staging/Lab/TRMM Variable examples.ps1 new file mode 100644 index 00000000..c0451628 --- /dev/null +++ b/scripts_staging/Lab/TRMM Variable examples.ps1 @@ -0,0 +1,119 @@ +<# +.SYNOPSIS + Outputs Tactical RMM pre-made variables with prefixes for exemple. + + Documentation for script variables: https://docs.tacticalrmm.com/script_variables/ + + Documentation for custom fields: https://docs.tacticalrmm.com/functions/custom_fields/ + + Documentation for global keystore/custom fields: https://docs.tacticalrmm.com/functions/keystore/ + +.EXEMPLE + Example input in Environment vars: + version={{agent.version}} + operating_system={{agent.operating_system}} + plat={{agent.plat}} + hostname={{agent.hostname}} + local_ips={{agent.local_ips}} + public_ip={{agent.public_ip}} + agent_id={{agent.agent_id}} + last_seen={{agent.last_seen}} + total_ram={{agent.total_ram}} + boot_time={{agent.boot_time}} + logged_in_username={{agent.logged_in_username}} + last_logged_in_user={{agent.last_logged_in_user}} + monitoring_type={{agent.monitoring_type}} + description={{agent.description}} + mesh_node_id={{agent.mesh_node_id}} + overdue_email_alert={{agent.overdue_email_alert}} + overdue_text_alert={{agent.overdue_text_alert}} + overdue_dashboard_alert={{agent.overdue_dashboard_alert}} + offline_time={{agent.offline_time}} + overdue_time={{agent.overdue_time}} + check_interval={{agent.check_interval}} + needs_reboot={{agent.needs_reboot}} + choco_installed={{agent.choco_installed}} + patches_last_installed={{agent.patches_last_installed}} + timezone={{agent.timezone}} + maintenance_mode={{agent.maintenance_mode}} + block_policy_inheritance={{agent.block_policy_inheritance}} + alert_template={{agent.alert_template}} + site={{agent.site}} + + client_name={{client.name}} + + site_name={{site.name}} + site_client={{site.client}} + + Custom: + agent.custom={{agent.custom}} + site.custom={{site.custom}} + client.custom={{client.custom}} + global.custom={{global.custom}} + +.NOTE + Author: SAN + Date: 06.01.25 + #public + +#> + +# Block 1: Agent pre-made variables +Write-Output "===== Agent Information =====" +Write-Output "agent.version: $env:version" +Write-Output "agent.operating_system: $env:operating_system" +Write-Output "agent.plat: $env:plat" +Write-Output "agent.hostname: $env:hostname" +Write-Output "agent.local_ips: $env:local_ips" +Write-Output "agent.public_ip: $env:public_ip" +Write-Output "agent.agent_id: $env:agent_id" +Write-Output "agent.last_seen: $env:last_seen" +Write-Output "agent.total_ram: $env:total_ram" +Write-Output "agent.boot_time: $env:boot_time" +Write-Output "agent.logged_in_username: $env:logged_in_username" +Write-Output "agent.last_logged_in_user: $env:last_logged_in_user" +Write-Output "agent.monitoring_type: $env:monitoring_type" +Write-Output "agent.description: $env:description" +Write-Output "agent.mesh_node_id: $env:mesh_node_id" +Write-Output "agent.overdue_email_alert: $env:overdue_email_alert" +Write-Output "agent.overdue_text_alert: $env:overdue_text_alert" +Write-Output "agent.overdue_dashboard_alert: $env:overdue_dashboard_alert" +Write-Output "agent.offline_time: $env:offline_time" +Write-Output "agent.overdue_time: $env:overdue_time" +Write-Output "agent.check_interval: $env:check_interval" +Write-Output "agent.needs_reboot: $env:needs_reboot" +Write-Output "agent.choco_installed: $env:choco_installed" +Write-Output "agent.patches_last_installed: $env:patches_last_installed" +Write-Output "agent.timezone: $env:timezone" +Write-Output "agent.maintenance_mode: $env:maintenance_mode" +Write-Output "agent.block_policy_inheritance: $env:block_policy_inheritance" +Write-Output "agent.alert_template: $env:alert_template" +Write-Output "agent.site: $env:site" +Write-Output "" + +# Block 2: Client pre-made variables +Write-Output "===== Client Information =====" +Write-Output "client.name: $env:client_name" +Write-Output "" + +# Block 3: Site pre-made variables +Write-Output "===== Site Information =====" +Write-Output "site.name: $env:site_name" +Write-Output "site.client: $env:site_client" +Write-Output "" + +# Block 4: Agent Custom fields +Write-Output "===== Agent Custom fields =====" +Write-Output "" + +# Block 5: Site Custom fields +Write-Output "===== Site Custom fields =====" +Write-Output "" + +# Block 6: Client Custom fields +Write-Output "===== Client Custom fields =====" +Write-Output "" + +# Block 7: Global Custom fields +Write-Output "===== Global Custom fields =====" +Write-Output "" \ No newline at end of file