diff --git a/gpu-dashboard.json b/gpu-dashboard.json index 1e855e2..4ff4e3e 100644 --- a/gpu-dashboard.json +++ b/gpu-dashboard.json @@ -372,7 +372,7 @@ }, { "title": "VRAM free (FB=0)", - "description": "GPUs with no framebuffer in use. PromQL: count(DCGM_FI_DEV_FB_USED == 0). Equals Total \u2212 VRAM allocated (FB>0).", + "description": "GPUs with no framebuffer in use. PromQL: count(DCGM_FI_DEV_FB_USED == 0). Equals Total − VRAM allocated (FB>0).", "type": "stat", "gridPos": { "h": 4, @@ -459,7 +459,7 @@ }, { "title": "Engine active (30m)", - "description": "GPUs with \u2265 1% engine activity at least once in 30m. Uses DCGM_FI_PROF_GR_ENGINE_ACTIVE or DCGM_FI_DEV_GPU_UTIL fallback. Equals Total \u2212 Engine idle (30m).", + "description": "GPUs with ≥ 1% engine activity at least once in 30m. Uses DCGM_FI_PROF_GR_ENGINE_ACTIVE or DCGM_FI_DEV_GPU_UTIL fallback. Equals Total − Engine idle (30m).", "type": "stat", "gridPos": { "h": 4, @@ -835,7 +835,7 @@ }, { "title": "Idle GPU Time by Deployment (30m)", - "description": "Deployments producing the most allocated GPU idle time at 0% utilization. Sorted by total idle GPU-hours (GPU count \u00d7 30min window). Uses kube_pod_labels when available; otherwise derives deployment name from pod name.", + "description": "Deployments producing the most allocated GPU idle time at 0% utilization. Sorted by total idle GPU-hours (GPU count × 30min window). Uses kube_pod_labels when available; otherwise derives deployment name from pod name.", "type": "table", "gridPos": { "h": 10, @@ -978,7 +978,7 @@ }, { "title": "GPU Memory Allocation Leaderboard (GiB)", - "description": "Estimated GPU memory allocated per namespace. Calculated as GPUs requested \u00d7 140 GiB (H200 FB total).", + "description": "Estimated GPU memory allocated per namespace. Calculated as GPUs requested × 140 GiB (H200 FB total).", "type": "barchart", "gridPos": { "h": 10, @@ -1075,7 +1075,7 @@ "targets": [ { "expr": "max(DCGM_FI_DEV_GPU_TEMP)", - "legendFormat": "Max \u00b0C" + "legendFormat": "Max °C" } ], "fieldConfig": { @@ -1388,7 +1388,7 @@ }, { "title": "Graphics/compute engine active by node", - "description": "Average graphics/compute engine activity (DCGM_FI_PROF_GR_ENGINE_ACTIVE, 0\u20131). Primary metric for idle detection in gpu-pruner and engine idle/active overview stats.", + "description": "Average graphics/compute engine activity (DCGM_FI_PROF_GR_ENGINE_ACTIVE, 0–1). Primary metric for idle detection in gpu-pruner and engine idle/active overview stats.", "type": "timeseries", "gridPos": { "h": 6, @@ -1521,6 +1521,268 @@ } } } + }, + { + "title": "GPU Leaderboard", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 126 + }, + "collapsed": false + }, + { + "title": "Idle GPU Hours (7 Days)", + "description": "Total GPU hours where utilization < 1% over the past 7 days, grouped by namespace. Higher values indicate more GPU waste.", + "type": "table", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 127 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "targets": [ + { + "expr": "sort_desc(sum by (namespace, pod) (count_over_time((DCGM_FI_PROF_GR_ENGINE_ACTIVE{namespace!~\"llm-d-nightly-.*|bench-guide-.*|cw-.*\"} < 0.01)[7d:1m]) / 60))", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "none", + "decimals": 1 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "exported_namespace" + }, + "properties": [ + { + "id": "displayName", + "value": "Namespace" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Idle Hours" + }, + { + "id": "custom.displayMode", + "value": "color-background" + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + } + }, + { + "title": "Utilized GPU Hours (7 Days)", + "description": "Total GPU hours where utilization >= 1% over the past 7 days, grouped by namespace. Higher values indicate more actual GPU usage.", + "type": "table", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 127 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "targets": [ + { + "expr": "sort_desc(sum by (namespace, pod) (count_over_time((DCGM_FI_PROF_GR_ENGINE_ACTIVE{namespace!~\"llm-d-nightly-.*|bench-guide-.*|cw-.*\"} >= 0.01)[7d:1m]) / 60))", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "green", + "value": 500 + } + ] + }, + "unit": "none", + "decimals": 1 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "exported_namespace" + }, + "properties": [ + { + "id": "displayName", + "value": "Namespace" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Utilized Hours" + }, + { + "id": "custom.displayMode", + "value": "color-background" + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + } + }, + { + "title": "GPU Efficiency (7 Days)", + "description": "Efficiency percentage: utilized hours / (utilized + idle hours) * 100. Higher is better.", + "type": "bargauge", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 135 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "targets": [ + { + "expr": "sum by (namespace) (count_over_time((DCGM_FI_PROF_GR_ENGINE_ACTIVE{namespace!~\"llm-d-nightly-.*|bench-guide-.*|cw-.*\"} >= 0.01)[7d:1m])) / sum by (namespace) (count_over_time(DCGM_FI_PROF_GR_ENGINE_ACTIVE{namespace!~\"llm-d-nightly-.*|bench-guide-.*|cw-.*\"}[7d:1m])) * 100", + "format": "time_series", + "instant": true, + "legendFormat": "{{exported_namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 25 + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 75 + } + ] + }, + "unit": "percent" + } + }, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + } } ], "schemaVersion": 39, diff --git a/gpu-pruner/hack/clusterrole.yaml b/gpu-pruner/hack/clusterrole.yaml index a8263be..b406e87 100644 --- a/gpu-pruner/hack/clusterrole.yaml +++ b/gpu-pruner/hack/clusterrole.yaml @@ -57,4 +57,19 @@ rules: - patch - delete apiGroups: + - serving.kserve.io + resources: - inferenceservices + - verbs: + - get + - list + - watch + - create + - update + - patch + - delete + apiGroups: + - leaderworkerset.x-k8s.io + resources: + - leaderworkersets + - leaderworkersets/scale diff --git a/gpu-pruner/src/lib.rs b/gpu-pruner/src/lib.rs index f31d5b0..3c0664a 100644 --- a/gpu-pruner/src/lib.rs +++ b/gpu-pruner/src/lib.rs @@ -11,7 +11,9 @@ use k8s_openapi::{ apimachinery::pkg::apis::meta::v1::MicroTime, }; use kube::{Client, ResourceExt, api::PostParams}; -use resources::{inferenceservice::InferenceService, notebook::Notebook}; +use resources::{ + inferenceservice::InferenceService, leaderworkerset::LeaderWorkerSet, notebook::Notebook, +}; use secrecy::ExposeSecret; use serde::Serialize; use std::{ @@ -95,6 +97,7 @@ pub enum ScaleKind { StatefulSet(StatefulSet), InferenceService(Box), Notebook(Notebook), + LeaderWorkerSet(LeaderWorkerSet), } impl PartialEq for ScaleKind { @@ -105,6 +108,7 @@ impl PartialEq for ScaleKind { (ScaleKind::StatefulSet(a), ScaleKind::StatefulSet(b)) => a == b, (ScaleKind::InferenceService(a), ScaleKind::InferenceService(b)) => a.uid() == b.uid(), (ScaleKind::Notebook(a), ScaleKind::Notebook(b)) => a.uid() == b.uid(), + (ScaleKind::LeaderWorkerSet(a), ScaleKind::LeaderWorkerSet(b)) => a.uid() == b.uid(), // If they are different variants, they are not equal _ => false, } @@ -132,6 +136,9 @@ impl Hash for ScaleKind { ScaleKind::Notebook(a) => { a.uid().hash(state); } + ScaleKind::LeaderWorkerSet(a) => { + a.uid().hash(state); + } } } } @@ -144,6 +151,7 @@ impl From for ResourceKind { ScaleKind::StatefulSet(_) => ResourceKind::STATEFUL_SET, ScaleKind::InferenceService(_) => ResourceKind::INFERENCE_SERVICE, ScaleKind::Notebook(_) => ResourceKind::NOTEBOOK, + ScaleKind::LeaderWorkerSet(_) => ResourceKind::LEADER_WORKER_SET, } } } @@ -156,19 +164,29 @@ bitflags! { const STATEFUL_SET = 0b00100; const INFERENCE_SERVICE = 0b01000; const NOTEBOOK = 0b10000; + const LEADER_WORKER_SET = 0b100000; } } /// Parse a string of resource flag characters into a [`ResourceKind`] bitflag set. /// +/// An empty string enables all resource types. +/// /// - `d` → Deployment /// - `r` → ReplicaSet /// - `s` → StatefulSet /// - `i` → InferenceService /// - `n` → Notebook +/// - `l` → LeaderWorkerSet +/// +/// LeaderWorkerSet is automatically enabled with any non-empty resource combination. /// /// Unknown characters are silently ignored. pub fn get_enabled_resources(enabled_resources: &str) -> ResourceKind { + if enabled_resources.is_empty() { + return ResourceKind::all(); + } + let mut resource_kind = ResourceKind::empty(); for c in enabled_resources.chars() { match c { @@ -177,9 +195,14 @@ pub fn get_enabled_resources(enabled_resources: &str) -> ResourceKind { 's' => resource_kind |= ResourceKind::STATEFUL_SET, 'i' => resource_kind |= ResourceKind::INFERENCE_SERVICE, 'n' => resource_kind |= ResourceKind::NOTEBOOK, + 'l' => resource_kind |= ResourceKind::LEADER_WORKER_SET, _ => {} } } + // Auto-enable LeaderWorkerSet whenever any resources are enabled + if !resource_kind.is_empty() { + resource_kind |= ResourceKind::LEADER_WORKER_SET; + } resource_kind } @@ -365,6 +388,7 @@ macro_rules! delegate_resource_ext { ScaleKind::StatefulSet(d) => d.$method(), ScaleKind::InferenceService(d) => d.$method(), ScaleKind::Notebook(d) => d.$method(), + ScaleKind::LeaderWorkerSet(d) => d.$method(), } }; } @@ -385,6 +409,7 @@ impl Meta for ScaleKind { ScaleKind::StatefulSet(_) => StatefulSet::API_VERSION.to_string(), ScaleKind::Notebook(_) => "v1".to_string(), ScaleKind::InferenceService(_) => "v1beta1".to_string(), + ScaleKind::LeaderWorkerSet(_) => "v1".to_string(), } } @@ -395,6 +420,7 @@ impl Meta for ScaleKind { ScaleKind::StatefulSet(_) => StatefulSet::KIND.to_string(), ScaleKind::Notebook(_) => "Notebook".to_string(), ScaleKind::InferenceService(_) => "InferenceService".to_string(), + ScaleKind::LeaderWorkerSet(_) => "LeaderWorkerSet".to_string(), } } @@ -441,6 +467,11 @@ impl Scaler for ScaleKind { Api::namespaced(client.clone(), &d.namespace().expect("No namespace!")); scale_to_zero(api, &d.name_unchecked()).await } + ScaleKind::LeaderWorkerSet(d) => { + let api: Api = + Api::namespaced(client.clone(), &d.namespace().expect("No namespace!")); + scale_to_zero(api, &d.name_unchecked()).await + } ScaleKind::Notebook(d) => { scale_notebook_to_zero( client.clone(), @@ -559,6 +590,7 @@ fn workload_annotations( ScaleKind::StatefulSet(s) => s.metadata.annotations.clone(), ScaleKind::Notebook(n) => n.metadata.annotations.clone(), ScaleKind::InferenceService(i) => i.metadata.annotations.clone(), + ScaleKind::LeaderWorkerSet(l) => l.metadata.annotations.clone(), } } @@ -695,6 +727,10 @@ pub async fn fetch_workload( let api: Api = Api::namespaced(client, namespace); Ok(ScaleKind::StatefulSet(api.get(name).await?)) } + "LeaderWorkerSet" => { + let api: Api = Api::namespaced(client, namespace); + Ok(ScaleKind::LeaderWorkerSet(api.get(name).await?)) + } "Notebook" => { let api: Api = Api::namespaced(client, namespace); Ok(ScaleKind::Notebook(api.get(name).await?)) @@ -731,6 +767,11 @@ async fn patch_workload( api.patch(name, &PatchParams::default(), &Patch::Merge(patch)) .await?; } + "LeaderWorkerSet" => { + let api: Api = Api::namespaced(client, namespace); + api.patch(name, &PatchParams::default(), &Patch::Merge(patch)) + .await?; + } "Notebook" => { let api: Api = Api::namespaced(client, namespace); api.patch(name, &PatchParams::default(), &Patch::Merge(patch)) @@ -762,6 +803,7 @@ pub async fn check_acknowledgment( ScaleKind::StatefulSet(s) => s.metadata.annotations.clone(), ScaleKind::Notebook(n) => n.metadata.annotations.clone(), ScaleKind::InferenceService(i) => i.metadata.annotations.clone(), + ScaleKind::LeaderWorkerSet(l) => l.metadata.annotations.clone(), }; let annotations = match annotations { @@ -877,6 +919,13 @@ pub async fn find_root_object( return Ok(ScaleKind::StatefulSet(ss)); } } + "LeaderWorkerSet" => { + tracing::info!("Found LeaderWorkerSet!"); + let lws_api: Api = Api::namespaced(client.clone(), &namespace); + if let Ok(lws) = lws_api.get(&or.name).await { + return Ok(ScaleKind::LeaderWorkerSet(lws)); + } + } "DaemonSet" | "Node" => { return Err(RootObjectError::NonScalable { pod: pod_meta.name.clone().unwrap_or_default(), @@ -964,7 +1013,11 @@ mod tests { use k8s_openapi::api::apps::v1::{Deployment, ReplicaSet, StatefulSet}; use kube::api::ObjectMeta; - use resources::{inferenceservice::InferenceService, notebook::NotebookSpec}; + use resources::{ + inferenceservice::InferenceService, + leaderworkerset::{LeaderWorkerSet, LeaderWorkerSetSpec, LeaderWorkerSetStatus}, + notebook::NotebookSpec, + }; use crate::{Meta, Notebook, ResourceKind, ScaleKind, Scaler, get_enabled_resources}; @@ -1034,6 +1087,25 @@ mod tests { ScaleKind::InferenceService(Box::new(is)) } + fn make_leader_worker_set(name: &str, ns: &str, uid: Option<&str>) -> ScaleKind { + ScaleKind::LeaderWorkerSet(LeaderWorkerSet { + metadata: ObjectMeta { + name: Some(name.into()), + namespace: Some(ns.into()), + uid: uid.map(Into::into), + ..Default::default() + }, + spec: LeaderWorkerSetSpec { + replicas: Some(1), + leader_worker_template: None, + }, + status: Some(LeaderWorkerSetStatus { + replicas: Some(1), + ready_replicas: Some(1), + }), + }) + } + // ── get_enabled_resources ──────────────────────────────────────────── #[test] @@ -1044,6 +1116,7 @@ mod tests { assert!(rk.contains(ResourceKind::STATEFUL_SET)); assert!(rk.contains(ResourceKind::INFERENCE_SERVICE)); assert!(rk.contains(ResourceKind::NOTEBOOK)); + assert!(rk.contains(ResourceKind::LEADER_WORKER_SET)); // Auto-enabled } #[test] @@ -1054,6 +1127,7 @@ mod tests { assert!(!rk.contains(ResourceKind::REPLICA_SET)); assert!(!rk.contains(ResourceKind::STATEFUL_SET)); assert!(!rk.contains(ResourceKind::INFERENCE_SERVICE)); + assert!(rk.contains(ResourceKind::LEADER_WORKER_SET)); // Auto-enabled } #[test] @@ -1069,7 +1143,7 @@ mod tests { #[test] fn enabled_resources_empty_string() { let rk = get_enabled_resources(""); - assert!(rk.is_empty()); + assert_eq!(rk, ResourceKind::all()); } #[test] @@ -1378,4 +1452,63 @@ mod tests { assert!(enabled.contains(nb)); assert!(!enabled.contains(ss)); } + + // ── LeaderWorkerSet specific tests ──────────────────────────────────── + + #[test] + fn leader_worker_set_auto_enabled() { + let rk = get_enabled_resources("drsin"); + assert!(rk.contains(ResourceKind::LEADER_WORKER_SET)); + assert!(rk.contains(ResourceKind::DEPLOYMENT)); + } + + #[test] + fn leader_worker_set_enabled_when_empty() { + let rk = get_enabled_resources(""); + assert!(rk.contains(ResourceKind::LEADER_WORKER_SET)); + } + + #[test] + fn scale_kind_to_resource_kind_leader_worker_set() { + let rk: ResourceKind = make_leader_worker_set("lws", "ns", None).into(); + assert_eq!(rk, ResourceKind::LEADER_WORKER_SET); + } + + #[test] + fn leader_worker_set_equality_uses_uid() { + let a = make_leader_worker_set("lws-a", "ns", Some("uid-x")); + let b = make_leader_worker_set("lws-b", "ns", Some("uid-x")); + assert_eq!(a, b); + } + + #[test] + fn hashset_deduplicates_leader_worker_sets_by_uid() { + let mut set = HashSet::new(); + set.insert(make_leader_worker_set("lws-1", "ns", Some("uid-lws"))); + set.insert(make_leader_worker_set("lws-2", "ns", Some("uid-lws"))); + assert_eq!(set.len(), 1); + } + + #[test] + fn meta_leader_worker_set() { + let sk = make_leader_worker_set("my-lws", "training", Some("lws-uid")); + assert_eq!(sk.name(), "my-lws"); + assert_eq!(sk.namespace(), Some("training".into())); + assert_eq!(sk.kind(), "LeaderWorkerSet"); + assert_eq!(sk.uid(), Some("lws-uid".into())); + assert_eq!(sk.api_version(), "v1"); + } + + #[test] + fn event_for_leader_worker_set() { + let sk = make_leader_worker_set("my-lws", "ml", Some("lws-uid")); + let event = sk.generate_scale_event().unwrap(); + + assert_eq!(event.involved_object.kind, Some("LeaderWorkerSet".into())); + assert_eq!(event.involved_object.api_version, Some("v1".into())); + assert_eq!( + event.reason, + Some("Pod ml::my-lws was not using GPU".into()) + ); + } } diff --git a/gpu-pruner/src/main.rs b/gpu-pruner/src/main.rs index 66e46bd..1dbdbf8 100644 --- a/gpu-pruner/src/main.rs +++ b/gpu-pruner/src/main.rs @@ -76,7 +76,9 @@ struct Cli { /// - `s` for StatefulSet /// - `i` for InferenceService /// - `n` for Notebook - #[clap(short, long, default_value = "drsin")] + /// + /// Note: LeaderWorkerSet is automatically enabled with any resource combination + #[clap(short, long, default_value = "drsinl")] enabled_resources: String, /// interval in seconds to check for idle pods, only used in daemon mode @@ -556,8 +558,8 @@ async fn main() -> anyhow::Result<()> { .slack_namespace_mentions .clone() .or_else(|| std::env::var("SLACK_NAMESPACE_MENTIONS").ok()) - .and_then(|json_str| { - match gpu_pruner::NamespaceMentionMapper::from_json(&json_str) { + .and_then( + |json_str| match gpu_pruner::NamespaceMentionMapper::from_json(&json_str) { Ok(mapper) => { tracing::info!( "Namespace mention mapping enabled with {} entries", @@ -569,11 +571,13 @@ async fn main() -> anyhow::Result<()> { tracing::error!("Failed to parse namespace mention mapping JSON: {}", e); None } - } - }); + }, + ); if namespace_mention_mapper.is_none() && slack_notifier.is_some() { - tracing::info!("Namespace mention mapping not configured - will only use annotation-based mentions"); + tracing::info!( + "Namespace mention mapping not configured - will only use annotation-based mentions" + ); } let env: Environment = Environment::new(); diff --git a/resources/src/leaderworkerset.rs b/resources/src/leaderworkerset.rs new file mode 100644 index 0000000..3aef39d --- /dev/null +++ b/resources/src/leaderworkerset.rs @@ -0,0 +1,46 @@ +// Minimal LeaderWorkerSet CRD definition for gpu-pruner +// We only need basic structure since we use the /scale subresource + +#[allow(unused_imports)] +mod prelude { + pub use kube::CustomResource; + pub use schemars::JsonSchema; + pub use serde::{Deserialize, Serialize}; +} +use self::prelude::*; + +#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, JsonSchema)] +#[kube( + group = "leaderworkerset.x-k8s.io", + version = "v1", + kind = "LeaderWorkerSet", + plural = "leaderworkersets" +)] +#[kube(namespaced)] +#[kube(status = "LeaderWorkerSetStatus")] +pub struct LeaderWorkerSetSpec { + /// Number of replicas (leader-worker groups) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replicas: Option, + + /// Size of each worker group + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "leaderWorkerTemplate" + )] + pub leader_worker_template: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct LeaderWorkerSetStatus { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replicas: Option, + + #[serde( + default, + skip_serializing_if = "Option::is_none", + rename = "readyReplicas" + )] + pub ready_replicas: Option, +} diff --git a/resources/src/lib.rs b/resources/src/lib.rs index 1f5d540..36b797e 100644 --- a/resources/src/lib.rs +++ b/resources/src/lib.rs @@ -1,2 +1,3 @@ pub mod inferenceservice; +pub mod leaderworkerset; pub mod notebook;