diff --git a/bundles/k3d-kubevirt-dev/uds-bundle.yaml b/bundles/k3d-kubevirt-dev/uds-bundle.yaml new file mode 100644 index 0000000000..c98ff69b6d --- /dev/null +++ b/bundles/k3d-kubevirt-dev/uds-bundle.yaml @@ -0,0 +1,138 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/uds-cli/refs/heads/main/uds.schema.json +kind: UDSBundle +metadata: + name: k3d-core-kubevirt-dev + description: UDS Core slim-dev with KubeVirt — for local development and teammate experimentation + # x-release-please-start-version + version: "1.7.0" + # x-release-please-end + +packages: + - name: uds-k3d-dev + repository: ghcr.io/defenseunicorns/packages/uds-k3d + ref: 0.20.1-airgap + overrides: + uds-dev-stack: + minio: + variables: + - name: buckets + description: "Set Minio Buckets" + path: buckets + - name: svcaccts + description: "Minio Service Accounts" + path: svcaccts + - name: users + description: "Minio Users" + path: users + - name: policies + description: "Minio policies" + path: policies + + - name: init + repository: ghcr.io/zarf-dev/packages/init + ref: v0.79.0 + keylessVerification: + certificateIdentityRegexp: https://github\.com/zarf-dev/zarf/\.github/workflows/release\.yml@refs/tags/v\d+\.\d+\.\d+ + certificateOIDCIssuer: https://token.actions.githubusercontent.com + + - name: core-base + path: ../../build/ + # x-release-please-start-version + ref: 1.7.0 + # x-release-please-end + overrides: + pepr-uds-core: + module: + variables: + - name: PEPR_WATCHER_MEMORY_REQUEST + description: "Memory requests for the pepr watcher pod" + path: "watcher.resources.requests.memory" + default: "64Mi" + - name: PEPR_ADMISSION_MEMORY_REQUEST + description: "Memory requests for the pepr admission pods" + path: "admission.resources.requests.memory" + default: "64Mi" + - name: PEPR_WATCHER_CPU_REQUEST + description: "CPU requests for the pepr watcher pod" + path: "watcher.resources.requests.cpu" + default: "100m" + - name: PEPR_ADMISSION_CPU_REQUEST + description: "CPU requests for the pepr admission pods" + path: "admission.resources.requests.cpu" + default: "100m" + istio-controlplane: + istiod: + variables: + - name: ISTIOD_MEMORY_REQUEST + description: "Memory request for Istiod" + path: "resources.requests.memory" + default: "1024Mi" + - name: ISTIOD_CPU_REQUEST + description: "CPU request for Istiod" + path: "resources.requests.cpu" + default: "100m" + - name: PROXY_MEMORY_REQUEST + description: "Memory request for the Istio Proxy Sidecar" + path: "global.proxy.resources.requests.memory" + default: "40Mi" + - name: PROXY_MEMORY_LIMIT + description: "Memory limit for the Istio Proxy Sidecar" + path: "global.proxy.resources.limits.memory" + default: "1024Mi" + - name: PROXY_CPU_REQUEST + description: "CPU request for the Istio Proxy Sidecar" + path: "global.proxy.resources.requests.cpu" + default: "10m" + - name: PROXY_CPU_LIMIT + description: "CPU limit for the Istio Proxy Sidecar" + path: "global.proxy.resources.limits.cpu" + default: "2000m" + istio-admin-gateway: + uds-istio-config: + variables: + - name: ADMIN_TLS_CERT + description: "The TLS cert for the admin gateway (must be base64 encoded)" + path: tls.cert + - name: ADMIN_TLS_KEY + description: "The TLS key for the admin gateway (must be base64 encoded)" + path: tls.key + istio-tenant-gateway: + uds-istio-config: + variables: + - name: TENANT_TLS_CERT + description: "The TLS cert for the tenant gateway (must be base64 encoded)" + path: tls.cert + - name: TENANT_TLS_KEY + description: "The TLS key for the tenant gateway (must be base64 encoded)" + path: tls.key + + - name: core-identity-authorization + path: ../../build/ + # x-release-please-start-version + ref: 1.7.0 + # x-release-please-end + overrides: + keycloak: + keycloak: + variables: + - name: INSECURE_ADMIN_PASSWORD_GENERATION + description: "Generate an insecure admin password for dev/test" + path: insecureAdminPasswordGeneration.enabled + default: "true" + + - name: core-kubevirt + path: ../../build/ + # x-release-please-start-version + ref: 1.7.0 + # x-release-please-end + overrides: + kubevirt: + uds-kubevirt-config: + variables: + - name: KUBEVIRT_USE_EMULATION + description: "Enable QEMU emulation for nodes without KVM (e.g. nested virt in CI)" + path: kubevirt.configuration.developerConfiguration.useEmulation + default: "false" diff --git a/docs/concepts/core-features/virtual-machines.mdx b/docs/concepts/core-features/virtual-machines.mdx new file mode 100644 index 0000000000..545b020da7 --- /dev/null +++ b/docs/concepts/core-features/virtual-machines.mdx @@ -0,0 +1,240 @@ +--- +title: KubeVirt on UDS Core +description: Understand how KubeVirt and CDI integrate with UDS Core's service mesh, policy enforcement, and observability stack before deploying VM workloads. +sidebar: + order: 2.009 +--- + +KubeVirt extends Kubernetes with native virtual machine support. On UDS Core, VMs run alongside +containers and participate in the same routing, policy, and observability stack, but the +integration has specific rules you need to understand before deploying VM workloads. + +## How KubeVirt works + +KubeVirt installs as an operator and a set of control-plane components in the `kubevirt` namespace: + +- **`virt-operator`**: Manages the KubeVirt installation lifecycle. Reads the `KubeVirt/kubevirt` CR + and reconciles the other control-plane components. +- **`virt-api`**: Serves the KubeVirt API (`subresources.kubevirt.io`). Handles `VirtualMachine` + and `VirtualMachineInstance` admission webhooks and exposes subresources like console, VNC, and + SSH proxying. +- **`virt-controller`**: Watches `VirtualMachine` and `VirtualMachineInstance` resources and + creates the `virt-launcher` pod for each running VM. +- **`virt-handler`**: A DaemonSet that runs on every node. Handles the on-node VM lifecycle: + starting `libvirt` domains, managing network interfaces, and triggering live migrations. + +When you create a `VirtualMachine` and set it to running, `virt-controller` creates a +`virt-launcher` pod in your VM workload namespace. That pod is the Kubernetes workload object that +represents your VM. Everything that Kubernetes and UDS Core care about (network policy, service +routing, metrics, logs, sidecar injection) applies to the `virt-launcher` pod, not to the VM +directly. + +## How CDI works + +CDI (Containerized Data Importer) adds disk import and management capabilities alongside KubeVirt. +It runs in the `cdi` namespace and provides the `DataVolume` custom resource. When you create a +`DataVolume`, CDI creates a temporary `importer-*` pod in your workload namespace to fetch the +disk image from the source (HTTP, S3, registry, or upload) and write it to a PersistentVolumeClaim. + +CDI components: + +- **`cdi-operator`**: Manages the CDI installation lifecycle. +- **`cdi-controller`**: Watches `DataVolume` resources and creates importer, cloner, and uploader + pods as needed. +- **`cdi-apiserver`**: Serves the CDI API and handles admission webhooks. +- **`cdi-uploadproxy`**: Accepts disk image uploads from `virtctl image-upload`. + +## Mesh mode rules + +KubeVirt has a specific constraint with Istio: the `virt-launcher` pod requires sidecar mode, not +ambient mode. + +**`kubevirt` and `cdi` namespaces use ambient mode.** The control-plane components (operators, +controllers, webhooks) run in ambient mode. They communicate through ztunnel HBONE tunneling and +benefit from ambient mesh mTLS and authorization policy without sidecar overhead. + +**VM workload namespaces must use sidecar mode.** The `virt-launcher` pod has a sidecar injected +and participates in the mesh as a standard sidecar workload. This is what makes Service routing, +gateway exposure, and monitoring work for VMs the same way they work for containers. + +**VM workload namespaces must NOT use ambient mode.** If a namespace where `virt-launcher` pods +run has the `istio.io/dataplane-mode=ambient` label, the following break: + +- `virtctl port-forward` and everything that depends on it (`virtctl ssh`, `virtctl scp`) +- `virtctl console` WebSocket connections +- Any kube-apiserver-proxied subresource connection to a `virt-launcher` pod + +The root cause is that ztunnel intercepts inbound traffic on ambient namespace pods and expects +HBONE tunneling. The kube-apiserver proxy that backs these `virtctl` commands does not speak HBONE, +so the connection fails with `unexpected EOF`. + +## Pepr policy allowances + +UDS Core's Pepr module enforces admission policies on all pods. Two policies that apply to +normal pods have scoped exceptions for KubeVirt and CDI-generated pods. + +### virt-launcher pods + +KubeVirt sets `traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0` on every `virt-launcher` +pod. This annotation tells the Istio sidecar how to handle the `k6t-eth0` virtual network +interface that bridges the pod network into the VM guest. Without it, Istio's traffic interception +breaks VM networking. + +UDS Core blocks this annotation by default because it modifies Istio traffic interception behavior. +The native KubeVirt support adds a scoped exception. The annotation is allowed only when all three +hold: the namespace carries `uds.dev/kubevirt-workload: "true"`, the pod name starts with +`virt-launcher-`, and the pod carries the `kubevirt.io=virt-launcher` label. KubeVirt sets the pod +name and label automatically. The namespace label is the trust anchor, because pod names and +labels are tenant-settable. + +### CDI pods + +CDI creates three types of ephemeral pods in your workload namespace, and all three set +`sidecar.istio.io/inject: "false"`: + +- `importer-*`: created for DataVolume imports (HTTP, registry, S3 sources) +- `cdi-upload-*`: created as the upload server target for `virtctl image-upload` +- `cdi-clone-*`: created for PVC-to-PVC cloning operations + +UDS Core blocks `sidecar.istio.io/inject: "false"` by default. The native CDI support adds a +scoped exception: injection disable is allowed only in a namespace labeled +`uds.dev/kubevirt-workload: "true"`, and only on pods whose name starts with `importer-`, +`cdi-upload-`, or `cdi-clone-`. + +> [!NOTE] +> Both sets of exceptions require the namespace label `uds.dev/kubevirt-workload: "true"` in +> addition to the KubeVirt/CDI pod names and labels. The namespace label is the security +> boundary: only the platform can label a namespace, while pod names and labels are +> tenant-settable. This behavior requires UDS Core with the native KubeVirt/CDI Pepr support +> included in this package. + +### The kubevirt-workload namespace label + +Every VM workload namespace must carry `uds.dev/kubevirt-workload: "true"`. The label scopes both +exceptions above to that namespace. Pods in unlabeled namespaces are denied the KubeVirt and CDI +annotations even when their names and labels match. + +Set `kubevirt.enabled: true` in the namespace's UDS Package and the UDS operator applies the label +for you, so you never label the namespace by hand: + +```yaml title="vm-app-package.yaml" +apiVersion: uds.dev/v1alpha1 +kind: Package +metadata: + name: my-vm-app + namespace: my-vm-app +spec: + kubevirt: + enabled: true + network: + serviceMesh: + mode: sidecar +``` + +The label is the security boundary, so only the operator or an admin can set it; a tenant pod +cannot. Label the namespace directly when it has no UDS Package, or when generated pods start +during deploy before the operator reconciles the Package: CDI importer pods, or a VM launcher when +the VM auto-starts with `runStrategy: Always`. The operator reconciles the Package asynchronously, +so a pod that races ahead of it is denied. Setting the label on the namespace removes that race. + +## The VM app contract + +A VM workload namespace must satisfy these requirements before VMs can run on UDS Core: + +- The namespace must carry the `uds.dev/kubevirt-workload: "true"` label so the KubeVirt and CDI + Pepr allowances apply to its generated pods. Set `kubevirt.enabled: true` in the UDS Package to + have the operator apply it, or label the namespace directly. +- The namespace must have a `Package` CR with `serviceMesh.mode: sidecar`. +- The namespace must have Istio sidecar injection enabled (`istio-injection: enabled` label or + equivalent). This is handled automatically when a UDS Package with `mode: sidecar` reconciles. +- `VirtualMachine` templates must have `sidecar.istio.io/inject: "true"` set explicitly in + `spec.template.metadata.annotations`. The UDS Package `mode: sidecar` alone does not force + injection on individual pods; the annotation makes it explicit. +- `VirtualMachine` templates must have `traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0` + set in `spec.template.metadata.annotations`. This is required for VM networking to work through + the Istio sidecar. +- The `private-registry` image pull secret must be present in the namespace and attached to the + service account, so that Istio sidecar container images can be pulled from the Zarf registry. + Zarf-managed namespaces get this automatically. +- VM interface ports that need Service, gateway, monitor, or readiness behavior must be declared + in `spec.template.spec.domain.devices.interfaces[].ports`. +- The declared interface port, the Kubernetes `Service` `targetPort`, the UDS `expose` `targetPort`, + and the UDS `monitor` `targetPort` must all match. + +> [!WARNING] +> Do not set `traffic.sidecar.istio.io/excludeInboundPorts` on `virt-launcher` pods. This breaks +> the gateway mTLS path. Do not use HTTP readiness probes with a custom `host` value; these +> interact badly with Istio's probe rewrite mechanism. Use TCP socket probes on declared ports +> instead. + +## Monitoring model + +KubeVirt and CDI expose Prometheus metrics on port `8443` in their control-plane namespaces. The +KubeVirt package installs `ServiceMonitor` resources for both. These are scraped by the UDS Core +Prometheus instance automatically once the package is deployed. + +For VM applications, monitoring works through the standard UDS Package `monitor` configuration. +The `virt-launcher` pod is the scrape target. Your VM application must expose a Prometheus metrics +endpoint on a declared interface port, and your UDS Package must include a `monitor` entry +pointing to that port. The metric endpoint runs inside the VM guest; the Istio sidecar on the +`virt-launcher` pod proxies the scrape request in. This is identical to the pattern for +containerized applications. + +For implementation details and a worked example, see +[Expose a VM application on the UDS mesh](/how-to-guides/virtual-machines/vm-app-on-uds-mesh/). + +## Logging model + +UDS Core's Vector agent collects logs from container stdout and stderr. The `virt-launcher` pod's +stdout is QEMU and libvirt process output (VM lifecycle events: boot, shutdown, migration, +errors), not your VM application's logs. Vector collects these automatically. + +To collect VM application logs, you need a log agent running inside the guest that ships logs to +the Loki endpoint on UDS Core. The approach depends on the guest OS, and your UDS Package must +include an egress rule to allow the outbound connection from the `virt-launcher` pod's namespace +to Loki. + +For implementation details, see +[Windows Server VMs on UDS Core](/how-to-guides/virtual-machines/windows-server-vm/). + +## Storage requirements + +Most KubeVirt features work with any StorageClass. Some require specific access modes: + +- **Container disk VMs**: No PVC required. The VM image is a container image pulled by + `virt-handler`. No StorageClass needed. +- **PVC-backed VMs**: Require a PVC. Any StorageClass that supports `ReadWriteOnce` works. + Local-path provisioner is sufficient. +- **CDI DataVolume imports**: Require a default StorageClass or an explicit + `scratchSpaceStorageClass` set in CDIConfig. Without scratch space, registry-source imports fail. +- **Live migration with persistent disks**: Requires a `ReadWriteMany` StorageClass. Local-path + provisioner is `ReadWriteOnce` only. Container disk VMs can migrate without RWX because each node + pulls the container image fresh. +- **VM snapshots**: Require a `VolumeSnapshotClass` configured in the cluster. + +## virt-api webhook routing + +KubeVirt's admission webhooks (in `virt-api`) use `failurePolicy: Fail`. The webhooks must be +reachable from kube-apiserver for any `VirtualMachine` create, update, or delete to succeed. + +KubeVirt v1.8.2 does not pin `virt-api` to control-plane nodes. It sets a `podAntiAffinity` +preference that spreads replicas across nodes, but places no node affinity requiring control-plane +placement. + +kube-apiserver is not mesh-enrolled and cannot present an Istio identity, so its webhook calls +reach `virt-api` as plaintext. The `virt-api` PeerAuthentication sets port `8443` to PERMISSIVE, +which is what allows those calls to succeed under global STRICT mTLS. + +> [!NOTE] +> On multi-node clusters, kube-apiserver webhook calls to a `virt-api` pod on a different node have +> been observed to fail on k3d: the cross-node delivery path through the destination pod's ztunnel +> does not complete. Single-node clusters keep the call on one node, so it succeeds. If you hit +> this, pin `virt-api` to the control-plane node with a node affinity rule in the KubeVirt CR, or +> cordon worker nodes during VM admission. + +## Related documentation + +- [KubeVirt upstream documentation](https://kubevirt.io/user-guide/) - KubeVirt user guide +- [CDI upstream documentation](https://github.com/kubevirt/containerized-data-importer/blob/main/doc/) - CDI documentation +- [Install KubeVirt on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) - Install and configure the KubeVirt package +- [Windows Server VMs on UDS Core](/how-to-guides/virtual-machines/windows-server-vm/) - Monitoring and logging implementation for VM workloads diff --git a/docs/how-to-guides/virtual-machines/cross-cluster-disk-movement.mdx b/docs/how-to-guides/virtual-machines/cross-cluster-disk-movement.mdx new file mode 100644 index 0000000000..f549f1d164 --- /dev/null +++ b/docs/how-to-guides/virtual-machines/cross-cluster-disk-movement.mdx @@ -0,0 +1,403 @@ +--- +title: Move a disk image between clusters +description: Export a PVC from a source cluster using the VMExport API, download it locally, and upload it to a destination cluster with virtctl image-upload. +sidebar: + order: 11.008 +--- + +import { Steps } from '@astrojs/starlight/components'; + +## What you'll accomplish + +Export a PVC from a source cluster using the KubeVirt VMExport API, download the disk image to +your workstation, and upload it to a PVC on a destination cluster using `virtctl image-upload`. +The resulting PVC is ready to use as a VM disk. + +## Prerequisites + +- [KubeVirt installed on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) on both source and destination clusters +- [A PVC to export](/how-to-guides/virtual-machines/upload-disk-image/) - A `Bound` PVC in a VM namespace on the source cluster +- `kubectl` and `virtctl` on your PATH, with kubeconfig access to both clusters +- Enough local disk space to hold the exported image + +## Before you begin + +> [!NOTE] +> This guide was verified end-to-end on 2026-06-29: a CirrOS disk was exported from a source +> cluster, downloaded as `disk.img.gz`, uploaded to a second independent k3d cluster running +> a second independent k3d cluster, and a VM booted from the imported disk +> successfully. On single-node k3d clusters, `virtctl image-upload` may fail due to a ztunnel +> same-node routing issue; see troubleshooting if you hit this. + +VMExport creates an ephemeral export server pod (`virt-export-`) in the same namespace +as the source PVC. The pod serves the disk over HTTPS with token authentication. The export +has a 2-hour TTL; after that KubeVirt removes the pod. Re-create the VMExport CR if you need +more time. + +The export server exposes each PVC in two formats: + +- `raw`: an uncompressed raw disk image (`disk.img`) +- `gz`: a gzip-compressed raw disk image (`disk.img.gz`) + +This guide uses the `gz` format. `virtctl image-upload` accepts both raw and gzip-compressed +images and handles decompression automatically. + +## Managing cluster contexts + +This guide requires running commands against two clusters. Before starting, confirm both +contexts are available: + +```bash +kubectl config get-contexts +``` + +Switch between clusters with `use-context`: + +```bash +kubectl config use-context source-cluster +kubectl config use-context dest-cluster +``` + +Steps 1-6 run against the source cluster. Step 7 switches to the destination cluster. +Set your context to the source cluster before beginning step 1: + +```bash +kubectl config use-context source-cluster +``` + +> [!IMPORTANT] +> The VMExport feature gate must be enabled before creating VMExport resources. This also +> deploys `virt-exportproxy` in the `kubevirt` namespace, which requires a KubeAPI egress rule +> in the KubeVirt UDS Package. Step 1 covers enabling the feature gate. The egress rule ships +> with the KubeVirt package; see troubleshooting if `virt-exportproxy` cannot reach the API. + +## Steps + + + +1. **Verify VMExport is active** + + The KubeVirt package ships with the `VMExport` feature gate enabled by default in + common-flavor deployments (`common-values.yaml`). Upstream-flavor bundles that do not + include `common-values.yaml` do not enable it. Confirm `virt-exportproxy` is running: + + ```bash + kubectl get pods -n kubevirt | grep exportproxy + ``` + + Expected: one or more pods showing `1/1 Running`. + + If `virt-exportproxy` is missing, the deployed package version predates the default + feature gate. Enable it explicitly in your bundle: + + ```yaml title="uds-bundle.yaml" + packages: + - name: kubevirt + repository: ghcr.io/defenseunicorns/packages/uds/kubevirt + ref: 1.8.2-upstream + overrides: + kubevirt: + uds-kubevirt-config: + values: + - path: kubevirt.configuration.developerConfiguration.featureGates + value: + - VMExport + ``` + + > [!WARNING] + > The `value` list **replaces** the entire `featureGates` array. Include any other feature + > gates your cluster uses, not just VMExport, or they will be removed on redeploy. + +2. **Create the export token secret** + + Choose a token value and create a secret in the namespace that holds the source PVC. + Replace `my-export-token` with a value of your choice: + + ```bash + EXPORT_TOKEN=my-export-token + kubectl create secret generic export-token \ + -n my-vms \ + --from-literal=token=$EXPORT_TOKEN + ``` + + Note the token value. You will need it in step 5 when downloading. + +3. **Create the VMExport CR** + + Create a VMExport that references the source PVC and the token secret: + + ```yaml title="disk-export.yaml" + apiVersion: export.kubevirt.io/v1beta1 + kind: VirtualMachineExport + metadata: + name: disk-export + namespace: my-vms + spec: + tokenSecretRef: export-token # secret from step 2 + source: + apiGroup: "" + kind: PersistentVolumeClaim + name: my-disk # PVC to export + ``` + + ```bash + kubectl apply -f disk-export.yaml + ``` + +4. **Wait for the export to become Ready** + + Watch the VMExport phase: + + ```bash + kubectl get virtualmachineexport disk-export -n my-vms -w + ``` + + The export moves through `Pending` → `Ready`. When `Ready`, the export server pod + is running 2/2 in your namespace. + + ```console + NAME PHASE AGE + disk-export Pending 5s + disk-export Ready 20s + ``` + +5. **Download the disk image** + + The simplest download method is `virtctl vmexport download`, which handles port-forwarding + and token authentication automatically: + + ```bash + virtctl vmexport download disk-export \ + --volume=my-disk \ + --output=my-disk.img.gz \ + --port-forward \ + -n my-vms + ``` + + Expected output: + + ```console + service virt-export-disk-export is ready for port-forwarding + Forwarding from 127.0.0.1: -> 8443 + Port forwarding is ready. + Downloading file: ... Download finished successfully + ``` + + Alternatively, use curl with a manual port-forward: + + ```bash + kubectl port-forward svc/virt-export-disk-export 18444:443 -n my-vms & + PF_PID=$! + ``` + + ```bash + curl -sk \ + -H "x-kubevirt-export-token: $EXPORT_TOKEN" \ + -o my-disk.img.gz \ + "https://localhost:18444/volumes/my-disk/disk.img.gz" + ``` + + ```bash + kill $PF_PID + ``` + + The volume URL path is `/volumes//disk.img.gz`. Confirm the PVC name matches + the name you used in the VMExport spec. + +6. **Delete the VMExport to release the export server pod** + + ```bash + kubectl delete virtualmachineexport disk-export -n my-vms + ``` + + KubeVirt removes the export server pod automatically. Deleting the VMExport does not affect + the source PVC. If you did not finish downloading before the 2-hour TTL expires, the export + pod disappears on its own. Re-create the VMExport CR from step 3 to start a new export. + +7. **Upload the image to the destination cluster** + + Switch to the destination cluster context: + + ```bash + kubectl config use-context + ``` + + Follow [Upload a local disk image with virtctl](/how-to-guides/virtual-machines/upload-disk-image/) + to upload `my-disk.img.gz` to a PVC in the destination cluster. The upload guide covers the + UDS Package network rule, PVC creation, port-forward, and `virtctl image-upload` command. + + The only difference from the upload guide: your image path is the `.img.gz` file from step 5. + `virtctl image-upload` accepts gzip-compressed images and handles decompression automatically. + + ```bash + virtctl image-upload pvc my-disk \ + --no-create \ + --uploadproxy-url=https://localhost:18443 \ + --insecure \ + --image-path=./my-disk.img.gz \ + -n my-vms + ``` + + > [!NOTE] + > On single-node clusters (including single-node k3d), `virtctl image-upload` fails with + > `http: proxy error: EOF` because the CDI uploadproxy and upload server pods co-schedule on + > one node and hit a ztunnel same-node routing issue. Use the DataVolume HTTP import in + > troubleshooting instead. Multi-node clusters are not affected. + + + +## Verification + +On the source cluster, confirm the export server pod was cleaned up: + +```bash +kubectl get pods -n my-vms | grep virt-export +``` + +No `virt-export-disk-export` pod should be present after step 6. + +On the destination cluster, confirm the PVC is bound: + +```bash +kubectl get pvc my-disk -n my-vms +``` + +Expected status: `Bound`. + +## Troubleshooting + +### Problem: Export server pod stuck in ImagePullBackOff + +**Symptom:** `kubectl get pods -n my-vms` shows `virt-export-` in `ErrImagePull` or +`ImagePullBackOff` pulling `127.0.0.1:31999/kubevirt/virt-exportserver:-*` (address shown is k3d-specific; yours will differ on other cluster types). + +**Solution:** Both `virt-exportproxy` and `virt-exportserver` images are bundled in the +KubeVirt package (`zarf.yaml`). If the pod cannot pull the image, the Zarf registry may not +have it loaded. Confirm the images are present: + +```bash +kubectl port-forward svc/zarf-docker-registry 15000:5000 -n zarf & +PF_PID=$! +sleep 2 +PASS=$(kubectl get secret private-registry -n zarf \ + -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d \ + | python3 -c " +import sys, json, base64 +d = json.load(sys.stdin) +auth = list(d['auths'].values())[0]['auth'] +print(base64.b64decode(auth).decode().split(':')[1]) +") +curl -s -u "zarf-pull:$PASS" "http://localhost:15000/v2/_catalog" \ + | python3 -m json.tool | grep -E "exportserver|exportproxy" +kill $PF_PID +``` + +If the images are absent despite being in the bundle, the Zarf deploy may have failed to push +them. Re-run `uds deploy` or check the deploy logs for push errors. + +### Problem: virt-exportproxy keeps failing to reach the Kubernetes API + +**Symptom:** `kubectl logs -n kubevirt -l kubevirt.io=virt-exportproxy` shows repeated +`TLS handshake timeout` or `EOF` errors against the cluster API IP. + +**Solution:** The KubeVirt package ships with a KubeAPI egress rule for `virt-exportproxy` +in `chart/templates/uds-package.yaml`. If the rule is missing, you are running a package +version older than this fix. Upgrade to the current package, or patch the Package CR +temporarily: + +```bash +kubectl patch package kubevirt -n kubevirt --type=json -p='[ + { + "op": "add", + "path": "/spec/network/allow/-", + "value": { + "direction": "Egress", + "selector": {"kubevirt.io": "virt-exportproxy"}, + "remoteGenerated": "KubeAPI", + "description": "virt-exportproxy to KubeAPI" + } + } +]' +``` + +This patch is not persistent across Package CR reconciliation. Upgrade the package for a +permanent fix. + +### Problem: VMExport stays in Pending + +**Symptom:** `kubectl get virtualmachineexport -n my-vms` shows `Pending` for more than 2 +minutes with no export server pod created. + +**Solution:** Check the events on the VMExport: + +```bash +kubectl describe virtualmachineexport disk-export -n my-vms +``` + +Common causes: the source PVC is not `Bound`, the `tokenSecretRef` secret does not exist in +the same namespace, or `virt-controller` cannot reconcile the resource. Confirm the PVC status: + +```bash +kubectl get pvc my-disk -n my-vms +``` + +### Problem: curl download returns empty or 401 + +**Symptom:** The download completes but the file is empty, or curl returns an HTTP 401 error. + +**Solution:** The token header is missing or wrong. The header name is exactly +`x-kubevirt-export-token` and the value must match the `token` key in the secret created in +step 2. Verify the secret: + +```bash +kubectl get secret export-token -n my-vms \ + -o jsonpath='{.data.token}' | base64 -d +``` + +### Problem: virtctl image-upload fails with `http: proxy error: EOF` on the destination cluster + +**Symptom:** `virtctl image-upload` exits with `http: proxy error: EOF` or `connection refused` +in CDI uploadproxy logs when uploading to the destination cluster. + +**Solution:** On single-node clusters (including k3d), the CDI uploadproxy pod and the CDI +upload server pod both schedule on the same node. Ztunnel's same-node routing path returns +`ECONNREFUSED` for CDI's internal mTLS connection between the two pods. This is a k3d-specific +limitation; on multi-node clusters the pods schedule on different nodes and cross-node traffic +works correctly. + +**Workaround:** Import the disk directly on the destination cluster using a DataVolume with an +HTTP source. Serve `my-disk.img.gz` from your workstation: + +```bash +python3 -m http.server 8888 +``` + +Then create a DataVolume on the destination cluster pointing at your workstation IP: + +```yaml title="workaround-dv.yaml" +apiVersion: cdi.kubevirt.io/v1beta1 +kind: DataVolume +metadata: + name: my-disk + namespace: my-vms +spec: + source: + http: + url: "http://:8888/my-disk.img.gz" + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +``` + +The CDI importer handles `.img.gz` decompression automatically. Add an egress rule to your +destination UDS Package for port 8888 so the importer pod can reach your workstation. + +## Related documentation + +- [Upload a local disk image with virtctl](/how-to-guides/virtual-machines/upload-disk-image/) - Upload the downloaded image to the destination cluster +- [Import a disk image with CDI](/how-to-guides/virtual-machines/pvc-backed-vms-with-cdi/) - Alternative import methods: Zarf registry and HTTP sources +- [KubeVirt on UDS Core](/concepts/core-features/virtual-machines/) - Architecture overview +- [KubeVirt VMExport documentation](https://kubevirt.io/user-guide/operations/export_api/) - Full VMExport spec reference diff --git a/docs/how-to-guides/virtual-machines/first-vm.mdx b/docs/how-to-guides/virtual-machines/first-vm.mdx new file mode 100644 index 0000000000..4572747e92 --- /dev/null +++ b/docs/how-to-guides/virtual-machines/first-vm.mdx @@ -0,0 +1,416 @@ +--- +title: Run your first VM on UDS Core +description: Create a container disk VirtualMachine, connect to it via console and SSH, and manage its lifecycle with virtctl start, stop, and pause commands. +sidebar: + order: 11.002 +--- + +import { Steps } from '@astrojs/starlight/components'; + +## What you'll accomplish + +Deploy a CirrOS container disk VM in a UDS-managed namespace, verify that the `virt-launcher` pod +comes up with the Istio sidecar and required KubeVirt annotations, connect to the VM via console, +and exercise the basic lifecycle commands. + +This guide uses CirrOS (a minimal Linux cloud image) as the example VM. CirrOS is included in the +KubeVirt package and does not require a separate image import. + +## Prerequisites + +- [KubeVirt installed on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) - KubeVirt and CDI operators running +- `kubectl` and `virtctl` on your PATH, pointed at the cluster + +## Before you begin + +Container disk VMs use a container image as the disk. KubeVirt pulls the image via containerd on +the node, not via a pod imagePullSecret. For images stored in the cluster's Zarf registry, you +must reference the image by its full Zarf registry URL. + +> [!NOTE] +> The Zarf agent does not rewrite image references in `VirtualMachine` specs. Always use the +> Zarf registry URL directly for container disk images in air-gapped deployments. + +**Finding your Zarf registry address.** On k3d clusters, the Zarf registry is exposed at +`127.0.0.1:31999`. On other cluster types, find the address by checking the Zarf registry +Service in the `zarf` namespace: + +```bash +kubectl get svc zarf-docker-registry -n zarf +``` + +Use the `CLUSTER-IP` and port as your registry address inside the cluster, or the NodePort if +accessing from outside. This guide uses `127.0.0.1:31999` throughout; replace this with your +cluster's Zarf registry address if you are not on k3d. + +Images bundled with the KubeVirt package are available at `/kubevirt/` after +deployment. This guide uses `127.0.0.1:31999/kubevirt/cirros-container-disk-demo:latest`. + +## Steps + + + +1. **Set up the VM workload namespace** + + The namespace where VMs run must: + - Carry the `uds.dev/kubevirt-workload: "true"` label so the KubeVirt Pepr allowances apply. + Set `kubevirt.enabled: true` in the namespace's UDS Package and the operator applies it, or + label the namespace directly (the manual path below does this) + - Have Istio sidecar injection enabled (`istio-injection: enabled`) + - Have a UDS Package CR with `serviceMesh.mode: sidecar` + - Have the `private-registry` imagePullSecret so the Istio sidecar can pull from the Zarf registry + + **(Recommended)** Create the namespace as part of a Zarf package. Zarf creates the + `private-registry` secret and configures the service account automatically. See + [Packaging applications](/how-to-guides/packaging-applications/overview/) for guidance. + + **Or** create the namespace manually for quick testing: + + ```bash + kubectl create namespace my-vms + kubectl label namespace my-vms istio-injection=enabled + kubectl label namespace my-vms uds.dev/kubevirt-workload=true + ``` + + Then copy the `private-registry` secret and patch the default service account: + + ```bash + DOCKER_CONFIG=$(kubectl get secret private-registry -n zarf \ + -o jsonpath='{.data.\.dockerconfigjson}') + + kubectl apply -f - < [!IMPORTANT] + > The `traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0` annotation is required. It tells + > the Istio sidecar not to intercept traffic on the VM's virtual interface `k6t-eth0`. Without + > it, VM networking does not work. + > + > The `sidecar.istio.io/inject: "true"` annotation is also required. The UDS Package `mode: + > sidecar` alone does not force injection; the annotation makes it explicit. + +3. **Wait for the VM to start** + + Watch the `virt-launcher` pod come up: + + ```bash + kubectl get pods -n my-vms -w + ``` + + The pod passes through several init phases before reaching `Running`. When the VM is ready, + you see: + + ```console + NAME READY STATUS RESTARTS AGE + virt-launcher-cirros-vm-xxxxx 4/4 Running 0 30s + ``` + + The `4/4` count includes: + - `compute` (the QEMU/libvirt container) + - `volumecontainerdisk` (serves the container image as a disk) + - `istio-proxy` (Istio sidecar; uses the Kubernetes 1.29 native sidecar mechanism so it counts in the container total and runs for the pod's lifetime) + - `guest-console-log` (captures serial console output, stays running) + + Check the VMI status: + + ```bash + kubectl get vmi cirros-vm -n my-vms + ``` + + Expected output: + + ```console + NAME AGE PHASE IP NODENAME READY + cirros-vm 1m Running 10.42.0.45 node-name True + ``` + +4. **Connect to the console** + + ```bash + virtctl console cirros-vm -n my-vms + ``` + + Press Enter to get a login prompt. Default CirrOS credentials: + - Username: `cirros` + - Password: `gocubsgo` + + To exit the console, press `Ctrl+]`. + +5. **Verify the Istio sidecar and KubeVirt annotations** + + Confirm the virt-launcher pod has the required annotations: + + ```bash + kubectl get pod -n my-vms -l kubevirt.io=virt-launcher -o jsonpath='{.items[0].metadata.annotations}' \ + | python3 -m json.tool | grep -E "kubevirtInterfaces|inject" + ``` + + Expected output includes: + + ```console + "traffic.sidecar.istio.io/kubevirtInterfaces": "k6t-eth0", + "sidecar.istio.io/inject": "true", + "istio.io/reroute-virtual-interfaces": "k6t-eth0" + ``` + + The `istio.io/reroute-virtual-interfaces` annotation is added automatically by Istio based on + the `kubevirtInterfaces` annotation. It confirms Istio is properly routing around the VM's + virtual network interface. + + + +## VM lifecycle commands + +All lifecycle commands target the `VirtualMachine` object, not the `VirtualMachineInstance`. +KubeVirt reconciles the VMI and `virt-launcher` pod in response. + +**Stop a running VM:** + +Sends a graceful shutdown signal. The VMI and pod are terminated; the `VirtualMachine` object persists. + +```bash +virtctl stop cirros-vm -n my-vms +``` + +The VMI reaches phase `Succeeded`, the `virt-launcher` pod status changes to `Completed`, and the +VM status shows `Stopped`. + +**Start a stopped VM:** + +Creates a new VMI and a new `virt-launcher` pod. + +```bash +virtctl start cirros-vm -n my-vms +``` + +The VM gets a new IP address on each start. + +**Pause a running VM:** + +Freezes the QEMU domain in place without stopping the `virt-launcher` pod. + +```bash +virtctl pause vm cirros-vm -n my-vms +``` + +The VM status changes to `Paused`. The VM's IP and pod are preserved. + +**Unpause:** + +Resumes the frozen QEMU domain. + +```bash +virtctl unpause vm cirros-vm -n my-vms +``` + +**Restart a running VM:** + +Replaces the VMI with a new one without stopping the `VirtualMachine`. + +```bash +virtctl restart cirros-vm -n my-vms +``` + +The pod is re-created with a new name and the VM gets a new IP address. + +**Delete the VM entirely:** + +Removes the `VirtualMachine` object and any running `VirtualMachineInstance`. + +```bash +kubectl delete vm cirros-vm -n my-vms +``` + +The `virt-launcher` pod is also cleaned up. + +> [!NOTE] +> Use `runStrategy: Always` (not the deprecated `spec.running: true`). `runStrategy: Always` is +> the current API and controls how KubeVirt reacts to VM failures and stops. + +## Verification + +Check the full VM status: + +```bash +kubectl get vm,vmi,pods -n my-vms +``` + +All three object types should show a `Running` or `Ready` state. + +Test that virtctl can connect to the VM API (confirms sidecar mode is working, not ambient mode): + +```bash +timeout 3 virtctl port-forward vm/cirros-vm -n my-vms 2222:22 +``` + +If this exits without "unexpected EOF" or a connection error, the `virtctl` subresource path to +the `virt-launcher` pod is working correctly. If the VM does not have SSH running, the connection +will time out normally, which is expected. + +## Troubleshooting + +### Problem: virt-launcher pod stuck in Init:ImagePullBackOff + +**Symptom:** The `virt-launcher` pod fails to start with an image pull error for +`127.0.0.1:31999/istio/proxyv2`. + +**Solution:** The pod is missing the `private-registry` imagePullSecret. Check whether the +default service account in the namespace has imagePullSecrets configured: + +```bash +kubectl get serviceaccount default -n my-vms -o jsonpath='{.imagePullSecrets}' +``` + +If empty, copy the secret and patch the service account as described in step 1. + +### Problem: container disk image not found in Zarf registry + +**Symptom:** The `virt-launcher` pod fails with +`127.0.0.1:31999/kubevirt/cirros-container-disk-demo:latest: not found`. + +**Solution:** The image was not bundled in the KubeVirt package or was pushed under a different +name. List images in the Zarf registry to find the correct path: + +```bash +# Port-forward to the Zarf registry +kubectl port-forward svc/zarf-docker-registry 15000:5000 -n zarf & +PF_PID=$! + +# Get registry credentials +kubectl get secret private-registry -n zarf \ + -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d | python3 -m json.tool + +# List all repositories (use the credentials from the secret) +curl -s -u "zarf-pull:" "http://localhost:15000/v2/_catalog" + +kill $PF_PID +``` + +### Problem: VM starts but networking does not work + +**Symptom:** The VM boots but cannot reach the pod network or external hosts. + +**Solution:** Check that the `traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0` annotation +is present on the `virt-launcher` pod: + +```bash +kubectl get pod -n my-vms -l kubevirt.io=virt-launcher \ + -o jsonpath='{.items[0].metadata.annotations.traffic\.sidecar\.istio\.io/kubevirtInterfaces}' +``` + +If missing, the annotation was blocked by Pepr policy. The +`traffic.sidecar.istio.io/kubevirtInterfaces` annotation is allowed only when the namespace +carries `uds.dev/kubevirt-workload: "true"` and the pod is a real `virt-launcher` pod (the +`virt-launcher-` name prefix plus the `kubevirt.io=virt-launcher` label, both set by KubeVirt). +A blocked annotation usually means the namespace label is missing, or you are running a UDS Core +version that does not include the native KubeVirt Pepr support. Confirm both (see +[KubeVirt on UDS Core](/concepts/core-features/virtual-machines/)). + +### Problem: virtctl console shows "unexpected EOF" + +**Symptom:** `virtctl console` or `virtctl port-forward` exits immediately with "unexpected EOF" +or hangs. + +**Solution:** This indicates the namespace is using ambient mode instead of sidecar mode. Confirm +the namespace does NOT have the `istio.io/dataplane-mode=ambient` label: + +```bash +kubectl get namespace my-vms -o jsonpath='{.metadata.labels}' +``` + +If `istio.io/dataplane-mode` is set to `ambient`, remove it and verify the UDS Package has +`serviceMesh.mode: sidecar`. + +## Related documentation + +- [KubeVirt on UDS Core](/concepts/core-features/virtual-machines/) - Architecture overview, sidecar mode requirements +- [Install KubeVirt on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) - Deploy the KubeVirt package +- [Expose a VM application on the UDS mesh](/how-to-guides/virtual-machines/vm-app-on-uds-mesh/) - Wire up Service, gateway routing, and monitoring for a VM running a network service +- [Import a disk image with CDI](/how-to-guides/virtual-machines/pvc-backed-vms-with-cdi/) - Run a VM from a PVC-backed disk instead of a container disk diff --git a/docs/how-to-guides/virtual-machines/install-kubevirt.mdx b/docs/how-to-guides/virtual-machines/install-kubevirt.mdx new file mode 100644 index 0000000000..11551ee444 --- /dev/null +++ b/docs/how-to-guides/virtual-machines/install-kubevirt.mdx @@ -0,0 +1,240 @@ +--- +title: Install KubeVirt on UDS Core +description: Deploy the KubeVirt and CDI operators on a UDS Core cluster, verify all components are healthy, and configure the VM workload namespace. +sidebar: + order: 11.001 +--- + +import { Steps } from '@astrojs/starlight/components'; + +## What you'll accomplish + +Deploy KubeVirt and CDI on a running UDS Core cluster. After this guide, both operators are healthy, the UDS Package CRs are Ready, and you have a VM workload namespace configured to run virtual machines. + +## Prerequisites + +- Access to a running UDS Core cluster (multi-node recommended; see the virt-api webhook routing + note in [KubeVirt on UDS Core](/concepts/core-features/virtual-machines/)) +- [UDS CLI](https://github.com/defenseunicorns/uds-cli/releases) installed +- A UDS Core release that includes native KubeVirt/CDI Pepr support. The `virt-launcher` and CDI + pod policy allowances are scoped by the `uds.dev/kubevirt-workload` namespace label and the + KubeVirt/CDI pod names; a UDS Core build without this support denies those annotations. See + [KubeVirt on UDS Core](/concepts/core-features/virtual-machines/) +- `kubectl` and `virtctl` available on your PATH (`virtctl` version must match the deployed KubeVirt version; install from the [KubeVirt releases page](https://github.com/kubevirt/kubevirt/releases) or via `kubectl krew install virt`) +- KVM available on the cluster nodes (or emulation enabled; see step 2) + +## Steps + + + +1. **Create a bundle that includes the KubeVirt package** + + The KubeVirt package is not part of UDS Core. You deploy it by building a UDS bundle that + includes both UDS Core and the KubeVirt package (or adding it on top of an existing UDS Core + deployment). + + Create a `uds-bundle.yaml` that includes the KubeVirt package: + + ```yaml title="uds-bundle.yaml" + kind: UDSBundle + metadata: + name: my-kubevirt-bundle + description: UDS Core with KubeVirt + version: 0.1.0 + + packages: + - name: core + repository: ghcr.io/defenseunicorns/packages/uds/core + ref: x.x.x-upstream + + - name: kubevirt + repository: ghcr.io/defenseunicorns/packages/uds/kubevirt + ref: 1.8.2-upstream + ``` + + > [!NOTE] + > No bundle variable is needed to scope VM namespaces. A VM workload namespace opts in by + > setting `kubevirt.enabled: true` in its UDS Package (the operator then labels the namespace + > `uds.dev/kubevirt-workload: "true"`), or by carrying that label directly. The VM guides cover + > this. Pods in unlabeled namespaces are denied the KubeVirt/CDI annotations. + +2. **(Optional) Enable QEMU emulation** + + If your cluster nodes do not have KVM available (common in nested virtualization or CI + environments), add this override to enable software emulation: + + > [!NOTE] + > To check if KVM is available on a k3d node: `docker exec ls /dev/kvm` + + ```yaml title="uds-bundle.yaml" + overrides: + kubevirt: + uds-kubevirt-config: + values: + - path: kubevirt.configuration.developerConfiguration.useEmulation + value: true + ``` + + The `kubevirt.configuration` path passes directly to the `KubeVirt` CR `spec.configuration`. + It is not a declared Helm value in `chart/values.yaml`; any key nested under + `kubevirt.configuration` is valid as long as it matches the KubeVirt CR spec. + + > [!WARNING] + > Emulation is significantly slower than KVM hardware acceleration. Do not use it in + > production. + +3. **(Optional) Configure the CDI scratch space storage class** + + CDI requires a scratch PVC when importing disk images from registry sources. By default it + uses the cluster's default StorageClass. To use a specific class: + + ```yaml title="uds-bundle.yaml" + overrides: + kubevirt: + uds-kubevirt-config: + values: + - path: cdi.scratchSpaceStorageClass + value: "my-storageclass" + ``` + +4. **Build and deploy the bundle** + + ```bash + uds create . + uds deploy uds-bundle-my-kubevirt-bundle-amd64-0.1.0.tar.zst --confirm + ``` + + The `uds deploy` command waits for all readiness conditions before completing: + - `KubeVirt/kubevirt` condition `Available` + - `CDI/cdi` condition `Available` + - Both UDS Packages reach `Ready` phase + + + +## Verification + +Confirm every component is healthy before proceeding. + +**Check the operators and control plane pods:** + +```bash +kubectl get pods -n kubevirt +``` + +Expected output (all pods Running and Ready; pod counts reflect a 2-node cluster): + +```console +NAME READY STATUS RESTARTS AGE +virt-api-c69d55fcb-2bfkh 1/1 Running 0 2m +virt-api-c69d55fcb-66rds 1/1 Running 0 2m +virt-controller-78d4cc5bb9-nmkmn 1/1 Running 0 2m +virt-controller-78d4cc5bb9-vvwfc 1/1 Running 0 2m +virt-handler-8qckv 2/2 Running 0 2m +virt-handler-sw6nb 2/2 Running 0 2m +virt-operator-598cbd6dc5-d6jkl 1/1 Running 0 3m +virt-operator-598cbd6dc5-pwtjg 1/1 Running 0 3m +``` + +The `virt-handler` DaemonSet runs one pod per node; a 2-node cluster shows 2 virt-handler pods. +The `2/2` count reflects two containers: `virt-handler` and `virt-launcher-image-holder` (holds +the virt-launcher image reference on the node). The `virt-api` Deployment runs 2 replicas and +spreads them across available nodes using pod anti-affinity. + +**Check CDI:** + +```bash +kubectl get pods -n cdi +``` + +Expected output (all pods Running and Ready): + +```console +NAME READY STATUS RESTARTS AGE +cdi-apiserver-5f64754756-kfs5s 1/1 Running 0 2m +cdi-deployment-67bfd4d7c6-l794s 1/1 Running 0 2m +cdi-operator-7b55c457df-8t2tt 1/1 Running 0 3m +cdi-uploadproxy-5dd77ddcc5-qqwwx 1/1 Running 0 2m +``` + +**Check the KubeVirt and CDI custom resource status:** + +```bash +kubectl get kubevirt kubevirt -n kubevirt +kubectl get cdi cdi +``` + +Both should show phase `Deployed` and condition `Available`. + +**Check the UDS Packages:** + +```bash +kubectl get packages -A +``` + +Both `kubevirt` (in the `kubevirt` namespace) and `cdi` (in the `cdi` namespace) should show +`STATUS: Ready`. + +**Check the UDS Exemption for VM workload namespaces:** + +```bash +kubectl get exemptions -n uds-policy-exemptions +``` + +A `kubevirt` exemption should be present. It contains a single entry: +- `virt-handler` pods in the `kubevirt` namespace: privileged + host path exemptions + +The KubeVirt and CDI Istio annotation allowances are not exemptions. They are scoped Pepr +admission checks keyed on the `uds.dev/kubevirt-workload` namespace label and the generated pod +names, so no per-namespace `RestrictIstioTrafficOverrides` exemption is created. + +**Check the virtctl client and server versions match:** + +```bash +virtctl version +``` + +Both `Client Version` and `Server Version` should show the same version tag. + +## Troubleshooting + +### Problem: virt-api pods stuck in Pending or CrashLoopBackOff + +**Symptom:** `kubectl get pods -n kubevirt` shows `virt-api` pods not running after 2-3 minutes. + +**Solution:** Check the pod events with `kubectl describe pod -n kubevirt `. If the issue is +a missing secret (`kubevirt-ca` or `kubevirt-virt-api-certs`), wait another minute; the operator +creates these on first run. If the pod is stuck due to resource constraints, check node capacity +with `kubectl describe node`. + +### Problem: cdi-uploadproxy fails with "secret cdi-api-signing-key not found" + +**Symptom:** `kubectl get pods -n cdi` shows `cdi-uploadproxy` in `CreateContainerConfigError` +immediately after deployment. + +**Solution:** This is normal. The CDI operator creates the `cdi-api-signing-key` secret during +initialization. The uploadproxy pod will restart and recover within 30-60 seconds. Wait for +`cdi-operator` to reach `Ready` before checking whether `cdi-uploadproxy` is running. + +### Problem: KubeVirt CR stays in Deploying phase + +**Symptom:** `kubectl get kubevirt kubevirt -n kubevirt` shows phase `Deploying` for more than +5 minutes. + +**Solution:** Check the virt-operator logs: `kubectl logs -n kubevirt -l kubevirt.io=virt-operator`. +Look for certificate or RBAC errors. If virt-handler pods are failing, check that KVM or +emulation is enabled correctly. + +### Problem: Package status is Pending, never reaches Ready + +**Symptom:** `kubectl get packages kubevirt -n kubevirt` shows `STATUS: Pending` indefinitely. + +**Solution:** Check the Pepr operator logs: `kubectl logs -n pepr-system -l app=pepr-uds-core`. +Look for validation failures on the KubeVirt UDS Package CR. A missing network allow rule or a +typo in the Package spec can block reconciliation. + +## Related documentation + +- [KubeVirt on UDS Core](/concepts/core-features/virtual-machines/) - Architecture overview before you deploy +- [KubeVirt upstream documentation](https://kubevirt.io/user-guide/) - KubeVirt user guide +- [CDI upstream documentation](https://github.com/kubevirt/containerized-data-importer/blob/main/doc/) - CDI documentation +- [Run your first VM on UDS Core](/how-to-guides/virtual-machines/first-vm/) - Deploy and connect to a container disk VM diff --git a/docs/how-to-guides/virtual-machines/l2-networking-with-multus.mdx b/docs/how-to-guides/virtual-machines/l2-networking-with-multus.mdx new file mode 100644 index 0000000000..3765c2d99a --- /dev/null +++ b/docs/how-to-guides/virtual-machines/l2-networking-with-multus.mdx @@ -0,0 +1,502 @@ +--- +title: Secondary L2 networks with Multus +description: Attach a secondary bridge network to a KubeVirt VM using Multus, verify the interface is present in the guest, and configure a static IP via cloud-init. +sidebar: + order: 11.007 +--- + +import { Steps } from '@astrojs/starlight/components'; + +## What you'll accomplish + +Create a `NetworkAttachmentDefinition` using the bridge CNI plugin, attach a secondary L2 +interface to two KubeVirt VMs, and verify that L2 traffic flows between them on the same node. +You will also configure static IPs on the secondary interfaces using cloud-init user-data and +understand how to persist those IPs across live migrations. + +## Prerequisites + +- [KubeVirt installed on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) +- [A running container disk VM](/how-to-guides/virtual-machines/first-vm/) - familiarity with the VM workload namespace and pull secret setup +- [Live migration verified](/how-to-guides/virtual-machines/live-migration/) - required for the IP persistence section +- Multus CNI installed on the cluster as a thick plugin daemonset (cluster-admin responsibility, not part of the KubeVirt package) +- `network-attachment-definitions.k8s.cni.cncf.io` CRD present on the cluster +- `kubectl` on your PATH + +> [!IMPORTANT] +> Multus is not installed by the KubeVirt package. Verify the CRD exists before starting: +> +> ```bash +> kubectl get crd network-attachment-definitions.k8s.cni.cncf.io +> ``` +> +> If the CRD is missing, you must install Multus before continuing. Two paths exist depending +> on your environment: +> +> **Internet-connected clusters:** Apply the upstream thick plugin manifest from the +> [Multus quickstart](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/quickstart.md). +> This pulls `ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick` directly from the internet. +> +> **Air-gapped clusters:** The upstream quickstart will not work because nodes cannot reach +> `ghcr.io`. Package the Multus image in a Zarf package and push it to the cluster's Zarf +> registry, then apply the daemonset manifest with the registry address rewritten to your +> Zarf registry URL. The Multus daemonset requires `privileged: true`; you must also create +> a UDS `Exemption` in the `uds-policy-exemptions` namespace that covers the Multus daemonset +> pods (`kube-system/kube-multus-ds-*`) for `DisallowPrivileged` before the pods will start. + +## Before you begin + +**Bridge CNI is node-local.** The bridge CNI plugin creates a Linux bridge (`br-l2` or whatever +name you choose) on each node independently. VMs on the same node share that bridge and can +communicate directly over L2. VMs on different nodes do NOT share the bridge; cross-node L2 +requires a VXLAN overlay or a physical L2 switch connecting the nodes. + +**host-local IPAM is also node-local.** The `host-local` IPAM plugin tracks IP leases +per-node. Two VMs on different nodes can receive the same IP address from `host-local` because +each node maintains its own allocation state. For cluster-wide unique L2 IPs, use +[whereabouts](https://github.com/k8snetworkplumbingwg/whereabouts) IPAM instead. + +**KubeVirt bridge mode networking.** When a VM declares a `bridge: {}` secondary interface, +KubeVirt creates a bridge (`k6t-`) inside the virt-launcher pod, attaches the Multus +interface to it, and creates a tap device for QEMU to use. The guest sees a virtio NIC. + +**Zarf registry address.** VM manifests in this guide use `127.0.0.1:31999` as the Zarf +registry address. This is the k3d default. On other cluster types, replace this with your +cluster's Zarf registry address. See +[Run your first VM on UDS Core](/how-to-guides/virtual-machines/first-vm/) for how to find it. + +**L2 traffic bypasses Istio.** Traffic on the secondary interface flows directly between the +Linux bridges on the node. It does not pass through ztunnel or the Istio sidecar. Mutual TLS, +authorization policies, and Istio telemetry do not apply to secondary interface traffic. Apply +network policies or firewall rules at the network level if the L2 segment carries sensitive +traffic. + +## Steps + + + +1. **Create the NetworkAttachmentDefinition** + + The `NetworkAttachmentDefinition` describes the CNI plugin configuration for the secondary + network. Create it in the same namespace as your VMs: + + ```yaml title="l2-bridge-nad.yaml" + apiVersion: k8s.cni.cncf.io/v1 + kind: NetworkAttachmentDefinition + metadata: + name: l2-bridge + namespace: my-vms + spec: + config: | + { + "cniVersion": "0.3.1", + "name": "l2-bridge", + "type": "bridge", + "bridge": "br-l2", + "isGateway": false, + "ipam": { + "type": "host-local", + "subnet": "192.168.100.0/24", + "rangeStart": "192.168.100.2", + "rangeEnd": "192.168.100.254" + } + } + ``` + + ```bash + kubectl apply -f l2-bridge-nad.yaml + ``` + + `bridge: "br-l2"` is the Linux bridge name the CNI plugin creates on the node. + The CNI plugin creates the bridge automatically the first time a pod using this NAD + schedules on that node. `isGateway: false` means no IP is assigned to the bridge + itself: the bridge is purely L2. + + > [!NOTE] + > The `host-local` IPAM assigns IPs from the per-node pool. VMs on the same node + > get unique IPs within that pool; VMs on different nodes may get the same IP. + > For cluster-wide unique IPs, replace the `ipam` block with a + > [whereabouts](https://github.com/k8snetworkplumbingwg/whereabouts) configuration. + +2. **Create the first VM with a secondary interface** + + Declare the secondary network in the VM spec using `multus.networkName` pointing at + the NAD, and set the interface type to `bridge`: + + ```yaml title="vm-l2-a.yaml" + apiVersion: kubevirt.io/v1 + kind: VirtualMachine + metadata: + name: vm-l2-a + namespace: my-vms + spec: + runStrategy: Always + template: + metadata: + labels: + kubevirt.io/vm: vm-l2-a + app: vm-l2-a + annotations: + sidecar.istio.io/inject: "true" + traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0 + spec: + domain: + cpu: + cores: 1 + resources: + requests: + memory: 128M + devices: + disks: + - name: containerdisk + disk: + bus: virtio + - name: cloudinit + disk: + bus: virtio + interfaces: + - name: default + masquerade: {} + - name: l2net + bridge: {} + networks: + - name: default + pod: {} + - name: l2net + multus: + networkName: l2-bridge # must match the NAD name in the same namespace + volumes: + - name: containerdisk + containerDisk: + image: 127.0.0.1:31999/kubevirt/cirros-container-disk-demo:latest + imagePullPolicy: IfNotPresent + - name: cloudinit + cloudInitNoCloud: + userData: | + #!/bin/sh + ip addr add 192.168.100.10/24 dev eth1 + ip link set eth1 up + ``` + + ```bash + kubectl apply -f vm-l2-a.yaml + ``` + + The `cloudinit` volume provides user-data that configures a static IP on the secondary + interface (`eth1`) at boot. The `#!/bin/sh` format causes the guest OS to run it as a + shell script via the cloud-init datasource. + +3. **Create the second VM** + + Create a second VM with a different static IP on the same L2 subnet: + + ```yaml title="vm-l2-b.yaml" + apiVersion: kubevirt.io/v1 + kind: VirtualMachine + metadata: + name: vm-l2-b + namespace: my-vms + spec: + runStrategy: Always + template: + metadata: + labels: + kubevirt.io/vm: vm-l2-b + app: vm-l2-b + annotations: + sidecar.istio.io/inject: "true" + traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0 + spec: + domain: + cpu: + cores: 1 + resources: + requests: + memory: 128M + devices: + disks: + - name: containerdisk + disk: + bus: virtio + - name: cloudinit + disk: + bus: virtio + interfaces: + - name: default + masquerade: {} + - name: l2net + bridge: {} + networks: + - name: default + pod: {} + - name: l2net + multus: + networkName: l2-bridge + volumes: + - name: containerdisk + containerDisk: + image: 127.0.0.1:31999/kubevirt/cirros-container-disk-demo:latest + imagePullPolicy: IfNotPresent + - name: cloudinit + cloudInitNoCloud: + userData: | + #!/bin/sh + ip addr add 192.168.100.11/24 dev eth1 + ip link set eth1 up + ``` + + ```bash + kubectl apply -f vm-l2-b.yaml + ``` + +4. **Wait for both VMs to reach Running** + + ```bash + kubectl get vmi -n my-vms -w + ``` + + Both VMIs reach `Running` when QEMU is live. Multus attaches the secondary interface + during pod setup before the VM starts. + + + +## Verification + +Check the Multus network-status annotation on each virt-launcher pod: + +```bash +kubectl get pod -n my-vms -l kubevirt.io/vm=vm-l2-a \ + -o jsonpath='{.items[0].metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' \ + | python3 -m json.tool +``` + +Expected output shows two entries: the default pod network and the L2 network: + +```console +[ + { + "name": "cbr0", + "interface": "eth0", + "ips": ["10.42.2.22"], + "mac": "a6:7a:70:1d:c9:e6", + "default": true + }, + { + "name": "my-vms/l2-bridge", + "interface": "pod", + "ips": ["192.168.100.2"], + "mac": "c2:64:b8:3e:76:44" + } +] +``` + +The IP in the `ips` field is the IPAM-assigned address from the `host-local` pool (starting at +`192.168.100.2`). This is the address the CNI plugin assigned to the pod network namespace +interface. It is separate from the static IP the cloud-init script configures inside the guest. + +Check the VMI status to confirm both interfaces are up: + +```bash +kubectl get vmi vm-l2-a -n my-vms \ + -o jsonpath='{.status.interfaces}' | python3 -m json.tool +``` + +The `l2net` interface should show `linkState: up` and `infoSource: domain, multus-status`: + +```console +[ + { + "name": "default", + "linkState": "up", + "ipAddress": "10.42.2.22", + ... + }, + { + "name": "l2net", + "linkState": "up", + "ipAddress": "192.168.100.2", + "infoSource": "domain, multus-status", + ... + } +] +``` + +> [!NOTE] +> The `ipAddress` in VMI status reflects the IPAM-assigned address, not the static IP set by +> cloud-init. The guest's `eth1` has `192.168.100.10` (for `vm-l2-a`) configured by the +> `#!/bin/sh` user-data script. To confirm the guest IP, connect via serial console: +> +> ```bash +> virtctl console vm-l2-a -n my-vms +> ``` +> +> Inside the guest: `ip addr show eth1` + +**Verify L2 connectivity between the two VMs.** Both VMs must be on the same node for bridge +CNI L2 to work. Confirm this first: + +```bash +kubectl get vmi -n my-vms +``` + +If both `NODENAME` values match, connect to `vm-l2-a`'s serial console and ping `vm-l2-b`: + +```bash +virtctl console vm-l2-a -n my-vms +``` + +Inside the guest: + +```console +$ ping -c3 192.168.100.11 +PING 192.168.100.11 (192.168.100.11): 56 data bytes +64 bytes from 192.168.100.11: seq=0 ttl=64 time=0.5 ms +64 bytes from 192.168.100.11: seq=1 ttl=64 time=0.5 ms +64 bytes from 192.168.100.11: seq=2 ttl=64 time=0.5 ms +--- 192.168.100.11 ping statistics --- +3 packets transmitted, 3 packets received, 0% packet loss +``` + +Three packets received with 0% loss confirms bidirectional L2 forwarding through the bridge. + +## L2 IP persistence across live migration + +When a VM migrates to a different node, KubeVirt creates a new virt-launcher pod on the +target node. Multus runs the bridge CNI ADD on the target node and the `host-local` IPAM +assigns an IP from that node's local pool. The migrated VM's secondary interface gets a new +IP from the target node's allocation state. + +To preserve the L2 IP across migrations, configure a static IP inside the guest via +cloud-init instead of relying on IPAM. Static IPs survive migration because the guest sets +them at boot, not because IPAM assigns them at pod creation time. When the VM migrates, +Multus runs CNI ADD on the target node and `host-local` assigns a new IP from the target +node's pool. The pod-side IPAM IP changes; the guest-side static IP does not. + +> [!NOTE] +> The `#!/bin/sh` format used in this guide's VMs is specific to CirrOS. On CirrOS 0.4.0, +> `#cloud-config` `bootcmd` and `runcmd` blocks are silently ignored; only the `#!/bin/sh` +> form runs. For production VMs running Fedora, Ubuntu, RHEL, or similar OSes, use +> `#cloud-config` with a `runcmd` block or a `networkData` field. + +For production VMs with a full OS, set the secondary interface IP using the cloud-init +`networkData` field so the network configuration persists across reboots. When providing +`networkData`, include all interfaces the guest uses, including the primary interface. Omitting +`eth0` from `networkData` causes cloud-init to write a network config that has no `eth0` entry, +which can break the primary interface on Ubuntu and other netplan-based systems: + +```yaml +volumes: + - name: cloudinit + cloudInitNoCloud: + networkData: | + version: 2 + ethernets: + eth0: + dhcp4: true + eth1: + addresses: + - 192.168.100.10/24 +``` + +For cluster-wide unique L2 IPs, use +[whereabouts](https://github.com/k8snetworkplumbingwg/whereabouts) as the IPAM plugin. +Whereabouts tracks IP leases across all nodes so no two pods in the cluster receive the same +address. Note that whereabouts provides uniqueness, not persistence: if the pod is deleted and +recreated (including during migration), whereabouts assigns the next available IP from the pool, +which may differ from the original. Use a guest static IP for stable addressing across restarts. + +## Troubleshooting + +### Problem: Pod stays in Init or FailedCreatePodSandBox with Multus error + +**Symptom:** `kubectl get events -n my-vms` shows +`plugin type="multus-shim" ... CmdAdd (shim): CNI request failed`. + +**Solution:** Check the Multus daemonset pod logs for the CNI error detail: + +```bash +kubectl logs -n kube-system -l app=multus --tail=50 | grep -i error +``` + +Common causes: +- The NAD `name` field does not match the `multus.networkName` in the VM spec +- The NAD is in a different namespace than the VM pod +- The CNI plugin binary (`bridge`, `host-local`) is missing from the CNI bin path on the node +- The Multus daemonset cannot see the pod network namespace (`/run/netns` or + `/var/run/netns` depending on the node OS) + +### Problem: Secondary interface missing from network-status annotation + +**Symptom:** The `k8s.v1.cni.cncf.io/network-status` annotation only shows the default network. + +**Solution:** Check the Multus daemonset logs for a NAD lookup failure: + +```bash +kubectl logs -n kube-system -l app=multus --tail=50 | grep "l2-bridge" +``` + +Confirm the NAD exists in the same namespace as the VM: + +```bash +kubectl get network-attachment-definitions -n my-vms +``` + +### Problem: VMI shows l2net linkState as down + +**Symptom:** `kubectl get vmi ... -o jsonpath='{.status.interfaces}'` shows +`"linkState": "down"` for `l2net`. + +**Solution:** The bridge on the node may not have been created yet, or the veth pair failed +to attach. Check that the `br-l2` bridge exists on the node the VM is running on. First find +which node the VM is on, then open a debug shell on that node: + +```bash +NODE=$(kubectl get vmi vm-l2-a -n my-vms -o jsonpath='{.status.nodeName}') +kubectl debug node/$NODE -it --image=busybox -- chroot /host ip link show type bridge_slave +``` + +Look for entries with `master br-l2`. If none appear, the CNI plugin did not run successfully. Force-delete the +virt-launcher pod so Multus re-runs CNI ADD: + +```bash +kubectl delete pod -n my-vms --grace-period=0 --force +``` + +### Problem: Guest cannot ping over the secondary interface + +**Symptom:** The secondary interface appears in the VMI status with an IP, but pings between +VMs fail or the interface has no traffic. + +**Solution:** Verify both VMs are on the same node. Bridge CNI is node-local; VMs on +different nodes cannot reach each other through the bridge: + +```bash +kubectl get vmi -n my-vms +``` + +Check the `NODENAME` column. Both `vm-l2-a` and `vm-l2-b` must show the same node for +same-node L2 communication to work. + +If both VMs are on the same node and pings still fail, confirm the guest configured the +secondary interface IP. Connect to the console and check: + +```bash +virtctl console vm-l2-a -n my-vms +``` + +Inside the guest, verify `eth1` has the expected IP: + +```console +$ ip addr show eth1 +``` + +If `eth1` is down or has no IP, the cloud-init script did not run. Verify the cloud-init +volume is attached and the user-data format is correct for your guest OS. + +## Related documentation + +- [Live-migrate a VM between nodes](/how-to-guides/virtual-machines/live-migration/) - Prerequisites for IP persistence across migration +- [KubeVirt on UDS Core](/concepts/core-features/virtual-machines/) - Architecture overview +- [Multus quickstart](https://github.com/k8snetworkplumbingwg/multus-cni/blob/master/docs/quickstart.md) - Installing Multus on a cluster +- [whereabouts IPAM](https://github.com/k8snetworkplumbingwg/whereabouts) - Cluster-wide IP address management for secondary networks +- [KubeVirt interfaces and networks](https://kubevirt.io/user-guide/network/interfaces_and_networks/) - Full reference for KubeVirt network interface types diff --git a/docs/how-to-guides/virtual-machines/live-migration.mdx b/docs/how-to-guides/virtual-machines/live-migration.mdx new file mode 100644 index 0000000000..017ea81b88 --- /dev/null +++ b/docs/how-to-guides/virtual-machines/live-migration.mdx @@ -0,0 +1,295 @@ +--- +title: Live-migrate a VM between nodes +description: Trigger a live migration on a container disk VM, observe the migration lifecycle, and understand why Service DNS should be used instead of the VMI IP for connections. +sidebar: + order: 11.006 +--- + +import { Steps } from '@astrojs/starlight/components'; + +## What you'll accomplish + +Trigger a live migration on a running container disk VM, watch the migration through to +`Succeeded`, and confirm that a Kubernetes Service tracks the VM across nodes automatically. +You will also observe the VMI IP change that happens during migration and understand why clients +must use Service DNS, not the VMI IP, for reliable connections. + +## Prerequisites + +- [KubeVirt installed on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) +- [A running container disk VM](/how-to-guides/virtual-machines/first-vm/) - Running `VirtualMachineInstance` in a VM workload namespace +- A cluster with at least two nodes (live migration moves the VM between nodes; a single-node cluster has nowhere to migrate to) +- `kubectl` and `virtctl` on your PATH + +> [!NOTE] +> This guide covers container disk VMs only. Live migration of PVC-backed VMs additionally +> requires a `ReadWriteMany` StorageClass (for example, Longhorn). The container disk path +> verified here uses `PreCopy` mode: memory pages are transferred to the target node before +> the final cutover. + +## Before you begin + +**VMI IP changes on every migration.** Each `virt-launcher` pod gets an IP from the node's pod +CIDR. When the VM migrates to a different node, KubeVirt creates a new pod there with a new IP. +The old pod enters `Completed` state and is removed shortly after. + +**Use Service DNS, not the VMI IP.** The Kubernetes `Service` endpoint tracks the `virt-launcher` +pod label and updates automatically when the pod changes. Any client that holds the VMI IP loses +its connection during migration and cannot reconnect. Clients that use the Service ClusterIP or +DNS name reconnect to the new pod transparently after a brief interruption. + +**The connection disruption window.** During the final cutover, inbound connections are +interrupted while the target pod takes over. For CirrOS container disk VMs on local +infrastructure, this measured around 1-2 seconds with a small percentage of in-flight requests +failing. Larger VM memory footprints extend this window proportionally. Clients should implement +a retry loop on connection errors if continuity is required. + +> [!IMPORTANT] +> Live migration requires a second node, and adding one changes `virt-api` scheduling. KubeVirt +> spreads `virt-api` replicas across nodes, so a replica can land on a worker node. Cross-node +> webhook calls from kube-apiserver to a `virt-api` pod fail because kube-apiserver does not speak +> HBONE, and that blocks every `VirtualMachine` create, update, and delete. Before adding the +> node, review +> [virt-api webhook routing](/concepts/core-features/virtual-machines/#virt-api-webhook-routing) +> and pin `virt-api` to the control-plane node, or cordon the new node during VM creation and +> uncordon it afterward. + +## Steps + + + +1. **Verify the VM is live-migratable** + + Check that the VMI shows `LIVE-MIGRATABLE: True`: + + ```bash + kubectl get vmi -n + ``` + + Expected output: + + ```console + NAME AGE PHASE IP NODENAME READY LIVE-MIGRATABLE PAUSED + migrate-test 2m Running 10.42.2.9 worker-node-1 True True + ``` + + If `LIVE-MIGRATABLE` is `False`, check the VMI conditions for the reason: + + ```bash + kubectl get vmi -n \ + -o jsonpath='{.status.conditions}' | python3 -m json.tool + ``` + +2. **Confirm a second node is available** + + Migration requires a target node. Verify at least two nodes are `Ready`: + + ```bash + kubectl get nodes + ``` + + KubeVirt schedules the target `virt-launcher` pod using normal Kubernetes scheduling. If only + one node is available, the migration pod stays `Pending` and the migration eventually times out. + +3. **Create a Service for the VM (if you do not have one)** + + If you plan to connect to the VM across migrations, create a Service that selects the + `virt-launcher` pod by label. Labels in `spec.template.metadata.labels` in the + `VirtualMachine` manifest become labels on the `virt-launcher` pod. The VM from + [Run your first VM on UDS Core](/how-to-guides/virtual-machines/first-vm/) already carries an + `app:` label for this purpose; match that label in the selector below. + + ```yaml title="vm-service.yaml" + apiVersion: v1 + kind: Service + metadata: + name: + namespace: + spec: + selector: + app: # must match a label on the virt-launcher pod + ports: + - name: ssh + port: 22 + targetPort: 22 + ``` + + ```bash + kubectl apply -f vm-service.yaml + ``` + +4. **Trigger the migration** + + ```bash + virtctl migrate -n + ``` + + This creates a `VirtualMachineInstanceMigration` CR named `kubevirt-migrate-vm-` in + the same namespace. You can also create the CR directly: + + ```yaml title="migration.yaml" + apiVersion: kubevirt.io/v1 + kind: VirtualMachineInstanceMigration + metadata: + name: migrate-to-other-node + namespace: + spec: + vmiName: + ``` + + ```bash + kubectl apply -f migration.yaml + ``` + +5. **Watch the migration** + + Watch the `VirtualMachineInstanceMigration` phase: + + ```bash + kubectl get vmim -n -w + ``` + + The migration moves through these phases: + + ```console + NAME PHASE VMI + kubevirt-migrate-vm-8l4h6 Pending migrate-test + kubevirt-migrate-vm-8l4h6 Scheduling migrate-test + kubevirt-migrate-vm-8l4h6 Scheduled migrate-test + kubevirt-migrate-vm-8l4h6 PreparingTarget migrate-test + kubevirt-migrate-vm-8l4h6 TargetReady migrate-test + kubevirt-migrate-vm-8l4h6 Running migrate-test + kubevirt-migrate-vm-8l4h6 Succeeded migrate-test + ``` + + Watch the pods to see the source pod complete and the target pod take over: + + ```bash + kubectl get pods -n -w + ``` + + During migration you briefly see two `virt-launcher` pods: one `Running` on the source node + and one `Running` on the target node. After cutover, the source pod transitions to + `Completed` and is removed. + + + +## Verification + +Check that the VMI moved to a different node and has a new IP: + +```bash +kubectl get vmi -n +``` + +The `NODENAME` and `IP` columns both change. The VMI IP moves to an address in the target +node's pod CIDR. + +Confirm the Service endpoint followed the VM to the new pod: + +```bash +kubectl get endpoints -n +``` + +The endpoint IP matches the new VMI IP. Clients reconnecting through the Service ClusterIP +reach the VM on its new node without any configuration change. + +Check the migration history: + +```bash +kubectl get vmim -n +``` + +Each migration creates one `VirtualMachineInstanceMigration` CR. Completed migrations stay in +the namespace until you delete them. + +## Persistent disk migration + +Live migration of PVC-backed VMs requires a `ReadWriteMany` StorageClass. With `local-path` +provisioner (the default on k3d), PVC-backed VMs show `LIVE-MIGRATABLE: False` because the +PVC cannot be mounted simultaneously on two nodes. + +To enable live migration for PVC-backed VMs, provision a `ReadWriteMany`-capable StorageClass +such as Longhorn, and create the DataVolume with `ReadWriteMany` access mode before creating +the VM. + +## Troubleshooting + +### Problem: LIVE-MIGRATABLE shows False + +**Symptom:** `kubectl get vmi` shows `LIVE-MIGRATABLE: False`. + +**Solution:** Check the VMI conditions for the non-migratable reason: + +```bash +kubectl get vmi -n \ + -o jsonpath='{.status.conditions}' | python3 -m json.tool | grep -A3 LiveMigratable +``` + +Common causes: +- The VM uses a PVC with `ReadWriteOnce` access mode (requires `ReadWriteMany` for live migration) +- The VM has a hostDisk volume (hostDisk volumes are node-local and cannot migrate) +- virt-handler is not running on both nodes + +### Problem: Migration stays in Scheduling or Pending + +**Symptom:** `kubectl get vmim` shows phase `Scheduling` or `Pending` for more than 2 minutes. + +**Solution:** No target node is available. Check node readiness: + +```bash +kubectl get nodes +``` + +Check the target pod events: + +```bash +kubectl get pods -n | grep virt-launcher +kubectl describe pod -n +``` + +If the pod is `Pending`, the scheduler cannot find a node. Common causes: all nodes are +cordoned, resource requests on the VM exceed available capacity on the second node, or a +node selector prevents scheduling on the available node. + +### Problem: Migration fails after reaching Running phase + +**Symptom:** `kubectl get vmim` shows `Failed` after reaching `Running`. + +**Solution:** Check the migration details: + +```bash +kubectl get vmim -n \ + -o jsonpath='{.status.migrationState}' | python3 -m json.tool +``` + +Also check the virt-handler logs on the source node. Identify the virt-handler pod on that +node first, then stream its logs: + +```bash +kubectl get pods -n kubevirt -l kubevirt.io=virt-handler \ + --field-selector=spec.nodeName= -o name \ + | xargs kubectl logs -n kubevirt +``` + +Common causes: the VM has a userspace device (GPU, SR-IOV NIC) that does not support migration, +or the memory transfer exceeded the timeout. + +### Problem: Clients lose connection after migration + +**Symptom:** Connections to the VM drop after migration completes and do not automatically recover. + +**Solution:** The client is connecting to the VMI IP directly instead of through a Service. The +VMI IP changes on every migration. Create a Kubernetes Service that selects the `virt-launcher` +pod by label (step 3), and update clients to connect through the Service ClusterIP or DNS name. + +The interruption window during live migration is expected. For container disk VMs on local +infrastructure this measured 1-2 seconds. Clients should implement a retry loop on the +connection if continuity is critical. + +## Related documentation + +- [Run your first VM on UDS Core](/how-to-guides/virtual-machines/first-vm/) - Create the VM used in this guide +- [Expose a VM application on the UDS mesh](/how-to-guides/virtual-machines/vm-app-on-uds-mesh/) - Full Service, gateway, and monitoring setup for VM applications +- [L2 networking with Multus](/how-to-guides/virtual-machines/l2-networking-with-multus/) - Secondary network interfaces and IP persistence across migrations +- [KubeVirt live migration documentation](https://kubevirt.io/user-guide/compute/live_migration/) - Full migration configuration reference diff --git a/docs/how-to-guides/virtual-machines/overview.mdx b/docs/how-to-guides/virtual-machines/overview.mdx new file mode 100644 index 0000000000..9d3cb9e0bd --- /dev/null +++ b/docs/how-to-guides/virtual-machines/overview.mdx @@ -0,0 +1,62 @@ +--- +title: Virtual machines +description: Guides for deploying and managing virtual machines on UDS Core using KubeVirt and CDI. +sidebar: + order: 11.000 +--- + +import { CardGrid, LinkCard } from '@astrojs/starlight/components'; + +UDS Core supports running virtual machines alongside container workloads using KubeVirt and CDI. This section covers installing the platform components, running VMs on the UDS mesh, and advanced workflows like PVC-backed disks, live migration, and L2 networking. + +For background on how KubeVirt and CDI integrate with UDS Core's service mesh, policy enforcement, and observability stack, see [Virtual machines](/concepts/core-features/virtual-machines/). + +## Guides + + + + + + + + + + + + diff --git a/docs/how-to-guides/virtual-machines/pvc-backed-vms-with-cdi.mdx b/docs/how-to-guides/virtual-machines/pvc-backed-vms-with-cdi.mdx new file mode 100644 index 0000000000..de173e8227 --- /dev/null +++ b/docs/how-to-guides/virtual-machines/pvc-backed-vms-with-cdi.mdx @@ -0,0 +1,376 @@ +--- +title: Import a disk image with CDI +description: Use CDI DataVolumes to import disk images from the Zarf registry or an HTTP server, then run VMs from the resulting PVCs. +sidebar: + order: 11.004 +--- + +import { Steps, Tabs, TabItem } from '@astrojs/starlight/components'; + +## What you'll accomplish + +Create a PersistentVolumeClaim backed by a real disk image using CDI DataVolumes. You'll import +from the Zarf registry and from an HTTP source, then boot a VM from the resulting PVC. PVC-backed +disks persist across VM restarts and can be snapshotted or migrated independently from the VM. + +## Prerequisites + +- [KubeVirt installed on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) - KubeVirt and CDI operators running +- [A working VM namespace](/how-to-guides/virtual-machines/first-vm/) - Namespace with UDS Package, sidecar injection, and `private-registry` secret +- `kubectl` and `virtctl` on your PATH +- A disk image accessible from the cluster (Zarf registry or HTTP endpoint) + +## Before you begin + +CDI's `DataVolume` resource describes a disk import operation. When you create a DataVolume, CDI +creates a temporary `importer-*` pod in your namespace to fetch the image and write it to a PVC. +When the import completes, the pod is removed and the DataVolume phase transitions to `Succeeded`. + +CDI importer pods set `sidecar.istio.io/inject: "false"`. UDS Core's Pepr module allows this +annotation on pods whose names start with `importer-`. No extra namespace configuration is needed. + +For the Zarf registry source, CDI uses `pullMethod: node`, which routes through the node's +containerd daemon. This uses the same credentials containerd uses for pulling container images, +so no `secretRef` is needed in the DataVolume spec. + +> [!NOTE] +> The `virt-launcher` pod for a PVC-backed VM shows `3/3` containers, not `4/4`. Container +> disk VMs include a `volumecontainerdisk` container that serves the disk image; PVC-backed +> VMs do not need this because the disk is already on a PVC. + +## Steps + + + +1. **Create the VM workload namespace** + + Set up a namespace with Istio sidecar injection and a UDS Package. CDI importer pods run in + this namespace, so the UDS Package must allow the egress traffic the importer needs. + + > [!NOTE] + > `kubevirt.enabled: true` makes the operator label the namespace + > `uds.dev/kubevirt-workload: "true"`. The label is also set directly on the namespace here: + > the CDI importer pod can start during deploy, before the operator reconciles the Package, and + > the importer is denied if the label is not already present. Setting it on the manifest removes + > that race. + + ```yaml title="vm-namespace-package.yaml" + apiVersion: v1 + kind: Namespace + metadata: + name: my-vms + labels: + istio-injection: enabled + uds.dev/kubevirt-workload: "true" + --- + apiVersion: uds.dev/v1alpha1 + kind: Package + metadata: + name: my-vms + namespace: my-vms + spec: + kubevirt: + enabled: true + network: + serviceMesh: + mode: sidecar + allow: + - direction: Egress + remoteGenerated: KubeAPI + description: virt-launcher to KubeAPI + # Add egress rules for the import source if using the HTTP tab below. + # CDI importer pods run in this namespace and need egress to the source URL. + # A UDS allow rule requires a remote target; a port-only rule is rejected. + # Example for an HTTP/HTTPS source outside the cluster: + # - direction: Egress + # remoteGenerated: Anywhere # scope to the source CIDR with remoteCidr in production + # port: 80 + # description: CDI importer HTTP source + # - direction: Egress + # remoteGenerated: Anywhere # scope to the source CIDR with remoteCidr in production + # port: 443 + # description: CDI importer HTTPS source + ``` + + Copy the `private-registry` secret so Istio sidecars can pull from the Zarf registry: + + ```bash + DOCKER_CONFIG=$(kubectl get secret private-registry -n zarf \ + -o jsonpath='{.data.\.dockerconfigjson}') + + kubectl apply -f - < + + + Use `pullMethod: node` and omit `secretRef`. CDI uses the node's containerd credentials, + which already have access to the Zarf registry (`127.0.0.1:31999` on k3d; find your address with `kubectl get svc zarf-docker-registry -n zarf` on other clusters). + + ```yaml title="fedora-dv.yaml" + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: fedora-disk + namespace: my-vms + spec: + source: + registry: + url: "docker://127.0.0.1:31999/containerdisks/fedora:latest" + pullMethod: node + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 6Gi + ``` + + > [!IMPORTANT] + > Do not set `secretRef` on a Zarf registry source. CDI will look for S3-style credentials + > (`accessKeyId`/`secretKey`) and fail if the secret is a Docker config format. The node + > pull path handles authentication through the node's containerd configuration. + + + + + Point the DataVolume at any HTTP URL that serves a raw, QCOW2, or ISO disk image. The URL + must be reachable from within the cluster. + + ```yaml title="fedora-dv.yaml" + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: fedora-disk + namespace: my-vms + spec: + source: + http: + url: "http://my-fileserver.example.com/fedora-40.qcow2" + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 6Gi + ``` + + Size the PVC at least as large as the uncompressed disk image. QCOW2 images expand on write; + provision enough overhead for the guest OS to operate. + + + + + Apply the DataVolume: + + ```bash + kubectl apply -f fedora-dv.yaml + ``` + +3. **Watch import progress** + + The DataVolume moves through phases: `ImportScheduled` → `ImportInProgress` → `Succeeded`. + + ```bash + kubectl get datavolume fedora-disk -n my-vms -w + ``` + + Watch the importer pod: + + ```bash + kubectl get pods -n my-vms -w + ``` + + The importer pod is named `importer-`. It runs, writes the image to the PVC, + and is removed when the import completes. + + When complete: + + ```console + NAME PHASE PROGRESS RESTARTS AGE + fedora-disk Succeeded 100.0% 45s + ``` + +4. **Create a VM from the PVC** + + Reference the DataVolume's PVC in the `volumes` section of the VM spec. The PVC name matches + the DataVolume name. + + ```yaml title="fedora-vm.yaml" + apiVersion: kubevirt.io/v1 + kind: VirtualMachine + metadata: + name: fedora-vm + namespace: my-vms + spec: + runStrategy: Always + template: + metadata: + labels: + kubevirt.io/vm: fedora-vm + annotations: + sidecar.istio.io/inject: "true" + traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0 + spec: + domain: + cpu: + cores: 2 + resources: + requests: + memory: 2Gi + devices: + disks: + - name: pvcdisk + disk: + bus: virtio + interfaces: + - name: default + masquerade: {} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 0 # dev only; use 60+ in production for graceful shutdown + volumes: + - name: pvcdisk + persistentVolumeClaim: + claimName: fedora-disk # must match DataVolume name + ``` + + ```bash + kubectl apply -f fedora-vm.yaml + ``` + +5. **Wait for the VM to reach Running** + + ```bash + kubectl get vmi fedora-vm -n my-vms -w + ``` + + When phase shows `Running`, connect to the console: + + ```bash + virtctl console fedora-vm -n my-vms + ``` + + + +## Verification + +**Check the DataVolume and PVC:** + +```bash +kubectl get datavolume,pvc -n my-vms +``` + +Both should exist. The DataVolume shows `Succeeded`; the PVC shows `Bound`. + +**Check the VM and virt-launcher pod:** + +```bash +kubectl get vm,vmi,pods -n my-vms +``` + +The `virt-launcher` pod shows `3/3` ready. The three containers are: +- `compute` (QEMU/libvirt process) +- `istio-proxy` (Istio sidecar; uses the Kubernetes 1.29 native sidecar mechanism so it counts in the container total and runs for the pod's lifetime) +- `guest-console-log` (captures serial console output, stays running) + +**Verify the disk is attached inside the VM:** + +Connect via console and check that the block device is visible: + +```bash +virtctl console fedora-vm -n my-vms +``` + +Inside the guest: + +```bash +lsblk +``` + +You should see a `vda` block device corresponding to the PVC. + +## Troubleshooting + +### Problem: DataVolume stuck in ImportScheduled + +**Symptom:** `kubectl get datavolume -n my-vms` shows `ImportScheduled` with no importer pod. + +**Solution:** The importer pod failed to schedule. Check events on the DataVolume: + +```bash +kubectl describe datavolume fedora-disk -n my-vms +``` + +Common causes: the namespace has no default StorageClass, or the PVC size exceeds available +node storage. Check cluster storage: + +```bash +kubectl get storageclass +kubectl describe nodes | grep -A5 "Allocated resources" +``` + +### Problem: Registry import fails with "couldn't find key accessKeyId" + +**Symptom:** The importer pod logs show `Error: couldn't find key accessKeyId in Secret`. + +**Solution:** A `secretRef` was set on a registry source DataVolume. Remove `secretRef` from the +DataVolume spec and re-create it. CDI treats `secretRef` on registry sources as S3 credentials +and looks for `accessKeyId`/`secretKey` fields. For Zarf registry sources with `pullMethod: +node`, no `secretRef` is needed. + +### Problem: HTTP import fails with connection timeout + +**Symptom:** The importer pod logs show a connection timeout or "no route to host". + +**Solution:** In a UDS namespace, the default-deny NetworkPolicy blocks the importer pod's egress +unless your UDS Package has an egress allow rule with a remote target (`remoteGenerated: Anywhere` +or a `remoteCidr`) for the source port. Add the rule shown in step 1 and re-create the DataVolume. +If the egress rule is already present, confirm the HTTP URL is reachable from inside the cluster: +check that the URL's host resolves and is reachable. In airgap environments, the HTTP server must +be in-cluster or on the cluster's internal network. To upload a local disk image instead, see +[Upload a local disk image with virtctl](/how-to-guides/virtual-machines/upload-disk-image/). + +### Problem: PVC-backed VM fails to start with disk not found + +**Symptom:** The VMI shows an error about the PVC not existing or not being bound. + +**Solution:** Verify the DataVolume phase is `Succeeded` before creating the VM: + +```bash +kubectl get datavolume fedora-disk -n my-vms +``` + +If the DataVolume is still importing, wait for it to complete. The PVC is not ready until the +DataVolume reaches `Succeeded`. + +### Problem: virt-launcher shows ErrImagePull for Istio sidecar + +**Symptom:** The `virt-launcher` pod fails with an image pull error for +`127.0.0.1:31999/istio/proxyv2`. + +**Solution:** The `private-registry` secret is missing or not attached to the service account. +Run the copy and patch commands from step 1. + +## Related documentation + +- [KubeVirt on UDS Core](/concepts/core-features/virtual-machines/) - Architecture overview, CDI importer pod model +- [Install KubeVirt on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) - CDI scratchSpaceStorageClass configuration +- [Run your first VM on UDS Core](/how-to-guides/virtual-machines/first-vm/) - Container disk VMs and namespace setup +- [Upload a local disk image with virtctl](/how-to-guides/virtual-machines/upload-disk-image/) - Import a disk image from your workstation via CDI upload +- [CDI DataVolume documentation](https://github.com/kubevirt/containerized-data-importer/blob/main/doc/datavolumes.md) - Full DataVolume spec reference diff --git a/docs/how-to-guides/virtual-machines/upload-disk-image.mdx b/docs/how-to-guides/virtual-machines/upload-disk-image.mdx new file mode 100644 index 0000000000..a68e82577d --- /dev/null +++ b/docs/how-to-guides/virtual-machines/upload-disk-image.mdx @@ -0,0 +1,332 @@ +--- +title: Upload a local disk image with virtctl +description: Use virtctl image-upload and CDI to upload a local disk image directly to a PVC in your VM namespace. Useful when the image is on your workstation and not accessible from inside the cluster. +sidebar: + order: 11.005 +--- + +import { Steps } from '@astrojs/starlight/components'; + +## What you'll accomplish + +Upload a local disk image (QCOW2, raw, or ISO) from your workstation to a PVC in your VM +namespace using `virtctl image-upload` and CDI's upload proxy. The resulting PVC is ready to use +as a VM disk, identical to one created by a CDI DataVolume import. + +## Prerequisites + +- [KubeVirt installed on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) - KubeVirt and CDI operators running +- [A working VM namespace](/how-to-guides/virtual-machines/first-vm/) - Namespace with UDS Package and `private-registry` secret +- `kubectl` and `virtctl` on your PATH +- A disk image on your workstation in a format CDI supports: QCOW2, raw, or ISO + +## Before you begin + +CDI's upload path uses an ephemeral upload server pod (`cdi-upload-`) that CDI creates +in your VM namespace when an upload starts. The pod receives the image data from the CDI +uploadproxy and writes it to the PVC. CDI removes the pod when the upload completes. + +Several platform behaviors affect this path: + +**Pepr allows the upload server pod.** CDI sets `sidecar.istio.io/inject=false` on the upload +server pod. Pepr's `RestrictIstioTrafficOverrides` policy permits this on pods whose names start +with `cdi-upload-`. No namespace-level exemption is required. + +**CDI uploadproxy bypasses ztunnel.** The `cdi-uploadproxy` deployment runs with the +`istio.io/dataplane-mode: none` label so Istio CNI does not enroll it in ztunnel. CDI uses its +own mTLS certificate (`cdi-uploadserver-client-cert`) for uploadproxy-to-upload-server traffic. +Ztunnel intercepting this path breaks the connection because the mTLS handshake does not use +Istio certificates. The KubeVirt package sets this label on the uploadproxy deployment via the +CDI CR `customizeComponents.patches` field. + +**Your VM namespace needs a network allow rule.** CDI uploadproxy (in the `cdi` namespace) +connects to the upload server pod in your namespace on port 8443. If your namespace has a UDS +Package, its default-deny NetworkPolicy will block this unless you add an explicit ingress allow +rule. Add the rule described in step 1 before starting an upload. + +## Steps + + + +1. **Add the CDI upload network rule to your VM namespace UDS Package** + + Your VM namespace's UDS Package needs an ingress allow rule so CDI uploadproxy can reach the + upload server pod. Add the following allow entry under `spec.network.allow`: + + ```yaml title="vm-namespace-package.yaml" + apiVersion: uds.dev/v1alpha1 + kind: Package + metadata: + name: my-vms + namespace: my-vms + spec: + network: + serviceMesh: + mode: sidecar + allow: + - direction: Egress + remoteGenerated: KubeAPI + description: virt-launcher to KubeAPI + + - direction: Ingress + remoteNamespace: cdi + remoteSelector: + cdi.kubevirt.io: cdi-uploadproxy + port: 8443 + description: CDI upload server + ``` + + Apply it: + + ```bash + kubectl apply -f vm-namespace-package.yaml + ``` + +2. **Copy the `private-registry` secret into your namespace** + + CDI pulls the upload server image from the Zarf registry. The upload server pod runs in your + namespace and needs the `private-registry` pull secret to authenticate. + + ```bash + DOCKER_CONFIG=$(kubectl get secret private-registry -n zarf \ + -o jsonpath='{.data.\.dockerconfigjson}') + + kubectl apply -f - < [!NOTE] + > If you followed [Import a disk image with CDI](/how-to-guides/virtual-machines/pvc-backed-vms-with-cdi/) + > and already copied the secret, this is idempotent: re-running it updates the secret in place. + +3. **Create a PVC to upload into** + + Create the PVC before starting the upload. Size it to hold the uncompressed disk image. + + ```yaml title="upload-pvc.yaml" + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: my-disk + namespace: my-vms + spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 10Gi + ``` + + ```bash + kubectl apply -f upload-pvc.yaml + ``` + + Size the PVC at least as large as the uncompressed image. QCOW2 images expand on write; a + compressed QCOW2 often occupies two to three times its compressed size on disk. + +4. **Port-forward the CDI uploadproxy** + + CDI uploadproxy is not exposed outside the cluster by default. Port-forward it to your + workstation for the duration of the upload. + + ```bash + kubectl port-forward svc/cdi-uploadproxy -n cdi 18443:443 & + PF_PID=$! + ``` + + Keep this running while you upload. The `PF_PID` variable lets you stop it cleanly in step 6. + +5. **Upload the image** + + Run `virtctl image-upload` targeting the existing PVC: + + ```bash + virtctl image-upload pvc my-disk \ + --no-create \ + --uploadproxy-url=https://localhost:18443 \ + --insecure \ + --image-path=/path/to/your/image.qcow2 \ + -n my-vms + ``` + + The `--no-create` flag tells `virtctl` to use the PVC you created in the previous step rather + than creating a new one. The `--insecure` flag skips TLS verification for the local + port-forward connection. + + A successful upload looks like: + + ```console + Using existing PVC my-vms/my-disk + Uploading data to https://localhost:18443 + + 4.80 GiB / 4.80 GiB [==============================] 100.00% ? p/s 0s + Uploading data completed successfully, waiting for processing to complete, you can hit ctrl-c without interrupting the progress + Processing completed successfully + Uploading /path/to/your/image.qcow2 completed successfully + ``` + + The progress bar reflects the image file size, not the PVC size. A 4.80 GiB QCOW2 uploaded + into a 10Gi PVC shows 4.80 GiB transferred. That is expected. + +6. **Stop the port-forward** + + After the upload completes, stop the port-forward: + + ```bash + kill $PF_PID + ``` + + Or if you started it in a separate terminal, press `Ctrl-C` there. + + + +## Verification + +Check that the PVC is bound and ready: + +```bash +kubectl get pvc my-disk -n my-vms +``` + +The PVC shows `Bound`: + +```console +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +my-disk Bound pvc-a1b2c3d4-e5f6-7890-abcd-ef1234567890 10Gi RWO local-path 2m +``` + +Confirm the upload server pod cleaned up: + +```bash +kubectl get pods -n my-vms +``` + +The `cdi-upload-my-disk` pod is gone after a successful upload. CDI removes it automatically. + +The PVC is now ready to use as a VM disk. Reference it in a VirtualMachine spec the same way as +a DataVolume-created PVC: + +```yaml +volumes: + - name: disk0 + persistentVolumeClaim: + claimName: my-disk # matches the PVC name from step 3 +``` + +## Troubleshooting + +### Problem: Upload server pod fails with ErrImagePull + +**Symptom:** `kubectl get pods -n my-vms` shows `cdi-upload-my-disk` in `ErrImagePull` or +`ImagePullBackOff`. + +**Solution:** The `private-registry` secret is missing from your namespace. Copy it from the +`zarf` namespace: + +```bash +DOCKER_CONFIG=$(kubectl get secret private-registry -n zarf \ + -o jsonpath='{.data.\.dockerconfigjson}') + +kubectl apply -f - < [!WARNING] +> A manual `kubectl patch` on the deployment **will be reverted** when the CDI operator +> reconciles. The persistent fix is to upgrade to a KubeVirt package version that includes the +> `customizeComponents.patches` entry on the CDI CR. + +### Problem: Upload fails with `connection refused` immediately + +**Symptom:** `virtctl image-upload` exits quickly with a connection error before the upload bar +appears. + +**Solution:** The CDI uploadproxy cannot reach the upload server pod in your namespace. Verify +the CDI UDS Package has the egress rule for port 8443: + +```bash +kubectl get networkpolicy -n cdi | grep uploadproxy +``` + +If the `allow-cdi-egress-cdi-uploadproxy-to-upload-server` (or similar) NetworkPolicy is +missing, the KubeVirt package version does not include the cross-namespace egress rule. Upgrade +the KubeVirt package, or add the NetworkPolicy manually for testing: + +```bash +kubectl apply -f - < [!WARNING] +> A manually applied NetworkPolicy in the `cdi` namespace will be reverted when the UDS operator +> reconciles the CDI `Package` CR. The persistent fix is to upgrade to a KubeVirt package version +> that includes the cross-namespace egress rule in `chart/templates/cdi-uds-package.yaml`. + +### Problem: Upload server pod denied by Pepr + +**Symptom:** The upload fails immediately and `kubectl get events -n my-vms` shows a Pepr +admission denial for the `cdi-upload-my-disk` pod. + +**Solution:** The Pepr module does not have the CDI pod exemption for +`sidecar.istio.io/inject=false`. This exemption is part of UDS Core and covers pods whose names +start with `cdi-upload-`. Check the UDS Core version in use and upgrade if needed. + +## Related documentation + +- [KubeVirt on UDS Core](/concepts/core-features/virtual-machines/) - Architecture overview, CDI upload server pod model +- [Import a disk image with CDI](/how-to-guides/virtual-machines/pvc-backed-vms-with-cdi/) - Import from Zarf registry or HTTP instead of a local file +- [Run your first VM on UDS Core](/how-to-guides/virtual-machines/first-vm/) - VM namespace setup and `private-registry` secret +- [CDI upload documentation](https://github.com/kubevirt/containerized-data-importer/blob/main/doc/upload.md) - Full virtctl image-upload reference diff --git a/docs/how-to-guides/virtual-machines/vm-app-on-uds-mesh.mdx b/docs/how-to-guides/virtual-machines/vm-app-on-uds-mesh.mdx new file mode 100644 index 0000000000..5bbcafdc51 --- /dev/null +++ b/docs/how-to-guides/virtual-machines/vm-app-on-uds-mesh.mdx @@ -0,0 +1,417 @@ +--- +title: Expose a VM application on the UDS mesh +description: Configure Service routing, tenant gateway exposure, and Prometheus monitoring for a VM running a network service on UDS Core. +sidebar: + order: 11.003 +--- + +import { Steps } from '@astrojs/starlight/components'; + +## What you'll accomplish + +Deploy a Fedora VM that runs a simple HTTP service, expose it through the UDS tenant gateway at +`https://.uds.dev`, and configure a ServiceMonitor so Prometheus can scrape metrics from the +VM's service port. This is the golden pattern for any VM application that needs to be reachable +from outside the cluster or observed by UDS Core's monitoring stack. + +This guide uses a Python HTTP server started by cloud-init as the example application. The same +pattern applies to any network service running inside a VM. + +## Prerequisites + +- [KubeVirt installed on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) - KubeVirt and CDI running +- [A working VM deployment](/how-to-guides/virtual-machines/first-vm/) - Familiar with `VirtualMachine` manifests and lifecycle +- `kubectl` and `virtctl` on your PATH + +## Before you begin + +Service routing for VM applications works through the `virt-launcher` pod. The Istio sidecar on +the `virt-launcher` pod intercepts inbound connections and routes them into the VM guest through +the `k6t-eth0` virtual interface. This means: + +- The Kubernetes `Service` selector must target the `virt-launcher` pod, not the VM itself. + Labels in `spec.template.metadata.labels` in the `VirtualMachine` manifest become labels on + the `virt-launcher` pod. +- Ports that will receive traffic from Services, gateways, or monitoring must be declared in + `spec.template.spec.domain.devices.interfaces[].ports`. Without the port declaration, KubeVirt + does not configure the masquerade interface to forward traffic on that port into the guest. +- The declared port number, the Kubernetes `Service` `targetPort`, the UDS Package `expose` + `targetPort`, and the UDS Package `monitor` `targetPort` must all match. + +## Steps + + + +1. **Create the VM workload namespace** + + The namespace needs Istio sidecar injection and a UDS Package with `mode: sidecar`. The + UDS Package also declares the gateway exposure and monitoring in the same manifest. + + > [!NOTE] + > The namespace carries `uds.dev/kubevirt-workload: "true"`, which is what permits the + > `virt-launcher` Istio annotations. `kubevirt.enabled: true` on the Package makes the operator + > apply that label, but this VM uses `runStrategy: Always`, so its launcher pod can start during + > deploy before the operator reconciles. The label is set on the namespace manifest too so it + > exists first; without it the launcher pod is denied. + + ```yaml title="vm-app-namespace.yaml" + apiVersion: v1 + kind: Namespace + metadata: + name: my-vm-app + labels: + istio-injection: enabled + uds.dev/kubevirt-workload: "true" + --- + apiVersion: uds.dev/v1alpha1 + kind: Package + metadata: + name: my-vm-app + namespace: my-vm-app + spec: + kubevirt: + enabled: true + network: + serviceMesh: + mode: sidecar + expose: + - service: my-vm-app + selector: + app.kubernetes.io/name: my-vm-app + gateway: tenant + host: my-vm-app + port: 80 + targetPort: 80 + allow: + - direction: Egress + remoteGenerated: KubeAPI + description: virt-launcher to KubeAPI + - direction: Ingress + remoteGenerated: IntraNamespace + - direction: Egress + remoteGenerated: IntraNamespace + monitor: + - selector: + app.kubernetes.io/name: my-vm-app + targetPort: 80 + portName: http + description: VM app metrics + kind: ServiceMonitor + ``` + + > [!NOTE] + > The `expose.selector` and `monitor.selector` target the `virt-launcher` pod, not the VM. + > Labels you put in the `VirtualMachine` `spec.template.metadata.labels` become labels on the + > `virt-launcher` pod. Use those same labels in the selectors here. + +2. **Create the Kubernetes Service** + + The Service selects the `virt-launcher` pod and routes to the VM's service port. + + ```yaml title="vm-app-service.yaml" + apiVersion: v1 + kind: Service + metadata: + name: my-vm-app + namespace: my-vm-app + labels: + app.kubernetes.io/name: my-vm-app + spec: + selector: + app.kubernetes.io/name: my-vm-app + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + ``` + +3. **Create the VirtualMachine** + + The VM manifest has three requirements beyond a basic VM: + + - Labels in `spec.template.metadata.labels` must match the Service selector (they become + `virt-launcher` pod labels). + - The service port must be declared in `spec.template.spec.domain.devices.interfaces[].ports`. + - Both `sidecar.istio.io/inject: "true"` and + `traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0` must be in + `spec.template.metadata.annotations`. + + ```yaml title="vm-app.yaml" + apiVersion: kubevirt.io/v1 + kind: VirtualMachine + metadata: + name: my-vm-app + namespace: my-vm-app + labels: + app.kubernetes.io/name: my-vm-app + spec: + runStrategy: Always + template: + metadata: + labels: + app.kubernetes.io/name: my-vm-app # must match Service selector + kubevirt.io/domain: my-vm-app + annotations: + sidecar.istio.io/inject: "true" + traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0 + spec: + domain: + cpu: + cores: 1 + resources: + requests: + memory: 1024M + devices: + disks: + - name: containerdisk + disk: + bus: virtio + - name: cloudinitdisk + disk: + bus: virtio + interfaces: + - name: default + masquerade: {} + ports: + - name: http # must match Service port name + port: 80 # must match Service targetPort + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 0 # dev only; use 60+ in production for graceful shutdown + volumes: + - name: containerdisk + containerDisk: + image: 127.0.0.1:31999/containerdisks/fedora:latest # k3d registry; replace with your Zarf registry address + imagePullPolicy: IfNotPresent + - name: cloudinitdisk + cloudInitNoCloud: + userData: |- + #cloud-config + password: fedora + chpasswd: { expire: False } + write_files: + - path: /usr/local/bin/app.py + permissions: '0755' + content: | + #!/usr/bin/env python3 + import json + from http.server import BaseHTTPRequestHandler, HTTPServer + + requests_total = 0 + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + global requests_total + requests_total += 1 + if self.path == "/metrics": + # Prometheus text exposition format + body = ( + "# HELP vm_app_requests_total Total HTTP requests served.\n" + "# TYPE vm_app_requests_total counter\n" + f"vm_app_requests_total {requests_total}\n" + ).encode() + content_type = "text/plain; version=0.0.4" + else: + body = json.dumps({"status": "ok"}).encode() + content_type = "application/json" + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass + + HTTPServer(("0.0.0.0", 80), Handler).serve_forever() + - path: /etc/systemd/system/app.service + permissions: '0644' + content: | + [Unit] + Description=VM app HTTP service + After=network-online.target + Wants=network-online.target + + [Service] + ExecStart=/usr/local/bin/app.py + Restart=always + + [Install] + WantedBy=multi-user.target + runcmd: + - [ systemctl, daemon-reload ] + - [ systemctl, enable, --now, app.service ] + ``` + + > [!WARNING] + > Do NOT use `traffic.sidecar.istio.io/excludeInboundPorts` on `virt-launcher` pods. This + > breaks the gateway mTLS path. Do not use HTTP readiness probes with a custom `host` value; + > these interact badly with Istio's probe rewrite mechanism. Use TCP socket probes on declared + > ports instead. + + > [!NOTE] + > The example app serves Prometheus metrics on `/metrics` (the default scrape path) and a + > `{"status": "ok"}` response on every other path. The `monitor` entry in step 1 scrapes + > `/metrics` on the same service port. + +4. **Apply all manifests and wait for the VM** + + **(Recommended)** Package these manifests in a Zarf package and deploy with Zarf. See + [Packaging applications](/how-to-guides/packaging-applications/overview/) for guidance. + + **Or** apply directly for quick testing: + + ```bash + kubectl apply -f vm-app-namespace.yaml + kubectl apply -f vm-app-service.yaml + kubectl apply -f vm-app.yaml + ``` + + Wait for the VM to start: + + ```bash + kubectl get vmi my-vm-app -n my-vm-app -w + ``` + + When phase reaches `Running`, the `virt-launcher` pod is up and the VM is booting. Cloud-init + runs during the first boot and may take 60-90 seconds to start your application. + + + +## Verification + +**Check the virt-launcher pod is running and has the correct labels and annotations:** + +```bash +kubectl get pod -n my-vm-app -l app.kubernetes.io/name=my-vm-app +kubectl get pod -n my-vm-app -l kubevirt.io=virt-launcher \ + -o jsonpath='{.items[0].metadata.annotations}' | python3 -m json.tool | \ + grep -E "kubevirtInterfaces|inject|reroute" +``` + +Expected annotations on the virt-launcher pod: + +```console +"traffic.sidecar.istio.io/kubevirtInterfaces": "k6t-eth0", +"sidecar.istio.io/inject": "true", +"istio.io/reroute-virtual-interfaces": "k6t-eth0" +``` + +**Check the Service has endpoints:** + +```bash +kubectl get endpoints my-vm-app -n my-vm-app +``` + +The endpoint address should be the `virt-launcher` pod IP. If ``, the Service selector +does not match the `virt-launcher` pod labels. + +**Check the UDS Package created the VirtualService:** + +```bash +kubectl get virtualservice -n my-vm-app +``` + +Expected output shows a VirtualService routing through `istio-tenant-gateway/tenant-gateway`. + +**Check the ServiceMonitor was created:** + +```bash +kubectl get servicemonitor -n my-vm-app +``` + +**Test the service through the gateway:** + +```bash +curl -sk --resolve "my-vm-app.uds.dev:443:127.0.0.1" https://my-vm-app.uds.dev/ +``` + +**Test the service directly via port-forward (bypasses gateway, useful for debugging):** + +```bash +kubectl port-forward svc/my-vm-app 8080:80 -n my-vm-app +curl http://localhost:8080/ +``` + +## How the routing path works + +Requests arrive at the tenant ingressgateway, which terminates TLS and routes to the `my-vm-app` +Service via the VirtualService the UDS Package operator creates. The Service selects the +`virt-launcher` pod; the Istio sidecar on that pod forwards traffic through `k6t-eth0` into the +VM guest because port 80 is declared in the VM interface spec. + +## Port alignment + +This table shows all the places port 80 must appear for the full routing chain to work. A +mismatch anywhere in this chain causes routing failures. + +| Location | Field | Value | +|---|---|---| +| VM interface | `domain.devices.interfaces[].ports[].port` | `80` | +| VM interface | `domain.devices.interfaces[].ports[].name` | `http` | +| Service | `spec.ports[].targetPort` | `80` | +| Service | `spec.ports[].name` | `http` | +| UDS Package expose | `expose[].targetPort` | `80` | +| UDS Package monitor | `monitor[].targetPort` | `80` | +| UDS Package monitor | `monitor[].portName` | `http` | + +## Troubleshooting + +### Problem: Service has no endpoints + +**Symptom:** `kubectl get endpoints my-vm-app -n my-vm-app` shows ``. + +**Solution:** The Service selector does not match the `virt-launcher` pod labels. Check that the +labels in `spec.template.metadata.labels` in the `VirtualMachine` manifest match the `selector` +in the Service. Verify with: + +```bash +kubectl get pod -n my-vm-app -l app.kubernetes.io/name=my-vm-app +``` + +If no pods are returned, the label is missing or different on the virt-launcher pod. + +### Problem: Gateway returns 503 or connection refused + +**Symptom:** `curl https://my-vm-app.uds.dev/` returns a 503 or the gateway cannot reach the +backend. + +**Solution:** Check that the VM application is running on port 80 inside the guest, and that the +port is declared in the VM interface spec. Also verify the Service's `targetPort` matches. Test +the direct path first: + +```bash +kubectl port-forward svc/my-vm-app 8080:80 -n my-vm-app +curl http://localhost:8080/ +``` + +If this also fails, the issue is inside the VM. Connect via console to check the application: + +```bash +virtctl console my-vm-app -n my-vm-app +``` + +### Problem: VM boots but service port is not reachable + +**Symptom:** The VM is running, the pod has endpoints, but requests to port 80 time out. + +**Solution:** The port is not declared in `spec.template.spec.domain.devices.interfaces[].ports`. +Without this declaration, KubeVirt's masquerade interface does not forward port 80 traffic into +the VM guest. Add the port declaration and recreate the VM. + +### Problem: Prometheus is not scraping the VM app + +**Symptom:** The ServiceMonitor exists but no metrics appear in Prometheus. + +**Solution:** Verify the ServiceMonitor `portName` matches the Service port name. Verify the VM +application exposes a Prometheus-compatible `/metrics` endpoint (or configure a different +`path` in the monitor spec). In slim-dev deployments, Prometheus is not included; scraping +requires the standard UDS Core bundle with the monitoring stack. + +## Related documentation + +- [KubeVirt on UDS Core](/concepts/core-features/virtual-machines/) - Architecture overview, mesh mode rules +- [Run your first VM on UDS Core](/how-to-guides/virtual-machines/first-vm/) - Basic VM lifecycle and container disk +- [Import a disk image with CDI](/how-to-guides/virtual-machines/pvc-backed-vms-with-cdi/) - Use a PVC-backed disk instead of a container disk +- [Windows Server VMs on UDS Core](/how-to-guides/virtual-machines/windows-server-vm/) - Apply this pattern to a Windows VM with monitoring diff --git a/docs/how-to-guides/virtual-machines/windows-server-vm.mdx b/docs/how-to-guides/virtual-machines/windows-server-vm.mdx new file mode 100644 index 0000000000..a595dd52dc --- /dev/null +++ b/docs/how-to-guides/virtual-machines/windows-server-vm.mdx @@ -0,0 +1,440 @@ +--- +title: Run a Windows Server VM on UDS Core +description: Import a Windows Server QCOW2 image with CDI, configure a KubeVirt VirtualMachine with UEFI firmware, HyperV enlightenments, and the VirtIO driver disk, and expose RDP through a UDS Package. +sidebar: + order: 11.009 +--- + +import { Steps } from '@astrojs/starlight/components'; + +## What you'll accomplish + +Import a Windows Server disk image into a PVC using CDI, create a `VirtualMachine` with the +correct UEFI firmware, HyperV timer, and VirtIO driver configuration, expose RDP (port 3389) +through the UDS mesh, and verify the guest is reachable via `virtctl port-forward`. + +## Prerequisites + +- [KubeVirt installed on UDS Core](/how-to-guides/virtual-machines/install-kubevirt/) +- [Import a disk image with CDI](/how-to-guides/virtual-machines/pvc-backed-vms-with-cdi/) - familiarity with DataVolume import +- `kubectl` and `virtctl` on your PATH, pointed at the cluster +- A Windows Server QCOW2 disk image accessible from inside the cluster (HTTP source or local upload) + +> [!NOTE] +> This guide uses Windows Server 2022. The KubeVirt manifest and UDS Package structure apply +> to Windows Server 2019 without changes. For a local disk image upload workflow, see +> [Upload a local disk image with virtctl](/how-to-guides/virtual-machines/upload-disk-image/). + +## Before you begin + +Windows Server VMs on KubeVirt require three things that Linux VMs typically do not: + +- **UEFI firmware** - Windows Server 2022 requires UEFI. Set `firmware.bootloader.efi.secureBoot: false` + unless you have a Secure Boot-enabled image. +- **HyperV enlightenments** - Paravirtualized timers, interrupt controllers, and other + HyperV features dramatically improve Windows guest performance under KVM. The full set + is listed in the manifest below. +- **HyperV timer** - The `clock.timer.hyperv` field must be present when using `synictimer`. + Omitting it causes a libvirt error at VM start: `'stimer' hyperv feature requires 'hypervclock' timer`. +- **VirtIO driver disk** - Windows does not ship with VirtIO disk or network drivers. The + `virtio-container-disk` image in the Zarf registry provides the VirtIO Windows driver ISO. + Attach it as a CD-ROM so Windows can find the drivers on first boot. + +## Steps + + + +1. **Set up the VM workload namespace** + + If you do not already have a sidecar-mode VM namespace, set one up following + [Run your first VM on UDS Core](/how-to-guides/virtual-machines/first-vm/). That guide + covers namespace labels, the `private-registry` pull secret, and the UDS Package CR. + + The UDS Package for a Windows VM needs one additional allow rule: an ingress on port 3389 + to the VM pod, so that `virtctl port-forward` can forward RDP traffic. Add it when creating + the Package: + + ```yaml title="vm-namespace-package.yaml" + apiVersion: uds.dev/v1alpha1 + kind: Package + metadata: + name: my-vms + namespace: my-vms + spec: + network: + serviceMesh: + mode: sidecar + allow: + - direction: Egress + remoteGenerated: KubeAPI + description: virt-launcher to KubeAPI + - direction: Ingress + port: 3389 + selector: + app: windows-vm # must match labels on the virt-launcher pod + remoteGenerated: Anywhere # restrict to your RDP client CIDR in production + description: RDP access + ``` + + ```bash + kubectl apply -f vm-namespace-package.yaml + ``` + +2. **Import the Windows disk image** + + Import the Windows Server QCOW2 image into a PVC using a DataVolume. The HTTP import + source works for images hosted on an accessible URL. For a local file, use + `virtctl image-upload` as described in + [Upload a local disk image with virtctl](/how-to-guides/virtual-machines/upload-disk-image/). + + ```yaml title="windows-disk.yaml" + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: windows-disk + namespace: my-vms + spec: + source: + http: + url: "http://your-server/windows-server-2022.qcow2" # QCOW2 HTTP URL + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi # must be larger than the uncompressed disk image + # storageClassName: omit to use the cluster default, or specify your StorageClass + ``` + + ```bash + kubectl apply -f windows-disk.yaml + ``` + + Watch import progress: + + ```bash + kubectl get datavolume windows-disk -n my-vms -w + ``` + + Import completes when the DataVolume phase reaches `Succeeded`. + +3. **Create the VirtualMachine** + + ```yaml title="windows-vm.yaml" + apiVersion: kubevirt.io/v1 + kind: VirtualMachine + metadata: + name: windows-vm + namespace: my-vms + spec: + runStrategy: Always + template: + metadata: + labels: + kubevirt.io/vm: windows-vm + app: windows-vm # matches the RDP ingress selector in step 1 + annotations: + sidecar.istio.io/inject: "true" + traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0 + spec: + domain: + cpu: + cores: 2 + resources: + requests: + memory: 4Gi + clock: + utc: {} + timer: + hpet: + present: false + pit: + tickPolicy: delay + rtc: + tickPolicy: catchup + hyperv: # required when synictimer is enabled + present: true + firmware: + bootloader: + efi: + secureBoot: false + features: + hyperv: + relaxed: {} + vapic: {} + vpindex: {} + spinlocks: + spinlocks: 8191 + synic: {} + synictimer: + direct: {} + reset: {} + runtime: {} + frequencies: {} + reenlightenment: {} + tlbflush: {} + ipi: {} + devices: + disks: + - name: windowsdisk + disk: + bus: virtio + bootOrder: 1 + - name: virtiocontainerdisk + cdrom: + bus: sata # sata so Windows can find it before VirtIO drivers load + bootOrder: 2 + interfaces: + - name: default + masquerade: {} + networks: + - name: default + pod: {} + volumes: + - name: windowsdisk + persistentVolumeClaim: + claimName: windows-disk + - name: virtiocontainerdisk + containerDisk: + image: /kubevirt/virtio-container-disk: # replace with your Zarf registry address and the KubeVirt version in your package + imagePullPolicy: IfNotPresent + ``` + + ```bash + kubectl apply -f windows-vm.yaml + ``` + + > [!IMPORTANT] + > The `clock.timer.hyperv.present: true` field is required. KubeVirt uses the field name + > `hyperv` (not `hypervclock`) under `clock.timer`. Omitting it when `synictimer` is enabled + > causes a libvirt error and the VM never starts. + + > [!NOTE] + > The VirtIO driver disk uses `bus: sata` for the CD-ROM because Windows cannot read a + > VirtIO disk before VirtIO drivers are installed. SATA is available without additional + > drivers and is accessible from the Windows installer or a running Windows guest. + +4. **Wait for the VM to start** + + Watch the `virt-launcher` pod: + + ```bash + kubectl get pods -n my-vms -w + ``` + + A Windows VM running with a VirtIO driver disk shows one more container than a standard + Linux VM: + + ```console + NAME READY STATUS RESTARTS AGE + virt-launcher-windows-vm-xxxxx 4/4 Running 0 30s + ``` + + The `4/4` count includes: + - `compute` (the QEMU/libvirt container) + - `volumevirtiocontainerdisk` (serves the VirtIO ISO as a CD-ROM) + - `istio-proxy` (Istio sidecar; uses the Kubernetes 1.29 native sidecar mechanism so it counts in the container total and runs for the pod's lifetime) + - `guest-console-log` (captures serial console output, stays running) + + Check the VMI: + + ```bash + kubectl get vmi windows-vm -n my-vms + ``` + + The phase shows `Running` when QEMU has started. The Windows guest takes several minutes + to boot for the first time. + + > [!NOTE] + > The `4/4` container count and container names above were verified on a real cluster using a + > PVC-backed VM with a `virtio-container-disk` volume. + > + > The platform mechanics for steps 5-7 were verified on 2026-06-29 using a CirrOS VM in a + > sidecar-mode namespace: `virtctl vnc` opened a VNC session and captured a live guest + > framebuffer screenshot; `virtctl port-forward vm/ -n 3389:3389` forwarded + > TCP port 3389 to localhost (TCP connection confirmed end-to-end); and `runStrategy: Always` + > restarted the VM automatically after a guest `poweroff`, with a new VMI created within the + > same second as shutdown. + > + > The Windows-specific procedures (VirtIO driver path inside the ISO, sysprep command and + > shutdown behavior, and RDP client authentication) follow standard Windows documentation and + > have not been executed against a live Windows guest. + +5. **Install VirtIO drivers (first boot)** + + On first boot, Windows cannot see the primary disk because VirtIO drivers are not yet + installed. The Windows installer (or Windows itself if the disk has an existing install) + must load the drivers from the VirtIO CD-ROM. + + Connect to the VM console: + + ```bash + virtctl console windows-vm -n my-vms + ``` + + Press `Ctrl+]` to exit the console. + + > [!WARNING] + > The serial console shows UEFI firmware boot messages and the Windows boot manager only. + > Windows Setup and the Windows desktop do not render in the serial console. You must use + > a graphical console to complete driver installation. Use `virtctl vnc` to open a VNC + > session (requires a VNC viewer on your workstation, such as `vncviewer` or TigerVNC), + > or skip ahead and wait for RDP to become available after the VM finishes its initial boot + > (this requires the VirtIO storage driver to already be present in the Windows image, which + > is the case if you used a pre-sysprep'd image with VirtIO drivers pre-installed). + + In the Windows installer or guest (accessed via VNC): + + 1. Open the VirtIO driver CD (typically drive `E:` or the last drive letter). + 2. Navigate to `virtio-win-gt-x64.msi` (for 64-bit Windows Server) and run it. + This installs all VirtIO drivers including the storage driver, network driver, + balloon driver, and guest agent (`virtio-win-guest-tools.exe`). + 3. Reboot when prompted. + + After the first reboot, Windows uses the VirtIO disk driver and the VM starts faster on + subsequent boots. + +6. **Verify the guest agent is running** + + After VirtIO drivers and the QEMU guest agent are installed, virt-controller reports the + guest's IP address via the VMI status: + + ```bash + kubectl get vmi windows-vm -n my-vms -o jsonpath='{.status.guestOSInfo}' + ``` + + A running guest agent reports the OS name, version, and hostname. If this field is empty, + the `qemu-guest-agent` service is not running inside the Windows guest. + +7. **Connect via RDP** + + Port-forward RDP through the Istio sidecar: + + ```bash + virtctl port-forward vm/windows-vm -n my-vms 3389:3389 + ``` + + Connect your RDP client to `localhost:3389`. Use the Windows username and password you + configured during installation. + + + +## Verification + +Check that the VM is running and the guest agent is reporting: + +```bash +kubectl get vm,vmi -n my-vms +``` + +Both the `VirtualMachine` and `VirtualMachineInstance` should show `Running` / `Ready: True`. + +Confirm the RDP NetworkPolicy exists and targets the correct pod label: + +```bash +kubectl get networkpolicy -n my-vms | grep rdp +``` + +Expected: one policy with selector `app=windows-vm` on port 3389. + +## Monitoring and logging + +Monitoring and logging for Windows VMs follow the same UDS Core patterns as any other VM +application, with the addition of in-guest agents for both metrics and logs. + +> [!NOTE] +> UDS Core slim-dev does not deploy Prometheus or Loki, so the monitoring and logging paths +> below are not verified end to end. Both follow the standard UDS Core application patterns +> and require a full UDS Core deployment to test. + +**Metrics (Windows Exporter):** Install +[Windows Exporter](https://github.com/prometheus-community/windows_exporter) inside the Windows +guest and configure it to listen on a port (default: 9182). Declare the metrics port in your +UDS Package and add a `monitor` entry, the same pattern as +[Expose a VM application on the UDS mesh](/how-to-guides/virtual-machines/vm-app-on-uds-mesh/). + +**Logs:** Ship Windows event logs and application logs to the UDS Core log stack using a +log forwarding agent installed inside the guest (for example, Promtail or Elastic Agent). +Configure the agent to send to the Loki or Elasticsearch endpoint in your cluster. + +## Troubleshooting + +### Problem: VM starts but stays at phase Scheduled + +**Symptom:** `kubectl get vmi windows-vm -n my-vms` shows `Scheduled` indefinitely with no +IP address. + +**Solution:** Check VMI conditions for libvirt errors: + +```bash +kubectl get vmi windows-vm -n my-vms \ + -o jsonpath='{.status.conditions}' | python3 -m json.tool +``` + +A common cause is missing the `clock.timer.hyperv` field when `synictimer` is enabled. The +error reads: `'stimer' hyperv feature requires 'hypervclock' timer`. Add +`clock.timer.hyperv.present: true` to the VirtualMachine spec and delete and re-create the VM. + +### Problem: Windows cannot find the boot disk + +**Symptom:** Windows setup or boot reports no disks available. + +**Solution:** VirtIO disk drivers are not installed. The primary disk is VirtIO bus and Windows +cannot see it without the driver. Confirm the VirtIO driver CD-ROM is attached: + +```bash +kubectl get vmi windows-vm -n my-vms \ + -o jsonpath='{.spec.domain.devices.disks}' | python3 -m json.tool +``` + +The `virtiocontainerdisk` disk should be present with `cdrom.bus: sata`. If missing, the VM +was created without the VirtIO driver volume. Update the VirtualMachine spec to add it and +restart the VM. + +### Problem: virt-launcher pod stuck in Init phase + +**Symptom:** The pod shows `Init:ImagePullBackOff` for the `volumevirtiocontainerdisk` container. + +**Solution:** The `virtio-container-disk` image is not in the Zarf registry. Verify: + +```bash +kubectl port-forward svc/zarf-docker-registry 15000:5000 -n zarf & +PF_PID=$! +sleep 2 +PASS=$(kubectl get secret private-registry -n zarf \ + -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d \ + | python3 -c " +import sys, json, base64 +d = json.load(sys.stdin) +auth = list(d['auths'].values())[0]['auth'] +print(base64.b64decode(auth).decode().split(':')[1]) +") +curl -s -u "zarf-pull:$PASS" "http://localhost:15000/v2/kubevirt/virtio-container-disk/tags/list" +kill $PF_PID +``` + +If the image is missing, rebuild and redeploy the KubeVirt bundle. The KubeVirt package source +includes the `virtio-container-disk` image. + +### Problem: virtctl port-forward exits immediately + +**Symptom:** `virtctl port-forward` exits with "unexpected EOF" or a connection error immediately +after connecting. + +**Solution:** The namespace is using ambient mode instead of sidecar mode. Confirm: + +```bash +kubectl get namespace my-vms -o jsonpath='{.metadata.labels.istio\.io/dataplane-mode}' +``` + +If the output is `ambient`, remove the label and verify the UDS Package has +`serviceMesh.mode: sidecar`. See the troubleshooting section in +[Run your first VM on UDS Core](/how-to-guides/virtual-machines/first-vm/) for full instructions. + +## Related documentation + +- [Run your first VM on UDS Core](/how-to-guides/virtual-machines/first-vm/) - Namespace setup, container disk VM basics +- [Import a disk image with CDI](/how-to-guides/virtual-machines/pvc-backed-vms-with-cdi/) - DataVolume import patterns +- [Upload a local disk image with virtctl](/how-to-guides/virtual-machines/upload-disk-image/) - Upload a local QCOW2 file instead of HTTP import +- [Expose a VM application on the UDS mesh](/how-to-guides/virtual-machines/vm-app-on-uds-mesh/) - Metrics port declaration and ServiceMonitor pattern +- [KubeVirt Windows guest documentation](https://kubevirt.io/user-guide/virtual_machines/windows_virtio_drivers/) - VirtIO driver installation details diff --git a/docs/reference/operator-and-crds/packages-v1alpha1-cr.md b/docs/reference/operator-and-crds/packages-v1alpha1-cr.md index f3ae4f64c9..9bc79f94c8 100644 --- a/docs/reference/operator-and-crds/packages-v1alpha1-cr.md +++ b/docs/reference/operator-and-crds/packages-v1alpha1-cr.md @@ -36,7 +36,7 @@ sidebar: - networkNetworkNetwork configuration for the packagemonitorMonitor[]Create Service or Pod Monitor configurationsssoSso[]Create SSO client configurationscaBundleCaBundleCA bundle configuration for the package + networkNetworkNetwork configuration for the packagemonitorMonitor[]Create Service or Pod Monitor configurationsssoSso[]Create SSO client configurationscaBundleCaBundleCA bundle configuration for the packagekubevirtKubevirtKubeVirt workload settings. When enabled, the UDS operator labels the namespace 'uds.dev/kubevirt-workload: "true"' so the KubeVirt and CDI Pepr policy allowances apply to the namespace's generated pods (virt-launcher, CDI importer/upload/clone). @@ -637,4 +637,22 @@ Valid Options: FROM_PROTOCOL_DEFAULT, FROM_REQUEST_PORTnamestringThe name of the ConfigMap to create (default: uds-trust-bundle)keystringThe key name inside the ConfigMap (default: ca-bundle.pem)labelsAdditional labels to apply to the generated ConfigMap (default: {})annotationsAdditional annotations to apply to the generated ConfigMap (default: {}) + + + +
+ +### Kubevirt + + + + + + + + + + + +
FieldTypeDescription
enabledbooleanEnable KubeVirt workload support for this namespace. Labels the namespace so KubeVirt and CDI generated pods are permitted their required Istio annotations.
\ No newline at end of file diff --git a/packages/kubevirt/chart/Chart.yaml b/packages/kubevirt/chart/Chart.yaml new file mode 100644 index 0000000000..2a435cd3e9 --- /dev/null +++ b/packages/kubevirt/chart/Chart.yaml @@ -0,0 +1,21 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: v2 +name: uds-kubevirt-config +description: uds-kubevirt-config + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 diff --git a/packages/kubevirt/chart/templates/authorizationpolicy.yaml b/packages/kubevirt/chart/templates/authorizationpolicy.yaml new file mode 100644 index 0000000000..f0fd3b54b9 --- /dev/null +++ b/packages/kubevirt/chart/templates/authorizationpolicy.yaml @@ -0,0 +1,20 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: allow-kubevirt-webhooks + namespace: {{ .Release.Namespace }} +spec: + action: ALLOW + selector: + matchLabels: + kubevirt.io: virt-operator + rules: + - to: + - operation: + # 8444: validating/mutating webhook (kube-apiserver). 8443: metrics (Prometheus scrape). + # An ALLOW policy makes Istio default-deny all other inbound to this workload, so the + # metrics port must be listed explicitly or scraping is blocked. + ports: ["8443", "8444"] diff --git a/packages/kubevirt/chart/templates/cdi-servicemonitor.yaml b/packages/kubevirt/chart/templates/cdi-servicemonitor.yaml new file mode 100644 index 0000000000..a15429065c --- /dev/null +++ b/packages/kubevirt/chart/templates/cdi-servicemonitor.yaml @@ -0,0 +1,20 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +{{- if .Values.cdi.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: cdi-metrics + namespace: cdi +spec: + endpoints: + - path: /metrics + port: metrics + scheme: https + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + prometheus.cdi.kubevirt.io: "true" +{{- end }} diff --git a/packages/kubevirt/chart/templates/cdi-uds-package.yaml b/packages/kubevirt/chart/templates/cdi-uds-package.yaml new file mode 100644 index 0000000000..ec572e3f1f --- /dev/null +++ b/packages/kubevirt/chart/templates/cdi-uds-package.yaml @@ -0,0 +1,66 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +{{- if .Values.cdi.enabled }} +apiVersion: uds.dev/v1alpha1 +kind: Package +metadata: + name: cdi + namespace: cdi +spec: + network: + serviceMesh: + mode: ambient + allow: + - direction: Ingress + remoteGenerated: IntraNamespace + + - direction: Egress + remoteGenerated: IntraNamespace + + - direction: Ingress + selector: + cdi.kubevirt.io: cdi-apiserver + remoteGenerated: Anywhere + description: "cdi-apiserver webhook" + + - direction: Ingress + remoteNamespace: monitoring + remoteSelector: + app: prometheus + selector: + prometheus.cdi.kubevirt.io: "true" + port: 8443 + description: "CDI metrics" + + - direction: Egress + selector: + cdi.kubevirt.io: cdi-operator + remoteGenerated: KubeAPI + description: "cdi-operator to KubeAPI" + + - direction: Egress + selector: + cdi.kubevirt.io: cdi-apiserver + remoteGenerated: KubeAPI + description: "cdi-apiserver to KubeAPI" + + - direction: Egress + selector: + cdi.kubevirt.io: cdi-deployment + remoteGenerated: KubeAPI + description: "cdi-controller to KubeAPI" + + - direction: Egress + selector: + cdi.kubevirt.io: cdi-uploadproxy + remoteGenerated: KubeAPI + description: "cdi-uploadproxy to KubeAPI" + + - direction: Egress + selector: + cdi.kubevirt.io: cdi-uploadproxy + remoteGenerated: Anywhere + port: 8443 + description: "cdi-uploadproxy to upload server pods in user namespaces" +{{- end }} diff --git a/packages/kubevirt/chart/templates/cdi.yaml b/packages/kubevirt/chart/templates/cdi.yaml new file mode 100644 index 0000000000..61b109fda1 --- /dev/null +++ b/packages/kubevirt/chart/templates/cdi.yaml @@ -0,0 +1,36 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +{{- if .Values.cdi.enabled }} +apiVersion: cdi.kubevirt.io/v1beta1 +kind: CDI +metadata: + name: cdi +spec: + imagePullPolicy: IfNotPresent + config: + imagePullSecrets: + - name: "private-registry" + {{- if .Values.cdi.scratchSpaceStorageClass }} + scratchSpaceStorageClass: {{ .Values.cdi.scratchSpaceStorageClass }} + {{- end }} + {{- if .Values.cdi.infra.nodeSelector }} + infra: + nodeSelector: + {{- toYaml .Values.cdi.infra.nodeSelector | nindent 6 }} + {{- end }} + {{- if .Values.cdi.workload.nodeSelector }} + workload: + nodeSelector: + {{- toYaml .Values.cdi.workload.nodeSelector | nindent 6 }} + {{- end }} + # Opt cdi-uploadproxy out of ztunnel interception in Istio ambient mode. + # CDI uses its own mTLS (cdi-uploadserver-client-cert) for uploadproxy→upload server traffic. + # Ztunnel intercepting this path causes ECONNREFUSED; bypassing it restores direct CDI mTLS. + customizeComponents: + patches: + - resourceName: cdi-uploadproxy + resourceType: Deployment + type: strategic + patch: '{"spec":{"template":{"metadata":{"labels":{"istio.io/dataplane-mode":"none"}}}}}' +{{- end }} diff --git a/packages/kubevirt/chart/templates/kubevirt.yaml b/packages/kubevirt/chart/templates/kubevirt.yaml new file mode 100644 index 0000000000..adb4a1e3a2 --- /dev/null +++ b/packages/kubevirt/chart/templates/kubevirt.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +{{- if .Values.kubevirt.enabled }} +apiVersion: kubevirt.io/v1 +kind: KubeVirt +metadata: + name: kubevirt + namespace: {{ .Release.Namespace }} +spec: + infra: + {{- toYaml .Values.kubevirt.infra | nindent 4 }} + workloads: + {{- toYaml .Values.kubevirt.workloads | nindent 4 }} + configuration: {{- toYaml .Values.kubevirt.configuration | nindent 4 }} + imagePullPolicy: IfNotPresent + imagePullSecrets: + - name: "private-registry" + imageRegistry: {{ .Values.image.registry }} + imageTag: {{ .Values.image.tag }} + {{- if .Values.kubevirt.extraValues }} + {{- toYaml .Values.kubevirt.extraValues | nindent 2 }} + {{- end }} +{{- end }} diff --git a/packages/kubevirt/chart/templates/peerauthentication-cdi-apiserver.yaml b/packages/kubevirt/chart/templates/peerauthentication-cdi-apiserver.yaml new file mode 100644 index 0000000000..bbe3c7702b --- /dev/null +++ b/packages/kubevirt/chart/templates/peerauthentication-cdi-apiserver.yaml @@ -0,0 +1,19 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +{{- if .Values.cdi.enabled }} +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: cdi-apiserver-peerauth + namespace: cdi +spec: + selector: + matchLabels: + cdi.kubevirt.io: cdi-apiserver + mtls: + mode: STRICT + portLevelMtls: + 8443: + mode: PERMISSIVE +{{- end }} diff --git a/packages/kubevirt/chart/templates/peerauthentication-operator.yaml b/packages/kubevirt/chart/templates/peerauthentication-operator.yaml new file mode 100644 index 0000000000..7ff7abe593 --- /dev/null +++ b/packages/kubevirt/chart/templates/peerauthentication-operator.yaml @@ -0,0 +1,17 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: virt-operator-peerauth + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + kubevirt.io: virt-operator + mtls: + mode: STRICT + portLevelMtls: + 8444: + mode: PERMISSIVE diff --git a/packages/kubevirt/chart/templates/peerauthentication-virt-api.yaml b/packages/kubevirt/chart/templates/peerauthentication-virt-api.yaml new file mode 100644 index 0000000000..80817f4e12 --- /dev/null +++ b/packages/kubevirt/chart/templates/peerauthentication-virt-api.yaml @@ -0,0 +1,17 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: virt-api-peerauth + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + kubevirt.io: virt-api + mtls: + mode: STRICT + portLevelMtls: + 8443: + mode: PERMISSIVE diff --git a/packages/kubevirt/chart/templates/servicemonitor.yaml b/packages/kubevirt/chart/templates/servicemonitor.yaml new file mode 100644 index 0000000000..ebde52f642 --- /dev/null +++ b/packages/kubevirt/chart/templates/servicemonitor.yaml @@ -0,0 +1,23 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kubevirt-metrics + namespace: {{ .Release.Namespace }} +spec: + endpoints: + - path: /metrics + port: metrics + scheme: https + tlsConfig: + ca: + secret: + key: tls.crt + name: kubevirt-ca + # KubeVirt certs do not contain a SAN and therefore cannot be verified by Prometheus + insecureSkipVerify: true + selector: + matchLabels: + prometheus.kubevirt.io: "true" diff --git a/packages/kubevirt/chart/templates/uds-exemption.yaml b/packages/kubevirt/chart/templates/uds-exemption.yaml new file mode 100644 index 0000000000..a4ac6d22af --- /dev/null +++ b/packages/kubevirt/chart/templates/uds-exemption.yaml @@ -0,0 +1,37 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: uds.dev/v1alpha1 +kind: Exemption +metadata: + name: kubevirt + namespace: uds-policy-exemptions +spec: + exemptions: + - policies: + - DisallowHostNamespaces + - DisallowPrivileged + - RestrictHostPathWrite + - RestrictVolumeTypes + - RequireNonRootUser + matcher: + namespace: kubevirt + name: "^virt-handler-.*" + title: "kubevirt virt-handler exemptions" + description: | + KubeVirt is a VM virtualization software and requires a broader + scope of exemptions than most packages. + DisallowPrivileged: + - The container must have access to virtualization hardware devices such as `/dev/kvm`. + RestrictHostPathWrite: + - `/var/run/kubevirt-libvirt-runtimes` (`libvirt-runtimes`): This path provides access to the runtime environment of `libvirt`, the virtualization tool KubeVirt relies on to manage virtual machine instances. It contains runtime information necessary for managing virtual machines, such as UNIX sockets and process state related to `libvirtd`. + - `/var/run/kubevirt` (`virt-share-dir`): This directory is used as a shared space between KubeVirt components, specifically to store information that needs to be exchanged between processes running on the same host (e.g., between `virt-handler` and `virt-launcher`). It typically holds sockets and temporary files that are critical for VM management during runtime. + - `/var/lib/kubevirt` (`virt-lib-dir`): This is the primary data directory for KubeVirt. It is where important VM-related metadata, logs, or disk images might be stored persistently. It can also be used for storing configuration files or caching important data for faster access. + - `/var/run/kubevirt-private` (`virt-private-dir`): This directory is used for private KubeVirt runtime files that need to remain isolated from other processes on the host for security or management reasons. This might include private UNIX domain sockets or other files that should only be accessed by KubeVirt components. + - `/var/lib/kubelet/device-plugins` (`device-plugin`): This directory is important for exposing device plugins to the Kubernetes `kubelet`, which is responsible for managing devices and scheduling workloads on the node. KubeVirt might rely on device plugins for accessing hardware like GPUs, SR-IOV network devices, or other specialized hardware needed by VMs. + - `/var/lib/kubelet/pods` (`kubelet-pods-shortened` and `kubelet-pods`): These paths are related to the Kubernetes node infrastructure and are used by KubeVirt to interact with the running Kubernetes pods on the node. KubeVirt needs access to these paths to correctly map and manage the pod environment for VMs, as each VM runs in a pod. This allows KubeVirt to manage pod lifecycle and resource allocation for the VMs. + - `/var/lib/kubevirt-node-labeller` (`node-labeller`): This path is used by the **node-labeller** component in KubeVirt, which inspects the host's hardware capabilities (e.g., CPU model, features like AVX or SSE, etc.) and labels the node accordingly. This ensures that VMs requiring specific hardware features are scheduled on compatible nodes. The node-labeller stores its metadata in this directory. + RestrictVolumeTypes: + - Allows for hostpath volumes to be created. + RequireNonRootUser: + - Required for privileged, see reason above. diff --git a/packages/kubevirt/chart/templates/uds-package.yaml b/packages/kubevirt/chart/templates/uds-package.yaml new file mode 100644 index 0000000000..5154866d65 --- /dev/null +++ b/packages/kubevirt/chart/templates/uds-package.yaml @@ -0,0 +1,89 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: uds.dev/v1alpha1 +kind: Package +metadata: + name: kubevirt + namespace: {{ .Release.Namespace }} +spec: + network: + serviceMesh: + mode: ambient + allow: + - direction: Ingress + remoteGenerated: IntraNamespace + + - direction: Ingress + remoteNamespace: monitoring + remoteSelector: + app: prometheus + selector: + prometheus.kubevirt.io: "true" + port: 8443 + description: Metrics + + - direction: Egress + remoteGenerated: IntraNamespace + + # Allow Kubernetes API server to connect to Kubevirt API webhook over HTTPS + - direction: Ingress + selector: + kubevirt.io: virt-api + # uds-core has a todo to evaluate a "KubeAPI" _ingress_ generated rule for webhook calls + # https://github.com/defenseunicorns/uds-core/blob/main/src/prometheus-stack/chart/templates/uds-package.yaml#L44-L50 + remoteGenerated: Anywhere + + - direction: Egress + selector: + kubevirt.io: virt-operator + remoteGenerated: KubeAPI + description: "virt-operator to KubeAPI" + + - direction: Egress + selector: + kubevirt.io: virt-operator-strategy-dumper + remoteGenerated: KubeAPI + description: "virt-operator-strategy-dumper to KubeAPI" + + - direction: Egress + selector: + kubevirt.io: virt-api + remoteGenerated: KubeAPI + description: "virt-api to KubeAPI" + + - direction: Egress + selector: + kubevirt.io: virt-controller + remoteGenerated: KubeAPI + description: "virt-controller to KubeAPI" + + - direction: Egress + selector: + kubevirt.io: virt-handler + remoteGenerated: KubeAPI + description: "virt-handler to KubeAPI" + + # virt-exportproxy is deployed when the VMExport feature gate is enabled. + # It watches VirtualMachineExport resources and proxies export server traffic. + - direction: Egress + selector: + kubevirt.io: virt-exportproxy + remoteGenerated: KubeAPI + description: "virt-exportproxy to KubeAPI" + + # Custom rules for unanticipated scenarios + {{- range .Values.additionalNetworkAllow }} + - direction: {{ .direction }} + selector: + {{ .selector | toYaml | nindent 10 }} + {{- if not .remoteGenerated }} + remoteNamespace: {{ .remoteNamespace }} + remoteSelector: + {{ .remoteSelector | toYaml | nindent 10 }} + port: {{ .port }} + {{- else }} + remoteGenerated: {{ .remoteGenerated }} + {{- end }} + description: {{ .description }} + {{- end }} diff --git a/packages/kubevirt/chart/values.yaml b/packages/kubevirt/chart/values.yaml new file mode 100644 index 0000000000..d2de55889e --- /dev/null +++ b/packages/kubevirt/chart/values.yaml @@ -0,0 +1,22 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +additionalNetworkAllow: [] +kubevirt: + enabled: true + configuration: {} + extraValues: {} + infra: + nodePlacement: {} + workloads: + nodePlacement: {} +cdi: + enabled: true + scratchSpaceStorageClass: "" + infra: + nodeSelector: {} + workload: + nodeSelector: {} +image: + registry: "" + tag: "" diff --git a/packages/kubevirt/tasks.yaml b/packages/kubevirt/tasks.yaml new file mode 100644 index 0000000000..d0511b3aa4 --- /dev/null +++ b/packages/kubevirt/tasks.yaml @@ -0,0 +1,299 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +tasks: + - name: all + actions: + - task: health-check + - task: vm-test + - task: live-migration-test + - task: podinfo-vm-test + + - name: health-check + actions: + - description: virt-api Health Check + wait: + cluster: + kind: Deployment + name: virt-api + namespace: kubevirt + condition: Available + - description: virt-controller Health Check + wait: + cluster: + kind: Deployment + name: virt-controller + namespace: kubevirt + condition: Available + - description: virt-operator Health Check + wait: + cluster: + kind: Deployment + name: virt-operator + namespace: kubevirt + condition: Available + - description: virt-handler Health Check + wait: + cluster: + kind: DaemonSet + name: virt-handler + namespace: kubevirt + - description: KubeVirt CR Health Check + wait: + cluster: + kind: KubeVirt + name: kubevirt + namespace: kubevirt + condition: Available + - description: cdi-operator Health Check + wait: + cluster: + kind: Deployment + name: cdi-operator + namespace: cdi + condition: Available + - description: CDI CR Health Check + wait: + cluster: + kind: CDI + name: cdi + condition: Available + + - name: vm-test + description: "Deploy and verify a basic container-disk VM" + actions: + - description: Create vm-test zarf package + cmd: ./uds zarf package create packages/kubevirt/tests/vm-test --architecture="${ZARF_ARCHITECTURE}" --confirm --no-progress --output build + - description: Deploy vm-test zarf package + cmd: ./uds zarf package deploy build/zarf-package-vm-test-"${ZARF_ARCHITECTURE}"-0.0.1.tar.zst --confirm --no-progress + - description: Start testvm + cmd: ./uds zarf tools kubectl patch virtualmachine testvm -n vm-test --type merge -p '{"spec":{"runStrategy":"Always"}}' + - description: Wait for testvm VirtualMachine to be ready + wait: + cluster: + kind: VirtualMachine + name: testvm + namespace: vm-test + condition: Ready + - description: Wait for testvm VirtualMachineInstance to be ready + wait: + cluster: + kind: VirtualMachineInstance + name: testvm + namespace: vm-test + condition: Ready + - description: Test HTTP server running in VM + cmd: | + ./uds zarf tools kubectl port-forward svc/testvm-svc -n vm-test 58835:80 & + PORT_FORWARD_PID=$! + sleep 5 + HTTP_STATUS=$(curl -o /dev/null -s -w "%{http_code}\n" http://localhost:58835) + kill $PORT_FORWARD_PID + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "HTTP test passed." + else + echo "HTTP test failed. status=${HTTP_STATUS}" + exit 1 + fi + + - name: podinfo-vm-test + description: "Deploy and verify a UDS-managed VM app with Istio sidecar (requires native KubeVirt Pepr support)" + actions: + - description: Create podinfo-vm zarf package + cmd: ./uds zarf package create packages/kubevirt/tests/podinfo-vm --architecture="${ZARF_ARCHITECTURE}" --confirm --no-progress --output build + - description: Deploy podinfo-vm zarf package + cmd: ./uds zarf package deploy build/zarf-package-podinfo-vm-"${ZARF_ARCHITECTURE}"-0.0.1.tar.zst --confirm --no-progress + - description: Wait for podinfo-vm UDS Package to be ready + wait: + cluster: + kind: Package + name: podinfo-vm + namespace: podinfo-vm + condition: Ready + - description: Verify podinfo-vm namespace registry secret + cmd: ./uds zarf tools kubectl get secret private-registry -n podinfo-vm + - description: Verify podinfo-vm uses native KubeVirt policy allowance path + cmd: | + ./uds zarf tools kubectl get namespace podinfo-vm -o jsonpath='{.metadata.labels.uds\.dev/kubevirt-workload}' | grep true + if ./uds zarf tools kubectl get exemption kubevirt -n uds-policy-exemptions -o yaml | grep -q 'namespace: podinfo-vm'; then + echo "podinfo-vm still has centralized exemption" + exit 1 + fi + - description: Start podinfo-vm + cmd: ./uds zarf tools kubectl patch virtualmachine podinfo-vm -n podinfo-vm --type merge -p '{"spec":{"runStrategy":"Always"}}' + - description: Wait for podinfo-vm VirtualMachine to be ready + wait: + cluster: + kind: VirtualMachine + name: podinfo-vm + namespace: podinfo-vm + condition: Ready + - description: Wait for podinfo-vm VirtualMachineInstance to be ready + wait: + cluster: + kind: VirtualMachineInstance + name: podinfo-vm + namespace: podinfo-vm + condition: Ready + - description: Verify virt-launcher Istio sidecar and native policy allowance + cmd: | + POD_NAME=$(./uds zarf tools kubectl get pod -n podinfo-vm -l app.kubernetes.io/name=podinfo-vm -o jsonpath='{.items[0].metadata.name}') + echo "virt-launcher pod: ${POD_NAME}" + echo "${POD_NAME}" | grep '^virt-launcher-' + ./uds zarf tools kubectl get pod "${POD_NAME}" -n podinfo-vm -o jsonpath='{.metadata.annotations.traffic\.sidecar\.istio\.io/kubevirtInterfaces}' | grep k6t-eth0 + PEPR_EXEMPTION=$(./uds zarf tools kubectl get pod "${POD_NAME}" -n podinfo-vm -o jsonpath='{.metadata.annotations.uds-core\.pepr\.dev/uds-core-policies\.RestrictIstioTrafficOverrides}') + if [ -n "${PEPR_EXEMPTION}" ]; then + echo "unexpected Pepr exemption annotation: ${PEPR_EXEMPTION}" + exit 1 + fi + ./uds zarf tools kubectl get pod "${POD_NAME}" -n podinfo-vm -o jsonpath='{.spec.initContainers[*].name}' | grep istio-proxy + - description: Verify non-virt-launcher pod cannot use KubeVirt Istio annotation + cmd: | + if ./uds zarf tools kubectl apply -f packages/kubevirt/tests/podinfo-vm/non-virt-kubevirt-annotation-pod.yaml >/tmp/non-virt-kubevirt-annotation.out 2>&1; then + echo "non-virt-launcher pod unexpectedly admitted" + exit 1 + fi + grep 'traffic.sidecar.istio.io/kubevirtInterfaces' /tmp/non-virt-kubevirt-annotation.out + - description: Verify non-importer pod cannot disable Istio sidecar + cmd: | + if ./uds zarf tools kubectl run non-importer-sidecar-disable -n podinfo-vm --image=busybox:1.37.0 --restart=Never --overrides='{"metadata":{"annotations":{"sidecar.istio.io/inject":"false"}},"spec":{"containers":[{"name":"non-importer-sidecar-disable","image":"busybox:1.37.0","command":["sh","-c","sleep 30"]}]}}' >/tmp/non-importer-sidecar-disable.out 2>&1; then + echo "non-importer pod unexpectedly admitted" + ./uds zarf tools kubectl delete pod non-importer-sidecar-disable -n podinfo-vm --ignore-not-found + exit 1 + fi + grep 'sidecar.istio.io/inject' /tmp/non-importer-sidecar-disable.out + - description: Wait for CDI DataVolume import + cmd: ./uds zarf tools kubectl wait datavolume cirros-import -n podinfo-vm --for=jsonpath='{.status.phase}'=Succeeded --timeout=900s + - description: Verify CDI DataVolume import artifacts + cmd: | + ./uds zarf tools kubectl get datavolume cirros-import -n podinfo-vm -o yaml + ./uds zarf tools kubectl get pvc cirros-import -n podinfo-vm + - description: Start PVC-backed cirros VM + cmd: ./uds zarf tools kubectl patch virtualmachine cirros-pvc-vm -n podinfo-vm --type merge -p '{"spec":{"runStrategy":"Always"}}' + - description: Wait for PVC-backed cirros VMI to be ready + wait: + cluster: + kind: VirtualMachineInstance + name: cirros-pvc-vm + namespace: podinfo-vm + condition: Ready + - description: Test HTTP server running in podinfo-vm via Service + cmd: | + ./uds zarf tools kubectl port-forward svc/podinfo-vm -n podinfo-vm 58836:80 & + PORT_FORWARD_PID=$! + sleep 5 + HTTP_STATUS=$(curl -o /dev/null -s -w "%{http_code}\n" http://localhost:58836) + BODY=$(curl -s http://localhost:58836) + kill $PORT_FORWARD_PID + if [ "$HTTP_STATUS" -eq 200 ] && echo "$BODY" | grep -q 'podinfo-vm'; then + echo "podinfo-vm service test passed." + else + echo "podinfo-vm service test failed. status=${HTTP_STATUS} body=${BODY}" + exit 1 + fi + - description: Test podinfo-vm through tenant gateway + cmd: | + HTTP_STATUS=$(curl -k -o /dev/null -s -w "%{http_code}\n" --resolve podinfo-vm.uds.dev:443:127.0.0.1 https://podinfo-vm.uds.dev) + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "podinfo-vm gateway test passed." + else + echo "podinfo-vm gateway test failed. status=${HTTP_STATUS}" + exit 1 + fi + - description: Verify ServiceMonitor target is up in Prometheus (skipped if monitoring not deployed) + cmd: | + if ! ./uds zarf tools kubectl get deployment kube-prometheus-stack-prometheus -n monitoring >/dev/null 2>&1; then + echo "Skipping: monitoring stack not deployed. Redeploy with the full standard bundle to test monitoring." + exit 0 + fi + PROM_POD=$(./uds zarf tools kubectl get pod -n monitoring -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[0].metadata.name}') + for i in $(seq 1 12); do + RESULT=$(./uds zarf tools kubectl exec -n monitoring "${PROM_POD}" -c prometheus -- \ + wget -q -O- 'http://localhost:9090/api/v1/targets?state=active' 2>/dev/null | \ + python3 -c " + import sys, json + data = json.load(sys.stdin) + targets = [t for t in data['data']['activeTargets'] if t.get('labels',{}).get('namespace') == 'podinfo-vm'] + up = [t for t in targets if t['health'] == 'up'] + print(f'{len(up)}/{len(targets)}') + " 2>/dev/null || echo "0/0") + UP=$(echo "$RESULT" | cut -d'/' -f1) + if [ "$UP" -gt 0 ] 2>/dev/null; then + echo "monitoring OK: ${RESULT} podinfo-vm target(s) up" + exit 0 + fi + echo "waiting for podinfo-vm Prometheus target (attempt ${i}/12): ${RESULT}" + sleep 5 + done + echo "ERROR: no podinfo-vm targets found up in Prometheus after 60s" + exit 1 + + - name: live-migration-test + description: "Trigger and verify a container disk VM live migration (requires 2+ nodes; run after vm-test)" + actions: + - description: Verify at least two nodes are available + cmd: | + NODE_COUNT=$(./uds zarf tools kubectl get nodes --no-headers | wc -l) + if [ "$NODE_COUNT" -lt 2 ]; then + echo "live-migration-test requires 2+ nodes, found ${NODE_COUNT} — skipping" + exit 0 + fi + echo "node count OK: ${NODE_COUNT}" + - description: Wait for testvm VMI to be Running + wait: + cluster: + kind: VirtualMachineInstance + name: testvm + namespace: vm-test + condition: Ready + - description: Verify testvm is live-migratable + cmd: | + LIVE=$(./uds zarf tools kubectl get vmi testvm -n vm-test \ + -o jsonpath='{.status.conditions[?(@.type=="LiveMigratable")].status}') + if [ "$LIVE" != "True" ]; then + echo "testvm is not live-migratable: ${LIVE}" + ./uds zarf tools kubectl get vmi testvm -n vm-test -o jsonpath='{.status.conditions}' | python3 -m json.tool + exit 1 + fi + echo "testvm is live-migratable" + - description: Record source node before migration + cmd: | + SOURCE_NODE=$(./uds zarf tools kubectl get vmi testvm -n vm-test -o jsonpath='{.status.nodeName}') + echo "${SOURCE_NODE}" > /tmp/testvm-source-node + echo "source node: ${SOURCE_NODE}" + - description: Trigger live migration + cmd: | + ./uds zarf tools kubectl apply -f - <<'EOF' + apiVersion: kubevirt.io/v1 + kind: VirtualMachineInstanceMigration + metadata: + name: testvm-migration + namespace: vm-test + spec: + vmiName: testvm + EOF + - description: Wait for migration to Succeed + cmd: | + ./uds zarf tools kubectl wait vmim testvm-migration -n vm-test \ + --for=jsonpath='{.status.phase}'=Succeeded \ + --timeout=300s + echo "migration Succeeded" + - description: Verify VM moved to a different node + cmd: | + SOURCE_NODE=$(cat /tmp/testvm-source-node) + TARGET_NODE=$(./uds zarf tools kubectl get vmi testvm -n vm-test -o jsonpath='{.status.nodeName}') + echo "source: ${SOURCE_NODE} → target: ${TARGET_NODE}" + if [ "${SOURCE_NODE}" = "${TARGET_NODE}" ]; then + echo "ERROR: VM is still on the same node after migration" + exit 1 + fi + echo "live-migration-test passed" + - description: Verify Service endpoint tracks the new pod + cmd: | + ENDPOINT=$(./uds zarf tools kubectl get endpoints testvm-svc -n vm-test \ + -o jsonpath='{.subsets[0].addresses[0].ip}') + if [ -z "${ENDPOINT}" ]; then + echo "ERROR: Service endpoint is empty after migration" + exit 1 + fi + echo "Service endpoint OK: ${ENDPOINT}" diff --git a/packages/kubevirt/tests/podinfo-vm/datavolume.yaml b/packages/kubevirt/tests/podinfo-vm/datavolume.yaml new file mode 100644 index 0000000000..cff131f855 --- /dev/null +++ b/packages/kubevirt/tests/podinfo-vm/datavolume.yaml @@ -0,0 +1,19 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: cdi.kubevirt.io/v1beta1 +kind: DataVolume +metadata: + name: cirros-import + namespace: podinfo-vm +spec: + source: + registry: + url: "docker://###ZARF_REGISTRY###/kubevirt/cirros-container-disk-demo:latest" + pullMethod: node + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/packages/kubevirt/tests/podinfo-vm/namespace.yaml b/packages/kubevirt/tests/podinfo-vm/namespace.yaml new file mode 100644 index 0000000000..a0991b82bb --- /dev/null +++ b/packages/kubevirt/tests/podinfo-vm/namespace.yaml @@ -0,0 +1,9 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: v1 +kind: Namespace +metadata: + name: podinfo-vm + labels: + uds.dev/kubevirt-workload: "true" diff --git a/packages/kubevirt/tests/podinfo-vm/non-virt-kubevirt-annotation-pod.yaml b/packages/kubevirt/tests/podinfo-vm/non-virt-kubevirt-annotation-pod.yaml new file mode 100644 index 0000000000..79b617d08c --- /dev/null +++ b/packages/kubevirt/tests/podinfo-vm/non-virt-kubevirt-annotation-pod.yaml @@ -0,0 +1,15 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: v1 +kind: Pod +metadata: + name: non-virt-kubevirt-annotation + namespace: podinfo-vm + annotations: + traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0 +spec: + restartPolicy: Never + containers: + - name: pause + image: registry.k8s.io/pause:3.10 diff --git a/packages/kubevirt/tests/podinfo-vm/pvc-vm.yaml b/packages/kubevirt/tests/podinfo-vm/pvc-vm.yaml new file mode 100644 index 0000000000..8fba52ecae --- /dev/null +++ b/packages/kubevirt/tests/podinfo-vm/pvc-vm.yaml @@ -0,0 +1,44 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: cirros-pvc-vm + namespace: podinfo-vm + labels: + app.kubernetes.io/name: cirros-pvc-vm +spec: + runStrategy: Halted + template: + metadata: + annotations: + sidecar.istio.io/inject: "true" + traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0 + labels: + app.kubernetes.io/name: cirros-pvc-vm + kubevirt.io/domain: cirros-pvc-vm + spec: + domain: + devices: + interfaces: + - name: default + masquerade: {} + ports: + - name: ssh + port: 22 + disks: + - disk: + bus: virtio + name: rootdisk + resources: + requests: + memory: 256M + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 0 + volumes: + - name: rootdisk + persistentVolumeClaim: + claimName: cirros-import diff --git a/packages/kubevirt/tests/podinfo-vm/svc.yaml b/packages/kubevirt/tests/podinfo-vm/svc.yaml new file mode 100644 index 0000000000..d2887dbc8a --- /dev/null +++ b/packages/kubevirt/tests/podinfo-vm/svc.yaml @@ -0,0 +1,18 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: v1 +kind: Service +metadata: + name: podinfo-vm + namespace: podinfo-vm + labels: + app.kubernetes.io/name: podinfo-vm +spec: + selector: + app.kubernetes.io/name: podinfo-vm + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP diff --git a/packages/kubevirt/tests/podinfo-vm/uds-package.yaml b/packages/kubevirt/tests/podinfo-vm/uds-package.yaml new file mode 100644 index 0000000000..97cb2f54bf --- /dev/null +++ b/packages/kubevirt/tests/podinfo-vm/uds-package.yaml @@ -0,0 +1,41 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: uds.dev/v1alpha1 +kind: Package +metadata: + name: podinfo-vm + namespace: podinfo-vm +spec: + # The operator labels the namespace uds.dev/kubevirt-workload=true from this marker. The + # namespace manifest also sets the label directly so the deploy-time CDI importer pod is + # admitted before the operator reconciles this Package. + kubevirt: + enabled: true + network: + serviceMesh: + mode: sidecar + expose: + - service: podinfo-vm + selector: + app.kubernetes.io/name: podinfo-vm + gateway: tenant + host: podinfo-vm + port: 80 + targetPort: 80 + uptime: + checks: + paths: + - / + allow: + - direction: Ingress + remoteGenerated: IntraNamespace + - direction: Egress + remoteGenerated: IntraNamespace + monitor: + - selector: + app.kubernetes.io/name: podinfo-vm + targetPort: 80 + portName: http + description: "podinfo-vm service monitor" + kind: ServiceMonitor diff --git a/packages/kubevirt/tests/podinfo-vm/vm.yaml b/packages/kubevirt/tests/podinfo-vm/vm.yaml new file mode 100644 index 0000000000..25c8817425 --- /dev/null +++ b/packages/kubevirt/tests/podinfo-vm/vm.yaml @@ -0,0 +1,89 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: podinfo-vm + namespace: podinfo-vm + labels: + app.kubernetes.io/name: podinfo-vm +spec: + runStrategy: Halted + template: + metadata: + annotations: + sidecar.istio.io/inject: "true" + traffic.sidecar.istio.io/kubevirtInterfaces: k6t-eth0 + labels: + app.kubernetes.io/name: podinfo-vm + kubevirt.io/domain: podinfo-vm + spec: + domain: + devices: + interfaces: + - name: default + masquerade: {} + ports: + - name: http + port: 80 + disks: + - disk: + bus: virtio + name: containerdisk + - disk: + bus: virtio + name: cloudinitdisk + resources: + requests: + memory: 1024M + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 0 + volumes: + - name: containerdisk + containerDisk: + image: kubevirt/fedora-cloud-container-disk-demo + - name: cloudinitdisk + cloudInitNoCloud: + userData: |- + #cloud-config + password: fedora + chpasswd: { expire: False } + write_files: + - path: /usr/local/bin/podinfo-vm.py + permissions: '0755' + content: | + #!/usr/bin/env python3 + import json + from http.server import BaseHTTPRequestHandler, HTTPServer + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = json.dumps({"app": "podinfo-vm", "status": "ok"}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + HTTPServer(("0.0.0.0", 80), Handler).serve_forever() + - path: /etc/systemd/system/podinfo-vm.service + permissions: '0644' + content: | + [Unit] + Description=Podinfo VM HTTP service + After=network-online.target + Wants=network-online.target + + [Service] + ExecStart=/usr/local/bin/podinfo-vm.py + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target + runcmd: + - [ systemctl, daemon-reload ] + - [ systemctl, enable, --now, podinfo-vm.service ] diff --git a/packages/kubevirt/tests/podinfo-vm/zarf.yaml b/packages/kubevirt/tests/podinfo-vm/zarf.yaml new file mode 100644 index 0000000000..28de7fdb1f --- /dev/null +++ b/packages/kubevirt/tests/podinfo-vm/zarf.yaml @@ -0,0 +1,28 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/zarf/main/zarf.schema.json +kind: ZarfPackageConfig +metadata: + name: podinfo-vm + description: "Tests a UDS-managed VM app with KubeVirt and Istio sidecar support" + architecture: "amd64" + version: "0.0.1" + +components: + - name: podinfo-vm + required: true + manifests: + - name: podinfo-vm + namespace: podinfo-vm + noWait: true + files: + - namespace.yaml + - uds-package.yaml + - svc.yaml + - datavolume.yaml + - pvc-vm.yaml + - vm.yaml + images: + - kubevirt/fedora-cloud-container-disk-demo + - quay.io/kubevirt/cirros-container-disk-demo:latest diff --git a/packages/kubevirt/tests/vm-test/namespace.yaml b/packages/kubevirt/tests/vm-test/namespace.yaml new file mode 100644 index 0000000000..9a513df05d --- /dev/null +++ b/packages/kubevirt/tests/vm-test/namespace.yaml @@ -0,0 +1,10 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: v1 +kind: Namespace +metadata: + name: vm-test + labels: + istio-injection: enabled + uds.dev/kubevirt-workload: "true" diff --git a/packages/kubevirt/tests/vm-test/svc.yaml b/packages/kubevirt/tests/vm-test/svc.yaml new file mode 100644 index 0000000000..480afa4ab7 --- /dev/null +++ b/packages/kubevirt/tests/vm-test/svc.yaml @@ -0,0 +1,13 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: v1 +kind: Service +metadata: + name: testvm-svc +spec: + selector: + kubevirt.io/domain: testvm + ports: + - port: 80 + protocol: TCP diff --git a/packages/kubevirt/tests/vm-test/vm.yaml b/packages/kubevirt/tests/vm-test/vm.yaml new file mode 100644 index 0000000000..cd7afb88b2 --- /dev/null +++ b/packages/kubevirt/tests/vm-test/vm.yaml @@ -0,0 +1,60 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: testvm +spec: + runStrategy: Halted + template: + metadata: + labels: + kubevirt.io/size: small + kubevirt.io/domain: testvm + annotations: + sidecar.istio.io/inject: "true" + spec: + domain: + devices: + interfaces: + - name: default + masquerade: {} + ports: + - port: 80 + disks: + - disk: + bus: virtio + name: containerdisk + - disk: + bus: virtio + name: cloudinitdisk + resources: + requests: + memory: 1024M + networks: + - name: default + pod: {} + readinessProbe: + httpGet: + port: 80 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 3 + terminationGracePeriodSeconds: 0 + volumes: + - name: containerdisk + containerDisk: + image: kubevirt/fedora-cloud-container-disk-demo + - cloudInitNoCloud: + userData: |- + #cloud-config + password: fedora + chpasswd: { expire: False } + packages: + - nginx + runcmd: + - [ "systemctl", "enable", "--now", "nginx" ] + name: cloudinitdisk diff --git a/packages/kubevirt/tests/vm-test/zarf.yaml b/packages/kubevirt/tests/vm-test/zarf.yaml new file mode 100644 index 0000000000..5b4f959b8a --- /dev/null +++ b/packages/kubevirt/tests/vm-test/zarf.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/zarf/main/zarf.schema.json +kind: ZarfPackageConfig +metadata: + name: vm-test + description: "Tests functionality of the VM CRD provided by KubeVirt" + architecture: "amd64" + version: "0.0.1" + +components: + - name: vm-test + required: true + manifests: + - name: vm-test + namespace: vm-test + noWait: true + files: + - namespace.yaml + - vm.yaml + - svc.yaml + images: + - kubevirt/fedora-cloud-container-disk-demo diff --git a/packages/kubevirt/values/common-values.yaml b/packages/kubevirt/values/common-values.yaml new file mode 100644 index 0000000000..04a6cf3f78 --- /dev/null +++ b/packages/kubevirt/values/common-values.yaml @@ -0,0 +1,13 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +kubevirt: + configuration: + developerConfiguration: + useEmulation: false + featureGates: + - VMExport + extraValues: {} +cdi: + enabled: true + scratchSpaceStorageClass: "" diff --git a/packages/kubevirt/values/upstream-values.yaml b/packages/kubevirt/values/upstream-values.yaml new file mode 100644 index 0000000000..89c89c7308 --- /dev/null +++ b/packages/kubevirt/values/upstream-values.yaml @@ -0,0 +1,7 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +image: + registry: quay.io/kubevirt + # renovate: datasource=docker depName=quay.io/kubevirt/virt-api + tag: v1.8.2 diff --git a/packages/kubevirt/zarf.yaml b/packages/kubevirt/zarf.yaml new file mode 100644 index 0000000000..7c1b7b4df1 --- /dev/null +++ b/packages/kubevirt/zarf.yaml @@ -0,0 +1,99 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/zarf/main/zarf.schema.json +kind: ZarfPackageConfig +metadata: + name: core-kubevirt + description: "KubeVirt and CDI for UDS Core — VM workload support with native Istio mesh integration" + authors: "Defense Unicorns - Product" + # x-release-please-start-version + version: "1.7.0" + # x-release-please-end + annotations: + dev.uds.title: UDS Core (KubeVirt) + dev.uds.tagline: Native VM workload support for UDS Core + dev.uds.categories: UDS-Core,Virtualization + dev.uds.keywords: uds,kubevirt,cdi,virtualmachine,vm + +components: + - name: kubevirt-crd + required: true + description: "KubeVirt and CDI CRDs and operator manifests" + only: + flavor: upstream + manifests: + - name: crds + files: + # renovate: datasource=github-releases depName=kubevirt/kubevirt + - https://github.com/kubevirt/kubevirt/releases/download/v1.8.2/kubevirt-operator.yaml + # renovate: datasource=github-releases depName=kubevirt/containerized-data-importer + - https://github.com/kubevirt/containerized-data-importer/releases/download/v1.65.0/cdi-operator.yaml + images: + # renovate: datasource=docker depName=quay.io/kubevirt/virt-operator versioning=semver + - quay.io/kubevirt/virt-operator:v1.8.2 + - quay.io/kubevirt/virt-api:v1.8.2 + - quay.io/kubevirt/virt-controller:v1.8.2 + - quay.io/kubevirt/virt-launcher:v1.8.2 + - quay.io/kubevirt/virt-handler:v1.8.2 + - quay.io/kubevirt/virtio-container-disk:v1.8.2 + - quay.io/kubevirt/virt-exportproxy:v1.8.2 + - quay.io/kubevirt/virt-exportserver:v1.8.2 + # renovate: datasource=docker depName=quay.io/kubevirt/cdi-operator versioning=semver + - quay.io/kubevirt/cdi-operator:v1.65.0 + - quay.io/kubevirt/cdi-controller:v1.65.0 + - quay.io/kubevirt/cdi-importer:v1.65.0 + - quay.io/kubevirt/cdi-cloner:v1.65.0 + - quay.io/kubevirt/cdi-apiserver:v1.65.0 + - quay.io/kubevirt/cdi-uploadserver:v1.65.0 + - quay.io/kubevirt/cdi-uploadproxy:v1.65.0 + - quay.io/kubevirt/cirros-container-disk-demo:latest + - quay.io/containerdisks/fedora:latest + + - name: kubevirt + required: true + description: "KubeVirt and CDI UDS Package configuration" + only: + flavor: upstream + charts: + - name: uds-kubevirt-config + namespace: kubevirt + version: 0.1.0 + localPath: chart + valuesFiles: + - values/common-values.yaml + - values/upstream-values.yaml + actions: + onDeploy: + after: + - description: Validate KubeVirt UDS Package + maxTotalSeconds: 300 + wait: + cluster: + kind: packages.uds.dev + name: kubevirt + namespace: kubevirt + condition: "'{.status.phase}'=Ready" + - description: Validate CDI UDS Package + maxTotalSeconds: 300 + wait: + cluster: + kind: packages.uds.dev + name: cdi + namespace: cdi + condition: "'{.status.phase}'=Ready" + - description: Validate KubeVirt CR + maxTotalSeconds: 300 + wait: + cluster: + kind: KubeVirt + name: kubevirt + namespace: kubevirt + condition: Available + - description: Validate CDI CR + maxTotalSeconds: 300 + wait: + cluster: + kind: CDI + name: cdi + condition: Available diff --git a/schemas/package-v1alpha1.schema.json b/schemas/package-v1alpha1.schema.json index cadda456ac..0cb2d04976 100644 --- a/schemas/package-v1alpha1.schema.json +++ b/schemas/package-v1alpha1.schema.json @@ -24,6 +24,10 @@ "$ref": "#/definitions/CABundle", "description": "CA bundle configuration for the package" }, + "kubevirt": { + "$ref": "#/definitions/Kubevirt", + "description": "KubeVirt workload settings. When enabled, the UDS operator labels the namespace\n'uds.dev/kubevirt-workload: \"true\"' so the KubeVirt and CDI Pepr policy allowances apply\nto the namespace's generated pods (virt-launcher, CDI importer/upload/clone)." + }, "monitor": { "type": "array", "items": { @@ -90,6 +94,19 @@ "title": "ConfigMap", "description": "ConfigMap configuration for CA bundle" }, + "Kubevirt": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable KubeVirt workload support for this namespace. Labels the namespace so KubeVirt and\nCDI generated pods are permitted their required Istio annotations." + } + }, + "required": [], + "title": "Kubevirt", + "description": "KubeVirt workload settings. When enabled, the UDS operator labels the namespace 'uds.dev/kubevirt-workload: \"true\"' so the KubeVirt and CDI Pepr policy allowances apply to the namespace's generated pods (virt-launcher, CDI importer/upload/clone)." + }, "Monitor": { "type": "object", "additionalProperties": false, diff --git a/src/pepr/operator/controllers/istio/namespace.spec.ts b/src/pepr/operator/controllers/istio/namespace.spec.ts index 276ce103d1..cd7d47faff 100644 --- a/src/pepr/operator/controllers/istio/namespace.spec.ts +++ b/src/pepr/operator/controllers/istio/namespace.spec.ts @@ -305,6 +305,89 @@ describe("enableIstio", () => { const applyCall = mockApply.mock.calls[0][0]; expect(applyCall.metadata.annotations["uds.dev/original-istio-state"]).toBe("none"); }); + + test("sets the kubevirt-workload label when spec.kubevirt.enabled is true", async () => { + mockGet.mockResolvedValue({ metadata: { name: "test-ns", labels: {}, annotations: {} } }); + + await enableIstio({ + metadata: { name: "test-pkg", namespace: "test-ns" }, + spec: { kubevirt: { enabled: true } }, + } as UDSPackage); + + expect(mockApply).toHaveBeenCalled(); + const applyCall = mockApply.mock.calls[0][0]; + expect(applyCall.metadata.labels).toHaveProperty("uds.dev/kubevirt-workload", "true"); + }); + + test("does not set the kubevirt-workload label when the marker is absent", async () => { + mockGet.mockResolvedValue({ metadata: { name: "test-ns", labels: {}, annotations: {} } }); + + await enableIstio({ + metadata: { name: "test-pkg", namespace: "test-ns" }, + spec: {}, + } as UDSPackage); + + expect(mockApply).toHaveBeenCalled(); + const applyCall = mockApply.mock.calls[0][0]; + expect(applyCall.metadata.labels).not.toHaveProperty("uds.dev/kubevirt-workload"); + }); + + test("removes the kubevirt-workload label when the marker is cleared", async () => { + mockGet.mockResolvedValue({ + metadata: { + name: "test-ns", + labels: { + "istio.io/dataplane-mode": "ambient", + "uds.dev/kubevirt-workload": "true", + }, + annotations: { + "uds.dev/pkg-test-pkg": "true", + "uds.dev/kubevirt-pkg-test-pkg": "true", + "uds.dev/original-istio-state": IstioState.None, + }, + }, + }); + + await enableIstio({ + metadata: { name: "test-pkg", namespace: "test-ns" }, + spec: {}, + } as UDSPackage); + + expect(mockApply).toHaveBeenCalled(); + const applyCall = mockApply.mock.calls[0][0]; + expect(applyCall.metadata.labels).not.toHaveProperty("uds.dev/kubevirt-workload"); + }); + + test("preserves the kubevirt-workload label when a non-kubevirt package reconciles", async () => { + // The namespace label is backed by another package's intent annotation. A package without the + // marker reconciling must not strip it (the multi-package flip-flop the per-package fix closes). + mockGet.mockResolvedValue({ + metadata: { + name: "test-ns", + labels: { + "istio.io/dataplane-mode": "ambient", + "uds.dev/kubevirt-workload": "true", + }, + annotations: { + "uds.dev/pkg-other-pkg": "true", + "uds.dev/kubevirt-pkg-other-pkg": "true", + "uds.dev/original-istio-state": IstioState.None, + }, + }, + }); + + await enableIstio({ + metadata: { name: "test-pkg", namespace: "test-ns" }, + spec: {}, + } as UDSPackage); + + const applied = mockApply.mock.calls[0]?.[0]; + if (applied) { + expect(applied.metadata.labels).toHaveProperty("uds.dev/kubevirt-workload", "true"); + } else { + expect(mockApply).not.toHaveBeenCalled(); + } + }); }); describe("cleanupNamespace", () => { @@ -496,6 +579,78 @@ describe("cleanupNamespace", () => { expect(applyCall.metadata.annotations["uds.dev/pkg-test-pkg"]).toBeUndefined(); expect(applyCall.metadata.annotations["uds.dev/pkg-other-pkg"]).toBe("true"); }); + + test("removes the kubevirt-workload label on last package cleanup", async () => { + mockGet.mockResolvedValue({ + metadata: { + labels: { + "istio.io/dataplane-mode": "ambient", + "uds.dev/kubevirt-workload": "true", + }, + annotations: { + "uds.dev/pkg-test-pkg": "true", + "uds.dev/kubevirt-pkg-test-pkg": "true", + "uds.dev/original-istio-state": IstioState.Ambient, + }, + }, + }); + const pkg: UDSPackage = { metadata: { namespace: "test-ns", name: "test-pkg" } }; + + await cleanupNamespace(pkg); + + expect(mockApply).toHaveBeenCalled(); + const applyCall = mockApply.mock.calls[0][0]; + expect(applyCall.metadata.labels).not.toHaveProperty("uds.dev/kubevirt-workload"); + }); + + test("keeps the kubevirt-workload label when another kubevirt package remains", async () => { + mockGet.mockResolvedValue({ + metadata: { + labels: { + "istio.io/dataplane-mode": "ambient", + "uds.dev/kubevirt-workload": "true", + }, + annotations: { + "uds.dev/pkg-test-pkg": "true", + "uds.dev/kubevirt-pkg-test-pkg": "true", + "uds.dev/pkg-other-pkg": "true", + "uds.dev/kubevirt-pkg-other-pkg": "true", + "uds.dev/original-istio-state": IstioState.Ambient, + }, + }, + }); + const pkg: UDSPackage = { metadata: { namespace: "test-ns", name: "test-pkg" } }; + + await cleanupNamespace(pkg); + + expect(mockApply).toHaveBeenCalled(); + const applyCall = mockApply.mock.calls[0][0]; + expect(applyCall.metadata.labels).toHaveProperty("uds.dev/kubevirt-workload", "true"); + }); + + test("removes the kubevirt-workload label when remaining packages are not kubevirt", async () => { + mockGet.mockResolvedValue({ + metadata: { + labels: { + "istio.io/dataplane-mode": "ambient", + "uds.dev/kubevirt-workload": "true", + }, + annotations: { + "uds.dev/pkg-test-pkg": "true", + "uds.dev/kubevirt-pkg-test-pkg": "true", + "uds.dev/pkg-other-pkg": "true", + "uds.dev/original-istio-state": IstioState.Ambient, + }, + }, + }); + const pkg: UDSPackage = { metadata: { namespace: "test-ns", name: "test-pkg" } }; + + await cleanupNamespace(pkg); + + expect(mockApply).toHaveBeenCalled(); + const applyCall = mockApply.mock.calls[0][0]; + expect(applyCall.metadata.labels).not.toHaveProperty("uds.dev/kubevirt-workload"); + }); }); describe("killPods", () => { diff --git a/src/pepr/operator/controllers/istio/namespace.ts b/src/pepr/operator/controllers/istio/namespace.ts index a251657ec9..9f2c3ea786 100644 --- a/src/pepr/operator/controllers/istio/namespace.ts +++ b/src/pepr/operator/controllers/istio/namespace.ts @@ -15,8 +15,17 @@ const log = setupLogger(Component.OPERATOR_ISTIO); const INJECTION_LABEL = "istio-injection"; const AMBIENT_LABEL = "istio.io/dataplane-mode"; +const KUBEVIRT_WORKLOAD_LABEL = "uds.dev/kubevirt-workload"; +// Per-package record of kubevirt intent (one annotation per package that sets kubevirt.enabled). +// The workload label is derived from whether any of these exist, so packages in a shared namespace +// do not flip-flop the label. Mirrors the uds.dev/pkg- presence annotations. +const KUBEVIRT_PKG_ANNOTATION_PREFIX = "uds.dev/kubevirt-pkg-"; const ISTIO_STATE_ANNOTATION = "uds.dev/original-istio-state"; -const MANAGED_LABEL_FIELDS = new Set([`f:${INJECTION_LABEL}`, `f:${AMBIENT_LABEL}`]); +const MANAGED_LABEL_FIELDS = new Set([ + `f:${INJECTION_LABEL}`, + `f:${AMBIENT_LABEL}`, + `f:${KUBEVIRT_WORKLOAD_LABEL}`, +]); export enum IstioState { Sidecar = Mode.Sidecar, @@ -62,6 +71,20 @@ export async function enableIstio(pkg: UDSPackage) { const result = getIstioLabels(labels, targetIstioState, currentIstioState); + // Record this package's kubevirt intent, then derive the workload label from whether ANY package + // in the namespace wants it. The label is the trust anchor the RestrictIstioTrafficOverrides + // policy checks before allowing KubeVirt and CDI generated pods their Istio annotations. Only the + // operator (or an admin) can set it, never a tenant pod. Tracking intent per package keeps two + // packages in one namespace from flip-flopping the label. The label change does not require + // restarting pods, so shouldRestartPods is untouched. + const kubevirtPkgKey = `${KUBEVIRT_PKG_ANNOTATION_PREFIX}${pkg.metadata.name}`; + if (pkg.spec?.kubevirt?.enabled) { + annotations[kubevirtPkgKey] = "true"; + } else { + delete annotations[kubevirtPkgKey]; + } + setKubeVirtWorkloadLabel(result.labels, annotations); + // Apply namespace updates and restart pods if needed await applyNamespaceUpdates( pkg.metadata.namespace, @@ -92,8 +115,9 @@ export async function cleanupNamespace(pkg: UDSPackage) { const currentState = getCurrentIstioState(labels); const originalIstioState = annotations[ISTIO_STATE_ANNOTATION] as IstioState; - // Remove the package annotation + // Remove the package presence annotation and this package's kubevirt intent annotation delete annotations[`uds.dev/pkg-${pkg.metadata.name}`]; + delete annotations[`${KUBEVIRT_PKG_ANNOTATION_PREFIX}${pkg.metadata.name}`]; // Check if there are any other package annotations // Backwards compatibility for multiple package CRs in a single namespace @@ -112,6 +136,10 @@ export async function cleanupNamespace(pkg: UDSPackage) { delete annotations[ISTIO_STATE_ANNOTATION]; } + // Recompute the kubevirt workload label from the remaining per-package intent annotations: it + // stays only if another package in the namespace still has kubevirt enabled. + setKubeVirtWorkloadLabel(result.labels, annotations); + // Apply the updated Namespace await applyNamespaceUpdates( pkg.metadata.namespace, @@ -214,7 +242,10 @@ export function nsEntryIsOverClaimed(entry: V1ManagedFieldsEntry): boolean { Object.keys(meta).some(k => k !== "f:labels" && k !== "f:annotations") || Object.keys(labels).some(k => !MANAGED_LABEL_FIELDS.has(k)) || Object.keys(annotations).some( - k => k !== `f:${ISTIO_STATE_ANNOTATION}` && !k.startsWith("f:uds.dev/pkg-"), + k => + k !== `f:${ISTIO_STATE_ANNOTATION}` && + !k.startsWith("f:uds.dev/pkg-") && + !k.startsWith(`f:${KUBEVIRT_PKG_ANNOTATION_PREFIX}`), ) ); } @@ -225,6 +256,8 @@ function pickManagedLabels(labels: Record | undefined): Record = {}; if (labels?.[INJECTION_LABEL] !== undefined) result[INJECTION_LABEL] = labels[INJECTION_LABEL]; if (labels?.[AMBIENT_LABEL] !== undefined) result[AMBIENT_LABEL] = labels[AMBIENT_LABEL]; + if (labels?.[KUBEVIRT_WORKLOAD_LABEL] !== undefined) + result[KUBEVIRT_WORKLOAD_LABEL] = labels[KUBEVIRT_WORKLOAD_LABEL]; return result; } @@ -234,11 +267,30 @@ function pickManagedAnnotations( ): Record { return Object.fromEntries( Object.entries(annotations || {}).filter( - ([k]) => k === ISTIO_STATE_ANNOTATION || k.startsWith("uds.dev/pkg-"), + ([k]) => + k === ISTIO_STATE_ANNOTATION || + k.startsWith("uds.dev/pkg-") || + k.startsWith(KUBEVIRT_PKG_ANNOTATION_PREFIX), ), ); } +// Derive the kubevirt workload label from the per-package intent annotations: the label is present +// iff at least one package in the namespace has kubevirt enabled. +function setKubeVirtWorkloadLabel( + labels: Record, + annotations: Record, +): void { + const anyKubeVirtPackage = Object.keys(annotations).some(k => + k.startsWith(KUBEVIRT_PKG_ANNOTATION_PREFIX), + ); + if (anyKubeVirtPackage) { + labels[KUBEVIRT_WORKLOAD_LABEL] = "true"; + } else { + delete labels[KUBEVIRT_WORKLOAD_LABEL]; + } +} + /** * Apply namespace updates if there are changes to labels or annotations * diff --git a/src/pepr/operator/crd/generated/package-v1alpha1.ts b/src/pepr/operator/crd/generated/package-v1alpha1.ts index 9251d9ddf3..6d26a25c79 100644 --- a/src/pepr/operator/crd/generated/package-v1alpha1.ts +++ b/src/pepr/operator/crd/generated/package-v1alpha1.ts @@ -15,6 +15,12 @@ export interface Spec { * CA bundle configuration for the package */ caBundle?: CABundle; + /** + * KubeVirt workload settings. When enabled, the UDS operator labels the namespace + * 'uds.dev/kubevirt-workload: "true"' so the KubeVirt and CDI Pepr policy allowances apply + * to the namespace's generated pods (virt-launcher, CDI importer/upload/clone). + */ + kubevirt?: Kubevirt; /** * Create Service or Pod Monitor configurations */ @@ -61,6 +67,19 @@ export interface ConfigMap { name?: string; } +/** + * KubeVirt workload settings. When enabled, the UDS operator labels the namespace + * 'uds.dev/kubevirt-workload: "true"' so the KubeVirt and CDI Pepr policy allowances apply + * to the namespace's generated pods (virt-launcher, CDI importer/upload/clone). + */ +export interface Kubevirt { + /** + * Enable KubeVirt workload support for this namespace. Labels the namespace so KubeVirt and + * CDI generated pods are permitted their required Istio annotations. + */ + enabled?: boolean; +} + export interface Monitor { /** * Authorization settings. diff --git a/src/pepr/operator/crd/sources/package/v1alpha1.ts b/src/pepr/operator/crd/sources/package/v1alpha1.ts index 7dce5415c5..533dd01485 100644 --- a/src/pepr/operator/crd/sources/package/v1alpha1.ts +++ b/src/pepr/operator/crd/sources/package/v1alpha1.ts @@ -792,6 +792,18 @@ export const v1alpha1: V1CustomResourceDefinitionVersion = { monitor, sso, caBundle, + kubevirt: { + type: "object", + description: + "KubeVirt workload settings. When enabled, the UDS operator labels the namespace 'uds.dev/kubevirt-workload: \"true\"' so the KubeVirt and CDI Pepr policy allowances apply to the namespace's generated pods (virt-launcher, CDI importer/upload/clone).", + properties: { + enabled: { + type: "boolean", + description: + "Enable KubeVirt workload support for this namespace. Labels the namespace so KubeVirt and CDI generated pods are permitted their required Istio annotations.", + }, + }, + }, }, } as V1JSONSchemaProps, }, diff --git a/src/pepr/policies/istio.spec.ts b/src/pepr/policies/istio.spec.ts index 607b50d6ef..d43be38b51 100644 --- a/src/pepr/policies/istio.spec.ts +++ b/src/pepr/policies/istio.spec.ts @@ -1,5 +1,5 @@ /** - * Copyright 2025 Defense Unicorns + * Copyright 2025-2026 Defense Unicorns * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial */ @@ -154,9 +154,9 @@ describe("checkIstioTrafficInterceptionOverrides", () => { }); it("should handle pod metadata variations", () => { - expect(checkIstioTrafficInterceptionOverrides(mockContainers, {} as V1Pod)).toEqual([]); + expect(checkIstioTrafficInterceptionOverrides(mockContainers, {} as V1Pod, false)).toEqual([]); expect( - checkIstioTrafficInterceptionOverrides(mockContainers, { metadata: {} } as V1Pod), + checkIstioTrafficInterceptionOverrides(mockContainers, { metadata: {} } as V1Pod, false), ).toEqual([]); }); @@ -176,7 +176,7 @@ describe("checkIstioTrafficInterceptionOverrides", () => { }, } as V1Pod; - const result = checkIstioTrafficInterceptionOverrides(mockContainers, pod); + const result = checkIstioTrafficInterceptionOverrides(mockContainers, pod, false); expect(result).toContain("annotation sidecar.istio.io/inject"); expect(result).toContain("annotation traffic.sidecar.istio.io/excludeInboundPorts"); expect(result).toContain("label sidecar.istio.io/inject"); @@ -187,7 +187,7 @@ describe("checkIstioTrafficInterceptionOverrides", () => { "sidecar.istio.io/inject": "true", }); - expect(checkIstioTrafficInterceptionOverrides(mockContainers, pod as V1Pod)).toEqual([]); + expect(checkIstioTrafficInterceptionOverrides(mockContainers, pod as V1Pod, false)).toEqual([]); }); it("should ignore waypoint pods", () => { @@ -195,6 +195,100 @@ describe("checkIstioTrafficInterceptionOverrides", () => { "sidecar.istio.io/inject": "false", }); - expect(checkIstioTrafficInterceptionOverrides(waypointContainers, pod as V1Pod)).toEqual([]); + expect(checkIstioTrafficInterceptionOverrides(waypointContainers, pod as V1Pod, false)).toEqual( + [], + ); + }); + + it("should allow sidecar.istio.io/inject=false on CDI-managed pods in a kubevirt-workload namespace", () => { + const annotation = { "sidecar.istio.io/inject": "false" }; + + const importerPod = { metadata: { name: "importer-my-dv", annotations: annotation } }; + const uploadPod = { metadata: { name: "cdi-upload-my-dv", annotations: annotation } }; + const clonePod = { metadata: { name: "cdi-clone-abc123", annotations: annotation } }; + + expect( + checkIstioTrafficInterceptionOverrides(mockContainers, importerPod as V1Pod, true), + ).toEqual([]); + expect( + checkIstioTrafficInterceptionOverrides(mockContainers, uploadPod as V1Pod, true), + ).toEqual([]); + expect(checkIstioTrafficInterceptionOverrides(mockContainers, clonePod as V1Pod, true)).toEqual( + [], + ); + }); + + it("should block sidecar.istio.io/inject=false on CDI-named pods when the namespace lacks the workload label", () => { + const annotation = { "sidecar.istio.io/inject": "false" }; + const importerPod = { metadata: { name: "importer-my-dv", annotations: annotation } }; + + // isKubeVirtNamespace=false: a CDI pod name alone must not grant the exception. + expect( + checkIstioTrafficInterceptionOverrides(mockContainers, importerPod as V1Pod, false), + ).toContain("annotation sidecar.istio.io/inject"); + }); + + it("should block sidecar.istio.io/inject=false on non-CDI pods", () => { + const pod = { + metadata: { + name: "my-app-pod", + annotations: { "sidecar.istio.io/inject": "false" }, + }, + }; + + expect(checkIstioTrafficInterceptionOverrides(mockContainers, pod as V1Pod, true)).toContain( + "annotation sidecar.istio.io/inject", + ); + }); + + it("should allow kubevirt interface annotations on virt-launcher pods in a kubevirt-workload namespace", () => { + const pod = { + metadata: { + name: "virt-launcher-my-vm-abc12", + namespace: "my-vms", + labels: { "kubevirt.io": "virt-launcher" }, + annotations: { + "traffic.sidecar.istio.io/kubevirtInterfaces": "k6t-eth0", + "istio.io/reroute-virtual-interfaces": "k6t-eth0", + }, + }, + } as V1Pod; + + expect(checkIstioTrafficInterceptionOverrides(mockContainers, pod, true)).toEqual([]); + }); + + it("should block kubevirt interface annotations on non-launcher pods in a kubevirt-workload namespace", () => { + const pod = { + metadata: { + name: "rogue-pod", + namespace: "my-vms", + annotations: { + "traffic.sidecar.istio.io/kubevirtInterfaces": "k6t-eth0", + "istio.io/reroute-virtual-interfaces": "k6t-eth0", + }, + }, + } as V1Pod; + + const result = checkIstioTrafficInterceptionOverrides(mockContainers, pod, true); + expect(result).toContain("annotation traffic.sidecar.istio.io/kubevirtInterfaces"); + expect(result).toContain("annotation istio.io/reroute-virtual-interfaces"); + }); + + it("should block kubevirt interface annotations on a launcher pod when the namespace lacks the workload label", () => { + const pod = { + metadata: { + name: "virt-launcher-my-vm-abc12", + namespace: "not-a-vm-namespace", + labels: { "kubevirt.io": "virt-launcher" }, + annotations: { + "traffic.sidecar.istio.io/kubevirtInterfaces": "k6t-eth0", + }, + }, + } as V1Pod; + + // isKubeVirtNamespace=false: launcher name + label are not enough without the ns label. + expect(checkIstioTrafficInterceptionOverrides(mockContainers, pod, false)).toContain( + "annotation traffic.sidecar.istio.io/kubevirtInterfaces", + ); }); }); diff --git a/src/pepr/policies/istio.ts b/src/pepr/policies/istio.ts index fa5d1a0061..dab0b8c573 100644 --- a/src/pepr/policies/istio.ts +++ b/src/pepr/policies/istio.ts @@ -1,9 +1,9 @@ /** - * Copyright 2025 Defense Unicorns + * Copyright 2025-2026 Defense Unicorns * SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial */ -import { a, sdk } from "pepr"; +import { a, K8s, kind, sdk } from "pepr"; import { V1Container, V1Pod, V1PodSecurityContext } from "@kubernetes/client-node"; import { Policy } from "../operator/crd"; @@ -67,15 +67,28 @@ export function checkIstioSidecarOverrides(pod: V1Pod) { When(a.Pod) .IsCreatedOrUpdated() .Mutate(markExemption(Policy.RestrictIstioTrafficOverrides)) - .Validate(request => { + .Validate(async request => { if (isExempt(request, Policy.RestrictIstioTrafficOverrides)) { return request.Approve(); } const podContainers = containers(request); + // KubeVirt and CDI generated pods get scoped exceptions, but only when their namespace + // carries `uds.dev/kubevirt-workload: "true"`. The namespace label is the trust anchor: + // tenants can set pod names and pod labels, but only the platform can label a namespace. + // Only pods whose name matches a generated-pod prefix trigger the lookup, so normal pods + // do not pay for the extra API call. + const isKubeVirtNamespace = isKubeVirtGeneratedPodName(request.Raw) + ? await isKubeVirtWorkloadNamespace(request.Raw.metadata?.namespace) + : false; + // Combine all violations and sort - const violations = checkIstioTrafficInterceptionOverrides(podContainers, request.Raw); + const violations = checkIstioTrafficInterceptionOverrides( + podContainers, + request.Raw, + isKubeVirtNamespace, + ); if (violations.length > 0) { return request.Deny( @@ -86,10 +99,56 @@ When(a.Pod) return request.Approve(); }); -export function checkIstioTrafficInterceptionOverrides(podContainers: V1Container[], pod: V1Pod) { +const KUBEVIRT_WORKLOAD_NAMESPACE_LABEL = "uds.dev/kubevirt-workload"; + +// CDI generates these pods in the target VM namespace; they are matched by name prefix. +const CDI_POD_NAME_PREFIXES = ["importer-", "cdi-upload-", "cdi-clone-"]; + +/** + * Returns true if the pod name matches a KubeVirt launcher or CDI generated pod prefix. + * This is a cheap, label-free check used only to decide whether the namespace trust-anchor + * lookup is needed. It does not by itself grant any exception. + */ +export function isKubeVirtGeneratedPodName(pod: V1Pod): boolean { + const name = pod.metadata?.name ?? ""; + return name.startsWith("virt-launcher-") || CDI_POD_NAME_PREFIXES.some(p => name.startsWith(p)); +} + +/** + * Returns true only if the namespace carries `uds.dev/kubevirt-workload: "true"`. + * Fails closed: if the namespace cannot be read, the exception is not granted. + */ +export async function isKubeVirtWorkloadNamespace(namespace?: string): Promise { + if (!namespace) { + return false; + } + try { + const ns = await K8s(kind.Namespace).Get(namespace); + return ns.metadata?.labels?.[KUBEVIRT_WORKLOAD_NAMESPACE_LABEL] === "true"; + } catch { + return false; + } +} + +export function checkIstioTrafficInterceptionOverrides( + podContainers: V1Container[], + pod: V1Pod, + isKubeVirtNamespace: boolean, +) { const namespace = pod.metadata?.namespace || "default"; const annotations = pod.metadata?.annotations || {}; const labels = pod.metadata?.labels || {}; + const name = pod.metadata?.name ?? ""; + // A launcher exception requires all three: a kubevirt-workload namespace, the launcher name + // prefix, and the launcher identity label. Pod name and label are tenant-settable, so the + // namespace label is the actual security boundary. + const isKubeVirtWorkload = + isKubeVirtNamespace && + name.startsWith("virt-launcher-") && + labels["kubevirt.io"] === "virt-launcher"; + // A CDI exception requires a kubevirt-workload namespace and a CDI generated pod name prefix. + const isCDIPod = + isKubeVirtNamespace && CDI_POD_NAME_PREFIXES.some(prefix => name.startsWith(prefix)); const blockedTrafficAnnotations = [ "sidecar.istio.io/inject", // Can disable sidecar injection "traffic.sidecar.istio.io/excludeInboundPorts", // Can bypass inbound port interception @@ -100,25 +159,34 @@ export function checkIstioTrafficInterceptionOverrides(podContainers: V1Containe "traffic.sidecar.istio.io/includeOutboundIPRanges", // Can modify outbound IP range interception "traffic.sidecar.istio.io/includeOutboundPorts", // Can modify outbound port interception "sidecar.istio.io/interceptionMode", // Can change interception mode (REDIRECT/TPROXY) - "traffic.sidecar.istio.io/kubevirtInterfaces", // Can modify kubevirt interface handling "istio.io/redirect-virtual-interfaces", // Can modify virtual interface traffic handling ]; + const kubevirtBlockedTrafficAnnotations = [ + "traffic.sidecar.istio.io/kubevirtInterfaces", // Deprecated (Istio 1.25) kubevirt interface handling + "istio.io/reroute-virtual-interfaces", // Replacement for kubevirtInterfaces; reroutes virtual interface traffic + ]; const blockedTrafficLabels = [ "sidecar.istio.io/inject", // Can disable sidecar injection ]; // Check annotations for violations + const effectiveBlockedAnnotations = isKubeVirtWorkload + ? blockedTrafficAnnotations + : [...blockedTrafficAnnotations, ...kubevirtBlockedTrafficAnnotations]; + const annotationViolations = Object.entries(annotations) .filter(([key]) => { if ( // Ignore 'sidecar.istio.io/inject' annotation in istio-system namespace (key === "sidecar.istio.io/inject" && namespace === "istio-system") || // Ignore 'sidecar.istio.io/inject=true' annotation - (key === "sidecar.istio.io/inject" && annotations[key].trim() === "true") + (key === "sidecar.istio.io/inject" && annotations[key].trim() === "true") || + // Ignore 'sidecar.istio.io/inject=false' annotation on CDI-managed pods (importer, upload, clone) + (key === "sidecar.istio.io/inject" && annotations[key].trim() === "false" && isCDIPod) ) { return false; } - return blockedTrafficAnnotations.includes(key); + return effectiveBlockedAnnotations.includes(key); }) .map(([key]) => `annotation ${key}`); diff --git a/src/pepr/uds-cluster-crds/templates/packages.uds.dev.yaml b/src/pepr/uds-cluster-crds/templates/packages.uds.dev.yaml index 0f8a939cea..fbcb702081 100644 --- a/src/pepr/uds-cluster-crds/templates/packages.uds.dev.yaml +++ b/src/pepr/uds-cluster-crds/templates/packages.uds.dev.yaml @@ -1279,3 +1279,18 @@ spec: additionalProperties: type: string default: {} + kubevirt: + type: object + description: >- + KubeVirt workload settings. When enabled, the UDS operator + labels the namespace 'uds.dev/kubevirt-workload: "true"' so + the KubeVirt and CDI Pepr policy allowances apply to the + namespace's generated pods (virt-launcher, CDI + importer/upload/clone). + properties: + enabled: + type: boolean + description: >- + Enable KubeVirt workload support for this namespace. + Labels the namespace so KubeVirt and CDI generated pods + are permitted their required Istio annotations. diff --git a/tasks.yaml b/tasks.yaml index 465a0cbf9b..9ef7cce238 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -14,6 +14,7 @@ includes: - test: ./tasks/test.yaml - lint: ./tasks/lint.yaml - diagrams: ./tasks/diagrams.yaml + - kubevirt: ./tasks/kubevirt.yaml tasks: - name: default diff --git a/tasks/create.yaml b/tasks/create.yaml index 82d7f16a1f..ed0863683b 100644 --- a/tasks/create.yaml +++ b/tasks/create.yaml @@ -59,6 +59,23 @@ tasks: - description: "Create the slim dev bundle (Base and Identity)" cmd: ./uds create bundles/k3d-slim-dev --confirm --no-progress --architecture=${ZARF_ARCHITECTURE} + - name: k3d-kubevirt-dev-bundle + description: "Create the k3d-kubevirt-dev bundle (base + identity + kubevirt)" + actions: + - description: "Create base package" + task: single-layer + with: + layer: base + shim_bundle: "false" + - description: "Create identity-authorization package" + task: single-layer + with: + layer: identity-authorization + - description: "Create kubevirt package" + cmd: mkdir -p build && ./uds zarf package create packages/kubevirt --architecture="${ZARF_ARCHITECTURE}" --confirm --no-progress --output build + - description: "Create k3d-kubevirt-dev bundle" + cmd: ./uds create bundles/k3d-kubevirt-dev --confirm --no-progress --architecture=${ZARF_ARCHITECTURE} + - name: checkpoint-dev-package description: "Create the K3d + UDS Core Checkpoint Zarf Package" actions: diff --git a/tasks/kubevirt.yaml b/tasks/kubevirt.yaml new file mode 100644 index 0000000000..2546fa666b --- /dev/null +++ b/tasks/kubevirt.yaml @@ -0,0 +1,75 @@ +# Copyright 2026 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +includes: + - kubevirt-pkg: ../packages/kubevirt/tasks.yaml + - create: ./create.yaml + +variables: + - name: KUBEVIRT_USE_EMULATION + default: "false" + - name: VERSION + description: "Version of the UDS Core packages to deploy" + # x-release-please-start-version + default: "1.7.0" + # x-release-please-end + +tasks: + # Build the kubevirt Zarf package into build/ + - name: build-package + description: "Build the core-kubevirt Zarf package" + actions: + - description: "Create build output directory" + cmd: mkdir -p build + - description: "Build core-kubevirt package" + cmd: ./uds zarf package create packages/kubevirt --flavor upstream --architecture="${ZARF_ARCHITECTURE}" --confirm --no-progress --output build + - description: "Rename flavored package file by removing the upstream suffix" + cmd: | + for f in build/zarf-package-core-kubevirt-*-upstream.tar.zst; do + [ -e "$f" ] && mv -v "$f" "${f%-upstream.tar.zst}.tar.zst" + done + + # Build core-base + kubevirt + the kubevirt-dev bundle + - name: build + description: "Build core-base, core-kubevirt, and the k3d-kubevirt-dev bundle" + actions: + - description: "Build core-base package" + task: create:single-layer + with: + layer: base + shim_bundle: "false" + - description: "Build core-identity-authorization package" + task: create:single-layer + with: + layer: identity-authorization + - task: build-package + - description: "Create k3d-kubevirt-dev bundle" + cmd: ./uds create bundles/k3d-kubevirt-dev --confirm --no-progress --architecture=${ZARF_ARCHITECTURE} + + # Deploy the kubevirt-dev bundle to an existing k3d cluster + - name: deploy + description: "Deploy k3d-kubevirt-dev bundle (cluster must already exist)" + actions: + - description: "Deploy k3d-kubevirt-dev bundle" + cmd: >- + ./uds deploy bundles/k3d-kubevirt-dev/uds-bundle-k3d-core-kubevirt-dev-${UDS_ARCH}-${VERSION}.tar.zst + --set INSECURE_ADMIN_PASSWORD_GENERATION=true + --set KUBEVIRT_USE_EMULATION=${KUBEVIRT_USE_EMULATION} + --set K3D_EXTRA_ARGS="--agents 1" + --confirm --no-progress + + # Full end-to-end: create cluster, build, deploy, test + - name: demo + description: "Full demo: create k3d cluster, build, deploy, and run KubeVirt tests" + actions: + - description: "Create k3d cluster" + task: setup:create-k3d-cluster + - task: kubevirt:build + - task: kubevirt:deploy + - task: kubevirt:test + + # Run all KubeVirt tests against a running cluster with kubevirt deployed + - name: test + description: "Run all KubeVirt tests (health-check, vm-test, podinfo-vm-test)" + actions: + - task: kubevirt-pkg:all