feat(project-cleanup): wait for volumes to detach between phases

After the server-delete phase, Cinder moves attached volumes through
``in-use → detaching → available`` asynchronously. The volume-delete
phase hit these mid-transition and received HTTP 400 ``status must be
available`` for each one — the 12-of-30 red wave in the 2026-04-23 run.

Introduce a settle window right before the volume phase: re-list the
project's volumes and poll until none remain in ``detaching`` or
``in-use``, bounded by a conservative 30 s timeout with a 2 s interval.
Module-level constants (``_VOLUME_SETTLE_TIMEOUT``,
``_VOLUME_SETTLE_INTERVAL``) keep the values tunable per test without
leaking global state.

The refresh step doubles as a second-pass filter: volumes cascaded by
the server delete disappear from the list entirely, so the downstream
``_delete_one`` sees fewer 404s. Remaining stuck volumes are returned
on timeout and surface normally as FAILED — silence would hide them.

Author: kallioli

orca_cli/commands/project.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import time
 from datetime import datetime
 from enum import Enum
 
@@ -188,6 +189,48 @@ def _before_cutoff(resource: dict, cutoff: datetime | None) -> bool:
         return True
 
 
+# Short settle window for volumes freshly detached by a server delete.
+# Kept module-level so tests can monkeypatch them to zero without global state
+# leaking into production. Values are deliberately conservative: 30 s is long
+# enough to cover typical Cinder detach latency on a loaded cloud, 2 s limits
+# the polling to ~15 round-trips in the worst case.
+_VOLUME_SETTLE_TIMEOUT = 30.0
+_VOLUME_SETTLE_INTERVAL = 2.0
+_VOLUME_TRANSIENT_STATUSES = frozenset({"detaching", "in-use"})
+
+
+def _refresh_and_wait_volumes(
+    client,
+    proj_id: str,
+    cutoff: datetime | None,
+) -> list[tuple[str, str]]:
+    """Re-list project volumes and wait until none are in a transient state.
+
+    Called between the server-delete and volume-delete phases. Server
+    deletion releases attached volumes asynchronously — they transition
+    ``in-use → detaching → available`` in Cinder, and deleting a volume
+    mid-detach raises HTTP 400. Polling here lets us meet the volume
+    phase with a clean set instead of firing 400s the user has to ignore.
+
+    On timeout we return whatever we last observed; a lingering DELETE
+    will surface as FAILED via ``_delete_one``, which is the right
+    signal — silence would hide a stuck volume.
+    """
+    deadline = time.monotonic() + _VOLUME_SETTLE_TIMEOUT
+    while True:
+        try:
+            vols: list = list(VolumeService(client).find(
+                params={"project_id": proj_id},
+            ))
+        except Exception:
+            vols = []
+        vols = [v for v in vols if _before_cutoff(v, cutoff)]
+        transient = [v for v in vols if v.get("status") in _VOLUME_TRANSIENT_STATUSES]
+        if not transient or time.monotonic() >= deadline:
+            return [(v["id"], v.get("name") or "—") for v in vols]
+        time.sleep(_VOLUME_SETTLE_INTERVAL)
+
+
 # device_owner values for ports attached to a router that need to be detached
 # via remove_router_interface before the router can be deleted. Router gateway
 # ports (network:router_gateway, network:router_centralized_snat) are released
@@ -585,6 +628,15 @@ def add(rtype: str, items: list, name_key: str = "name") -> None:
 
     tally: dict[Outcome, int] = {o: 0 for o in Outcome}
     for rtype in DELETION_ORDER:
+        if rtype == "volume" and by_type.get("volume"):
+            # Server deletion releases attached volumes asynchronously; wait
+            # for them to settle before attempting the volume-phase DELETEs.
+            with console.status(
+                "[bold cyan]Waiting for volumes to detach…[/bold cyan]"
+            ):
+                by_type["volume"] = _refresh_and_wait_volumes(
+                    client, proj_id, cutoff,
+                )
         for rid, rname in by_type.get(rtype, []):
             tally[_delete_one(client, rtype, rid, rname)] += 1
 

tests/test_project_cleanup.py

@@ -240,6 +240,95 @@ def test_unexpected_exception_is_failed(self, monkeypatch):
         assert out is Outcome.FAILED
 
 
+class TestRefreshAndWaitVolumes:
+    """Volume settle window between server-delete and volume-delete phases.
+
+    Tests pin the settle timings to near-zero so the suite stays fast.
+    """
+
+    def _zero_timing(self, monkeypatch):
+        monkeypatch.setattr(proj_mod, "_VOLUME_SETTLE_INTERVAL", 0.0)
+        monkeypatch.setattr(proj_mod, "_VOLUME_SETTLE_TIMEOUT", 0.01)
+        monkeypatch.setattr(proj_mod.time, "sleep", lambda _s: None)
+
+    def test_returns_immediately_when_no_transient(self, monkeypatch):
+        self._zero_timing(monkeypatch)
+        svc = MagicMock()
+        svc.find.return_value = [
+            {"id": "v1", "name": "n1", "status": "available"},
+            {"id": "v2", "name": "n2", "status": "error"},
+        ]
+        monkeypatch.setattr(proj_mod, "VolumeService", lambda _c: svc)
+
+        result = proj_mod._refresh_and_wait_volumes(object(), "p-1", None)
+
+        assert result == [("v1", "n1"), ("v2", "n2")]
+        svc.find.assert_called_once()
+
+    def test_polls_until_transient_clears(self, monkeypatch):
+        self._zero_timing(monkeypatch)
+        svc = MagicMock()
+        # First call: one volume still detaching. Second call: both settled.
+        svc.find.side_effect = [
+            [
+                {"id": "v1", "name": "n1", "status": "detaching"},
+                {"id": "v2", "name": "n2", "status": "available"},
+            ],
+            [
+                {"id": "v1", "name": "n1", "status": "available"},
+                {"id": "v2", "name": "n2", "status": "available"},
+            ],
+        ]
+        monkeypatch.setattr(proj_mod, "VolumeService", lambda _c: svc)
+
+        result = proj_mod._refresh_and_wait_volumes(object(), "p-1", None)
+
+        assert result == [("v1", "n1"), ("v2", "n2")]
+        assert svc.find.call_count == 2
+
+    def test_returns_last_seen_on_timeout(self, monkeypatch):
+        """Timeout must not raise — the stuck volume surfaces as FAILED later."""
+        self._zero_timing(monkeypatch)
+        svc = MagicMock()
+        svc.find.return_value = [
+            {"id": "stuck", "name": "s", "status": "detaching"},
+        ]
+        monkeypatch.setattr(proj_mod, "VolumeService", lambda _c: svc)
+
+        result = proj_mod._refresh_and_wait_volumes(object(), "p-1", None)
+
+        assert result == [("stuck", "s")]
+
+    def test_missing_service_is_tolerated(self, monkeypatch):
+        self._zero_timing(monkeypatch)
+        svc = MagicMock()
+        svc.find.side_effect = RuntimeError("cinder down")
+        monkeypatch.setattr(proj_mod, "VolumeService", lambda _c: svc)
+
+        result = proj_mod._refresh_and_wait_volumes(object(), "p-1", None)
+
+        assert result == []
+
+    def test_cutoff_is_applied_on_refresh(self, monkeypatch):
+        """Fresh list honors --created-before, same as the initial scan."""
+        from datetime import datetime, timezone
+
+        self._zero_timing(monkeypatch)
+        svc = MagicMock()
+        svc.find.return_value = [
+            {"id": "old", "name": "o", "status": "available",
+             "created_at": "2020-01-01T00:00:00+00:00"},
+            {"id": "new", "name": "n", "status": "available",
+             "created_at": "2099-01-01T00:00:00+00:00"},
+        ]
+        monkeypatch.setattr(proj_mod, "VolumeService", lambda _c: svc)
+
+        cutoff = datetime(2025, 1, 1, tzinfo=timezone.utc)
+        result = proj_mod._refresh_and_wait_volumes(object(), "p-1", cutoff)
+
+        assert result == [("old", "o")]
+
+
 class TestSummaryRendering:
     """End-to-end through the Click command: the summary line must aggregate
     the four outcomes independently — a mix of 404s, 409s and real failures