test(devstack): add live e2e suite (read+write) against a real cloud

Builds on the read-only DevStack smoke harness with end-to-end tests
that actually create and destroy resources, gated behind a `live`
pytest marker so they're skipped by default and only run with
`pytest -m live tests/devstack/`.

Coverage breakdown (12 tests):

* compute  — flavor + keypair create/list/show/delete
* network  — network, subnet, security-group create/list/delete
* volume   — volume create/show/delete
* object_store — container create/list/delete
* dns      — zone create/show/list/delete
* key_manager — secret create/list/delete
* identity — project + user create/show/list/delete
* workflow — full server boot: flavor → keypair → security-group →
  network → subnet → server (--wait until ACTIVE) → reverse cleanup,
  catching cross-module integration bugs that isolated tests miss

Infrastructure (`conftest.py`):
* `live_invoke` — Click CliRunner against the live profile (default
  `devstack`, override via `ORCA_LIVE_PROFILE`)
* `live_client` — authenticated `OrcaClient` for tests that need the
  service layer directly
* `cleanup` — LIFO finaliser registry that runs even on failure, so a
  botched test can't leave orphaned resources behind
* `live_name` — unique traceable resource names (`orca-live-<kind>-<uuid8>`)
* `extract_uuid` — pulls IDs out of orca's human-readable create
  messages, recognising both canonical UUIDs (Nova/Cinder/Neutron) and
  bare 32-hex Keystone IDs

Configuration: `pyproject.toml` registers the `live` marker and pins
`addopts = -m 'not live'` so the default `pytest` invocation continues
to run only the 2383 mock-based tests.

Pre-condition: an orca profile pointing at a working cloud must exist
in `~/.orca/config.yaml`. The cirros image fixture lazy-uploads from
`/tmp/cirros.img` (or downloads from cirros-cloud.net) on first run
and reuses the resulting `orca-live-cirros` image across runs.

Author: kallioli

pyproject.toml

@@ -86,6 +86,10 @@ warn_redundant_casts = true
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+addopts = "-m 'not live'"
+markers = [
+    "live: end-to-end tests against a real OpenStack cloud (run with `pytest -m live tests/devstack/`)",
+]
 
 [tool.coverage.run]
 source = ["orca_cli"]

tests/devstack/conftest.py

@@ -0,0 +1,138 @@
+"""Shared fixtures for live e2e tests against a real OpenStack cloud.
+
+Tests in this directory are marked ``@pytest.mark.live`` and excluded
+from the default ``pytest`` run (see ``pyproject.toml``). To execute::
+
+    pytest -m live tests/devstack/
+
+A working orca profile named ``devstack`` (or whatever
+``ORCA_LIVE_PROFILE`` points to) must exist in ``~/.orca/config.yaml``.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import uuid
+from collections.abc import Callable
+
+import pytest
+from click.testing import CliRunner
+
+from orca_cli.core.client import OrcaClient
+from orca_cli.core.config import load_config
+from orca_cli.main import cli
+
+# ── Profile resolution & client ─────────────────────────────────────────
+
+
+@pytest.fixture(scope="session")
+def live_profile_name() -> str:
+    return os.environ.get("ORCA_LIVE_PROFILE", "devstack")
+
+
+@pytest.fixture(scope="session")
+def live_config(live_profile_name: str) -> dict:
+    try:
+        return load_config(live_profile_name)
+    except Exception as exc:
+        pytest.skip(f"orca profile {live_profile_name!r} not loadable: {exc}")
+
+
+@pytest.fixture(scope="session")
+def live_client(live_config: dict) -> OrcaClient:
+    """Authenticated OrcaClient against the live cloud."""
+    client = OrcaClient(live_config)
+    client.authenticate()
+    return client
+
+
+# ── CLI runner ───────────────────────────────────────────────────────────
+
+
+@pytest.fixture
+def live_invoke(live_profile_name: str):
+    """Invoke the orca CLI in-process against the live profile.
+
+    Returns a ``click.testing.Result`` — check ``result.exit_code`` and
+    ``result.output``.
+    """
+    runner = CliRunner()
+
+    def _invoke(*args: str, input: str | None = None):
+        return runner.invoke(
+            cli,
+            ["-P", live_profile_name, *args],
+            catch_exceptions=False,
+            input=input,
+        )
+
+    return _invoke
+
+
+# ── Cleanup tracker ──────────────────────────────────────────────────────
+
+
+@pytest.fixture
+def cleanup():
+    """LIFO cleanup callback registry that runs even on test failure.
+
+    Use it like::
+
+        def test_foo(live_client, cleanup):
+            res = create_something(live_client)
+            cleanup(lambda: delete_something(live_client, res["id"]))
+            ...
+
+    Callbacks run in reverse-registration order during teardown, and
+    failures are logged (not re-raised) so a botched cleanup doesn't
+    mask the original test result.
+    """
+    callbacks: list[Callable[[], None]] = []
+
+    yield callbacks.append
+
+    for cb in reversed(callbacks):
+        try:
+            cb()
+        except Exception as exc:  # noqa: BLE001 - cleanup must be lenient
+            print(f"[live-cleanup] {cb!r} failed: {exc}")
+
+
+# ── Naming helper ────────────────────────────────────────────────────────
+
+
+@pytest.fixture
+def live_name() -> Callable[[str], str]:
+    """Return a unique, traceable resource name (prefix + short uuid).
+
+    Lets cleanup-by-prefix work if a test crashes hard (Ctrl+C between
+    creation and registration).
+    """
+    def _name(prefix: str) -> str:
+        return f"orca-live-{prefix}-{uuid.uuid4().hex[:8]}"
+    return _name
+
+
+# ── Output parsing helpers ───────────────────────────────────────────────
+
+
+_ID_RE = re.compile(
+    # Canonical UUID (Nova/Cinder/Neutron/Glance) and bare 32-hex (Keystone).
+    r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{32}",
+    re.IGNORECASE,
+)
+
+
+def extract_uuid(text: str) -> str:
+    """Pull the first resource ID out of a CLI message.
+
+    orca's create commands print messages like ``Volume 'foo' (uuid) created.``
+    rather than emitting the ID via ``-f value``, so live tests parse the ID
+    out of the human-readable output. Both canonical UUIDs (Nova/Cinder/...)
+    and bare 32-hex IDs (Keystone) are recognised.
+    """
+    match = _ID_RE.search(text)
+    if not match:
+        raise AssertionError(f"no resource ID found in output: {text!r}")
+    return match.group(0)

tests/devstack/test_live_compute.py

@@ -0,0 +1,43 @@
+"""Live e2e: Nova compute (flavors, keypairs)."""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.devstack.conftest import extract_uuid
+
+pytestmark = pytest.mark.live
+
+
+def test_flavor_create_show_delete(live_invoke, cleanup, live_name):
+    name = live_name("flavor")
+
+    res = live_invoke("flavor", "create", name,
+                      "--vcpus", "1", "--ram", "64", "--disk", "1")
+    assert res.exit_code == 0, res.output
+    flavor_id = extract_uuid(res.output)
+    cleanup(lambda: live_invoke("flavor", "delete", flavor_id, "--yes"))
+
+    res = live_invoke("flavor", "list", "-f", "value", "-c", "ID")
+    assert res.exit_code == 0
+    assert flavor_id in res.output
+
+    res = live_invoke("flavor", "show", flavor_id, "-f", "value", "-c", "name")
+    assert res.exit_code == 0
+    assert name in res.output
+
+
+def test_keypair_create_delete(live_invoke, cleanup, live_name, tmp_path):
+    name = live_name("kp")
+    key_path = tmp_path / f"{name}.pem"
+
+    res = live_invoke("keypair", "create", name, "--save-to", str(key_path))
+    assert res.exit_code == 0, res.output
+    cleanup(lambda: live_invoke("keypair", "delete", name, "--yes"))
+
+    assert key_path.exists()
+    assert key_path.stat().st_mode & 0o777 == 0o600
+
+    res = live_invoke("keypair", "list", "-f", "value", "-c", "Name")
+    assert res.exit_code == 0
+    assert name in res.output

tests/devstack/test_live_dns.py

@@ -0,0 +1,30 @@
+"""Live e2e: Designate DNS zones."""
+
+from __future__ import annotations
+
+import uuid
+
+import pytest
+
+from tests.devstack.conftest import extract_uuid
+
+pytestmark = pytest.mark.live
+
+
+def test_zone_create_show_delete(live_invoke, cleanup):
+    # DNS zones must be FQDN ending with a dot. Generate a unique one.
+    domain = f"orca-live-{uuid.uuid4().hex[:8]}.example.com."
+
+    res = live_invoke("zone", "create", domain,
+                      "--email", "[email protected]")
+    assert res.exit_code == 0, res.output
+    zone_id = extract_uuid(res.output)
+    cleanup(lambda: live_invoke("zone", "delete", zone_id, "--yes"))
+
+    res = live_invoke("zone", "list", "-f", "value", "-c", "ID")
+    assert res.exit_code == 0
+    assert zone_id in res.output
+
+    res = live_invoke("zone", "show", zone_id, "-f", "value", "-c", "name")
+    assert res.exit_code == 0
+    assert domain in res.output

tests/devstack/test_live_identity.py

@@ -0,0 +1,40 @@
+"""Live e2e: Keystone identity (project, user)."""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.devstack.conftest import extract_uuid
+
+pytestmark = pytest.mark.live
+
+
+def test_project_create_show_delete(live_invoke, cleanup, live_name):
+    name = live_name("proj")
+
+    res = live_invoke("project", "create", name,
+                      "--description", "live test project")
+    assert res.exit_code == 0, res.output
+    project_id = extract_uuid(res.output)
+    cleanup(lambda: live_invoke("project", "delete", project_id, "--yes"))
+
+    res = live_invoke("project", "list", "-f", "value", "-c", "ID")
+    assert res.exit_code == 0
+    assert project_id in res.output
+
+    res = live_invoke("project", "show", project_id, "-f", "value", "-c", "name")
+    assert res.exit_code == 0
+    assert name in res.output
+
+
+def test_user_create_show_delete(live_invoke, cleanup, live_name):
+    name = live_name("user")
+
+    res = live_invoke("user", "create", name, "--password", "pw")
+    assert res.exit_code == 0, res.output
+    user_id = extract_uuid(res.output)
+    cleanup(lambda: live_invoke("user", "delete", user_id, "--yes"))
+
+    res = live_invoke("user", "list", "-f", "value", "-c", "ID")
+    assert res.exit_code == 0
+    assert user_id in res.output

tests/devstack/test_live_key_manager.py

@@ -0,0 +1,27 @@
+"""Live e2e: Barbican secrets."""
+
+from __future__ import annotations
+
+import pytest
+
+pytestmark = pytest.mark.live
+
+
+def test_secret_create_list_delete(live_invoke, cleanup, live_name):
+    name = live_name("secret")
+
+    res = live_invoke("secret", "create", name,
+                      "--payload", "live-test-payload",
+                      "--secret-type", "passphrase")
+    assert res.exit_code == 0, res.output
+
+    # Barbican exposes secrets via a "secret_ref" URL whose last segment is the
+    # secret UUID; orca's create message includes the full ref. List by name.
+    res = live_invoke("secret", "list", "-f", "value", "-c", "Name", "-c", "Secret href")
+    assert res.exit_code == 0
+    assert name in res.output
+
+    # Find the secret ref so we can clean up by ID.
+    line = next((line for line in res.output.splitlines() if name in line), "")
+    secret_ref = line.split()[-1]
+    cleanup(lambda: live_invoke("secret", "delete", secret_ref, "--yes"))

tests/devstack/test_live_network.py

@@ -0,0 +1,56 @@
+"""Live e2e: Neutron networks, subnets, security groups."""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.devstack.conftest import extract_uuid
+
+pytestmark = pytest.mark.live
+
+
+def test_network_create_show_delete(live_invoke, cleanup, live_name):
+    name = live_name("net")
+
+    res = live_invoke("network", "create", name)
+    assert res.exit_code == 0, res.output
+    net_id = extract_uuid(res.output)
+    cleanup(lambda: live_invoke("network", "delete", net_id, "--yes"))
+
+    res = live_invoke("network", "list", "-f", "value", "-c", "ID")
+    assert res.exit_code == 0
+    assert net_id in res.output
+
+
+def test_subnet_create_delete_on_new_network(live_invoke, cleanup, live_name):
+    net_name = live_name("net")
+    sub_name = live_name("sub")
+
+    res = live_invoke("network", "create", net_name)
+    assert res.exit_code == 0, res.output
+    net_id = extract_uuid(res.output)
+    cleanup(lambda: live_invoke("network", "delete", net_id, "--yes"))
+
+    res = live_invoke("network", "subnet", "create", sub_name,
+                      "--network-id", net_id, "--cidr", "10.99.0.0/24")
+    assert res.exit_code == 0, res.output
+    sub_id = extract_uuid(res.output)
+    cleanup(lambda: live_invoke("network", "subnet", "delete", sub_id, "--yes"))
+
+    res = live_invoke("network", "subnet", "list", "-f", "value", "-c", "ID")
+    assert res.exit_code == 0
+    assert sub_id in res.output
+
+
+def test_security_group_create_delete(live_invoke, cleanup, live_name):
+    name = live_name("sg")
+
+    res = live_invoke("security-group", "create", name,
+                      "--description", "live test sg")
+    assert res.exit_code == 0, res.output
+    sg_id = extract_uuid(res.output)
+    cleanup(lambda: live_invoke("security-group", "delete", sg_id, "--yes"))
+
+    res = live_invoke("security-group", "list", "-f", "value", "-c", "ID")
+    assert res.exit_code == 0
+    assert sg_id in res.output

tests/devstack/test_live_object_store.py

@@ -0,0 +1,19 @@
+"""Live e2e: Swift object-store containers."""
+
+from __future__ import annotations
+
+import pytest
+
+pytestmark = pytest.mark.live
+
+
+def test_container_create_list_delete(live_invoke, cleanup, live_name):
+    name = live_name("cont")
+
+    res = live_invoke("container", "create", name)
+    assert res.exit_code == 0, res.output
+    cleanup(lambda: live_invoke("container", "delete", name, "--yes"))
+
+    res = live_invoke("container", "list", "-f", "value", "-c", "Name")
+    assert res.exit_code == 0
+    assert name in res.output

tests/devstack/test_live_volume.py

@@ -0,0 +1,26 @@
+"""Live e2e: Cinder block-storage."""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.devstack.conftest import extract_uuid
+
+pytestmark = pytest.mark.live
+
+
+def test_volume_create_show_delete(live_invoke, cleanup, live_name):
+    name = live_name("vol")
+
+    res = live_invoke("volume", "create", "--name", name, "--size", "1")
+    assert res.exit_code == 0, res.output
+    vol_id = extract_uuid(res.output)
+    cleanup(lambda: live_invoke("volume", "delete", vol_id, "--yes"))
+
+    res = live_invoke("volume", "list", "-f", "value", "-c", "ID")
+    assert res.exit_code == 0
+    assert vol_id in res.output
+
+    res = live_invoke("volume", "show", vol_id, "-f", "value", "-c", "name")
+    assert res.exit_code == 0
+    assert name in res.output