From b056710a81e181bb7c68630aa9cdb4636a122f5e Mon Sep 17 00:00:00 2001 From: Aadarsh Sankar Date: Tue, 23 Jun 2026 15:11:59 -0500 Subject: [PATCH] Add OpenCost pod restart stability check (#91) Assert OpenCost container restart count is zero at end of integration suite; skip when kubectl/cluster unavailable; dump pod describe and previous-container logs on failure. Signed-off-by: Aadarsh Sankar --- pkg/env/env.go | 22 +++++ .../reliability/no_pod_restarts_test.go | 90 +++++++++++++++++++ test/integration/reliability/test.bats | 12 +++ 3 files changed, 124 insertions(+) create mode 100644 test/integration/reliability/no_pod_restarts_test.go create mode 100644 test/integration/reliability/test.bats diff --git a/pkg/env/env.go b/pkg/env/env.go index e7eace0..f343e26 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -13,6 +13,8 @@ const defaultMCPURL = "http://localhost:8081" const defaultApproxThreshold = 0.0001 // 0.01% const defaultOracleBillingURL = "https://apexapps.oracle.com/" const defaultDataResolutionMinutes = 1 // demo.infra.opencost.io sets queryResolutionSeconds: 60 +const defaultOpenCostNamespace = "opencost" +const defaultOpenCostLabelSelector = "app.kubernetes.io/name=opencost" func GetDefaultURL() string { url := defaultURL @@ -24,6 +26,7 @@ func GetDefaultURL() string { return strings.TrimRight(url, "/") } +// checks if OPENCOST_URL is set, if not, use the default URL func GetDefaultOracleBillingURL() string { url := defaultOracleBillingURL @@ -34,6 +37,7 @@ func GetDefaultOracleBillingURL() string { return strings.TrimRight(url, "/") } +// checks if COMPARISON_OPENCOST_URL is set, if not, use the default URL func GetComparisonURL() string { url := defaultURL @@ -44,6 +48,7 @@ func GetComparisonURL() string { return strings.TrimRight(url, "/") } +// checks if APPROX_THRESHOLD is set, if not, use the default threshold func GetApproxThreshold() float64 { approxThreshold := defaultApproxThreshold @@ -59,6 +64,7 @@ func GetApproxThreshold() float64 { return approxThreshold } +// checks if OPENCOST_MCP_URL is set, if not, use the default MCP URL func GetMCPURL() string { url := defaultMCPURL @@ -69,6 +75,7 @@ func GetMCPURL() string { return strings.TrimRight(url, "/") } +// checks if OPENCOST_DATA_RESOLUTION_MINUTES is set, if not, use the default data resolution minutes func GetDataResolutionMinutes() int { minutes := defaultDataResolutionMinutes @@ -84,6 +91,7 @@ func GetDataResolutionMinutes() int { return minutes } +// checks if SHOW_DIFF is set, if not, use the default show diff func GetShowDiff() bool { value := os.Getenv("SHOW_DIFF") if value != "" { @@ -97,3 +105,17 @@ func GetShowDiff() bool { return false } + +func GetOpenCostNamespace() string { + if ns := os.Getenv("OPENCOST_NAMESPACE"); ns != "" { //looks outside of th eprogram and check the host computer's system settings for an environment variables names OPENCOST_NAMESPACE + return ns + } + return defaultOpenCostNamespace +} + +func GetOpenCostLabelSelector() string { + if sel := os.Getenv("OPENCOST_LABEL_SELECTOR"); sel != "" { //looks outside of th eprogram and check the host computer's system settings for an environment variables names OPENCOST_LABEL_SELECTOR + return sel + } + return defaultOpenCostLabelSelector +} diff --git a/test/integration/reliability/no_pod_restarts_test.go b/test/integration/reliability/no_pod_restarts_test.go new file mode 100644 index 0000000..bcaaa92 --- /dev/null +++ b/test/integration/reliability/no_pod_restarts_test.go @@ -0,0 +1,90 @@ +package reliability + +import ( + "encoding/json" + "os/exec" + "testing" + + "github.com/opencost/opencost-integration-tests/pkg/env" +) + +type podList struct { + Items []pod `json:"items"` +} + +type pod struct { + Metadata podMetadata `json:"metadata"` + Status podStatus `json:"status"` +} + +type podMetadata struct { + Name string `json:"name"` +} + +type podStatus struct { + ContainerStatuses []containerStatus `json:"containerStatuses"` +} + +type containerStatus struct { + Name string `json:"name"` + RestartCount int32 `json:"restartCount"` +} + +func requireKubectl(t *testing.T) { + t.Helper() + + if _, err := exec.LookPath("kubectl"); err != nil { //chwcks PATH to see if kubectl is installed + t.Skipf("kubectl not found: %v", err) + } + + if err := exec.Command("kubectl", "cluster-info").Run(); err != nil { //checks if the kubernetes cluster is reachable + t.Skipf("kubernetes cluster not reachable: %v", err) + } +} + +func dumpPodDiagnostics(t *testing.T, podName, containerName string) { + t.Helper() + ns := env.GetOpenCostNamespace() + + describe, _ := exec.Command("kubectl", "describe", "pod", "-n", ns, podName).CombinedOutput() + t.Logf("kubectl describe pod %s:\n%s", podName, describe) + + logs, _ := exec.Command( + "kubectl", "logs", "-n", ns, podName, + "-c", containerName, "--previous", + ).CombinedOutput() + t.Logf("kubectl logs --previous %s/%s:\n%s", podName, containerName, logs) +} + +func TestNoOpenCostPodRestarts(t *testing.T) { + requireKubectl(t) + + out, err := exec.Command( + "kubectl", "get", "pods", + "-n", env.GetOpenCostNamespace(), + "-l", env.GetOpenCostLabelSelector(), + "-o", "json", + ).CombinedOutput() + if err != nil { + t.Fatalf("kubectl get pods: %v\n%s", err, out) + } + + var pods podList + if err := json.Unmarshal(out, &pods); err != nil { + t.Fatalf("decode pods json: %v", err) + } + + if len(pods.Items) == 0 { + t.Fatalf("no OpenCost pods found in namespace %q", env.GetOpenCostNamespace()) + } + + for _, p := range pods.Items { + for _, cs := range p.Status.ContainerStatuses { + if cs.RestartCount != 0 { + t.Errorf("pod %s container %s restartCount=%d, want 0", + p.Metadata.Name, cs.Name, cs.RestartCount) + dumpPodDiagnostics(t, p.Metadata.Name, cs.Name) + } + } + } +} diff --git a/test/integration/reliability/test.bats b/test/integration/reliability/test.bats new file mode 100644 index 0000000..b189227 --- /dev/null +++ b/test/integration/reliability/test.bats @@ -0,0 +1,12 @@ +setup() { + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + cd "$DIR" +} + +teardown() { + : # nothing to tear down +} + +@test "reliability: no OpenCost pod restarts" { + go test -count=1 no_pod_restarts_test.go +}