Add OpenCost pod restart stability check (#91)

pkg/env/env.go

@@ -13,6 +13,8 @@ const defaultMCPURL = "http://localhost:8081"
 const defaultApproxThreshold = 0.0001 // 0.01%
 const defaultOracleBillingURL = "https://apexapps.oracle.com/"
 const defaultDataResolutionMinutes = 1 // demo.infra.opencost.io sets queryResolutionSeconds: 60
+const defaultOpenCostNamespace = "opencost"
+const defaultOpenCostLabelSelector = "app.kubernetes.io/name=opencost"
 
 func GetDefaultURL() string {
 	url := defaultURL
@@ -24,6 +26,7 @@ func GetDefaultURL() string {
 	return strings.TrimRight(url, "/")
 }
 
+// checks if OPENCOST_URL is set, if not, use the default URL
 func GetDefaultOracleBillingURL() string {
 	url := defaultOracleBillingURL
 
@@ -34,6 +37,7 @@ func GetDefaultOracleBillingURL() string {
 	return strings.TrimRight(url, "/")
 }
 
+// checks if COMPARISON_OPENCOST_URL is set, if not, use the default URL
 func GetComparisonURL() string {
 	url := defaultURL
 
@@ -44,6 +48,7 @@ func GetComparisonURL() string {
 	return strings.TrimRight(url, "/")
 }
 
+// checks if APPROX_THRESHOLD is set, if not, use the default threshold
 func GetApproxThreshold() float64 {
 	approxThreshold := defaultApproxThreshold
 
@@ -59,6 +64,7 @@ func GetApproxThreshold() float64 {
 	return approxThreshold
 }
 
+// checks if OPENCOST_MCP_URL is set, if not, use the default MCP URL
 func GetMCPURL() string {
 	url := defaultMCPURL
 
@@ -69,6 +75,7 @@ func GetMCPURL() string {
 	return strings.TrimRight(url, "/")
 }
 
+// checks if OPENCOST_DATA_RESOLUTION_MINUTES is set, if not, use the default data resolution minutes
 func GetDataResolutionMinutes() int {
 	minutes := defaultDataResolutionMinutes
 
@@ -84,6 +91,7 @@ func GetDataResolutionMinutes() int {
 	return minutes
 }
 
+// checks if SHOW_DIFF is set, if not, use the default show diff
 func GetShowDiff() bool {
 	value := os.Getenv("SHOW_DIFF")
 	if value != "" {
@@ -97,3 +105,17 @@ func GetShowDiff() bool {
 
 	return false
 }
+
+func GetOpenCostNamespace() string {
+	if ns := os.Getenv("OPENCOST_NAMESPACE"); ns != "" { //looks outside of th eprogram and check the host computer's system settings for an environment variables names OPENCOST_NAMESPACE
+		return ns
+	}
+	return defaultOpenCostNamespace
+}
+
+func GetOpenCostLabelSelector() string {
+	if sel := os.Getenv("OPENCOST_LABEL_SELECTOR"); sel != "" { //looks outside of th eprogram and check the host computer's system settings for an environment variables names OPENCOST_LABEL_SELECTOR
+		return sel
+	}
+	return defaultOpenCostLabelSelector
+}

test/integration/reliability/no_pod_restarts_test.go

@@ -0,0 +1,90 @@
+package reliability
+
+import (
+	"encoding/json"
+	"os/exec"
+	"testing"
+
+	"github.com/opencost/opencost-integration-tests/pkg/env"
+)
+
+type podList struct {
+	Items []pod `json:"items"`
+}
+
+type pod struct {
+	Metadata podMetadata `json:"metadata"`
+	Status   podStatus   `json:"status"`
+}
+
+type podMetadata struct {
+	Name string `json:"name"`
+}
+
+type podStatus struct {
+	ContainerStatuses []containerStatus `json:"containerStatuses"`
+}
+
+type containerStatus struct {
+	Name         string `json:"name"`
+	RestartCount int32  `json:"restartCount"`
+}
+
+func requireKubectl(t *testing.T) {
+	t.Helper()
+
+	if _, err := exec.LookPath("kubectl"); err != nil { //chwcks PATH to see if kubectl is installed
+		t.Skipf("kubectl not found: %v", err)
+	}
+
+	if err := exec.Command("kubectl", "cluster-info").Run(); err != nil { //checks if the kubernetes cluster is reachable
+		t.Skipf("kubernetes cluster not reachable: %v", err)
+	}
+}
+
+func dumpPodDiagnostics(t *testing.T, podName, containerName string) {
+	t.Helper()
+	ns := env.GetOpenCostNamespace()
+
+	describe, _ := exec.Command("kubectl", "describe", "pod", "-n", ns, podName).CombinedOutput()
+	t.Logf("kubectl describe pod %s:\n%s", podName, describe)
+
+	logs, _ := exec.Command(
+		"kubectl", "logs", "-n", ns, podName,
+		"-c", containerName, "--previous",
+	).CombinedOutput()
+	t.Logf("kubectl logs --previous %s/%s:\n%s", podName, containerName, logs)
+}
+
+func TestNoOpenCostPodRestarts(t *testing.T) {
+	requireKubectl(t)
+
+	out, err := exec.Command(
+		"kubectl", "get", "pods",
+		"-n", env.GetOpenCostNamespace(),
+		"-l", env.GetOpenCostLabelSelector(),
+		"-o", "json",
+	).CombinedOutput()
+	if err != nil {
+		t.Fatalf("kubectl get pods: %v\n%s", err, out)
+	}
+
+	var pods podList
+	if err := json.Unmarshal(out, &pods); err != nil {
+		t.Fatalf("decode pods json: %v", err)
+	}
+
+	if len(pods.Items) == 0 {
+		t.Fatalf("no OpenCost pods found in namespace %q", env.GetOpenCostNamespace())
+	}
+
+	for _, p := range pods.Items {
+		for _, cs := range p.Status.ContainerStatuses {
+			if cs.RestartCount != 0 {
+				t.Errorf("pod %s container %s restartCount=%d, want 0",
+					p.Metadata.Name, cs.Name, cs.RestartCount)
+				dumpPodDiagnostics(t, p.Metadata.Name, cs.Name)
+			}
+		}
+	}
+}

test/integration/reliability/test.bats

@@ -0,0 +1,12 @@
+setup() {
+    DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )"
+    cd "$DIR"
+}
+
+teardown() {
+    : # nothing to tear down
+}
+
+@test "reliability: no OpenCost pod restarts" {
+    go test -count=1 no_pod_restarts_test.go
+}