Improve tests resiliency

tests/cluster_certs_test.go

@@ -21,6 +21,10 @@ var _ = When("a cluster with custom certificates is installed with individual ce
 
 		namespace := NewNamespace()
 
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		// create custom cert secret
 		customCertDir := "testdata/customcerts/"
 

tests/cluster_network_test.go

@@ -28,7 +28,6 @@ var _ = When("two virtual clusters are installed", Label("e2e"), Label(networkin
 
 		var (
 			stdout  string
-			stderr  string
 			curlCmd string
 			err     error
 		)
@@ -70,25 +69,25 @@ var _ = When("two virtual clusters are installed", Label("e2e"), Label(networkin
 		// Pods in Cluster 1 should not be able to reach the Pod in Cluster 2
 
 		curlCmd = "curl --no-progress-meter " + pod1Cluster2IP
-		_, stderr, err = cluster1.ExecCmd(pod1Cluster1, curlCmd)
+		stdout, _, err = cluster1.ExecCmd(pod1Cluster1, curlCmd)
 		Expect(err).Should(HaveOccurred())
-		Expect(stderr).To(ContainSubstring("Failed to connect"))
+		Expect(stdout).To(Not(ContainSubstring("Welcome to nginx!")))
 
 		curlCmd = "curl --no-progress-meter " + pod1Cluster2IP
-		_, stderr, err = cluster1.ExecCmd(pod2Cluster1, curlCmd)
+		stdout, _, err = cluster1.ExecCmd(pod2Cluster1, curlCmd)
 		Expect(err).To(HaveOccurred())
-		Expect(stderr).To(ContainSubstring("Failed to connect"))
+		Expect(stdout).To(Not(ContainSubstring("Welcome to nginx!")))
 
 		// Pod in Cluster 2 should not be able to reach Pods in Cluster 1
 
 		curlCmd = "curl --no-progress-meter " + pod1Cluster1IP
-		_, stderr, err = cluster2.ExecCmd(pod1Cluster2, curlCmd)
+		stdout, _, err = cluster2.ExecCmd(pod1Cluster2, curlCmd)
 		Expect(err).To(HaveOccurred())
-		Expect(stderr).To(ContainSubstring("Failed to connect"))
+		Expect(stdout).To(Not(ContainSubstring("Welcome to nginx!")))
 
 		curlCmd = "curl --no-progress-meter " + pod2Cluster1IP
-		_, stderr, err = cluster2.ExecCmd(pod1Cluster2, curlCmd)
+		stdout, _, err = cluster2.ExecCmd(pod1Cluster2, curlCmd)
 		Expect(err).To(HaveOccurred())
-		Expect(stderr).To(ContainSubstring("Failed to connect"))
+		Expect(stdout).To(Not(ContainSubstring("Welcome to nginx!")))
 	})
 })

tests/cluster_persistence_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/x509"
 	"errors"
-	"fmt"
 	"time"
 
 	"k8s.io/utils/ptr"
@@ -154,7 +153,9 @@ var _ = When("a dynamic cluster is installed", Label("e2e"), Label(persistenceTe
 
 		namespace := NewNamespace()
 
-		By(fmt.Sprintf("Creating new virtual cluster in namespace %s", namespace.Name))
+		DeferCleanup(func() {
+			DeleteNamespaces(virtualCluster.Cluster.Namespace)
+		})
 
 		cluster := NewCluster(namespace.Name)
 		cluster.Spec.Persistence.Type = v1beta1.DynamicPersistenceMode
@@ -164,8 +165,6 @@ var _ = When("a dynamic cluster is installed", Label("e2e"), Label(persistenceTe
 
 		client, restConfig := NewVirtualK8sClientAndConfig(cluster)
 
-		By(fmt.Sprintf("Created virtual cluster %s/%s", cluster.Namespace, cluster.Name))
-
 		virtualCluster := &VirtualCluster{
 			Cluster:    cluster,
 			RestConfig: restConfig,

tests/cluster_status_test.go

@@ -27,7 +27,6 @@ var _ = When("a cluster's status is tracked", Label("e2e"), Label(statusTestsLab
 	// This BeforeEach/AfterEach will create a new namespace and a default policy for each test.
 	BeforeEach(func() {
 		ctx := context.Background()
-		namespace = NewNamespace()
 
 		vcp = &v1beta1.VirtualClusterPolicy{
 			ObjectMeta: metav1.ObjectMeta{
@@ -36,6 +35,11 @@ var _ = When("a cluster's status is tracked", Label("e2e"), Label(statusTestsLab
 		}
 		Expect(k8sClient.Create(ctx, vcp)).To(Succeed())
 
+		namespace = NewNamespace()
+
+		err := k8sClient.Get(ctx, client.ObjectKeyFromObject(namespace), namespace)
+		Expect(err).To(Not(HaveOccurred()))
+
 		namespace.Labels = map[string]string{
 			policy.PolicyNameLabelKey: vcp.Name,
 		}

tests/cluster_update_test.go

@@ -75,8 +75,11 @@ var _ = When("a shared mode cluster update its envs", Label("e2e"), Label(update
 		Expect(ok).To(BeTrue())
 		Expect(serverEnv2).To(Equal("toBeRemoved"))
 
+		var nodes v1.NodeList
+		Expect(k8sClient.List(ctx, &nodes)).To(Succeed())
+
 		aPods := listAgentPods(ctx, virtualCluster)
-		Expect(len(aPods)).To(Equal(1))
+		Expect(aPods).To(HaveLen(len(nodes.Items)))
 
 		agentPod := aPods[0]
 
@@ -136,8 +139,11 @@ var _ = When("a shared mode cluster update its envs", Label("e2e"), Label(update
 			g.Expect(serverEnv3).To(Equal("new"))
 
 			// agent pods
+			var nodes v1.NodeList
+			g.Expect(k8sClient.List(ctx, &nodes)).To(Succeed())
+
 			aPods := listAgentPods(ctx, virtualCluster)
-			g.Expect(len(aPods)).To(Equal(1))
+			g.Expect(aPods).To(HaveLen(len(nodes.Items)))
 
 			agentEnv1, ok := getEnv(&aPods[0], "TEST_AGENT_ENV_1")
 			g.Expect(ok).To(BeTrue())
@@ -362,6 +368,10 @@ var _ = When("a virtual mode cluster update its server args", Label("e2e"), Labe
 	BeforeEach(func() {
 		namespace := NewNamespace()
 
+		DeferCleanup(func() {
+			DeleteNamespaces(namespace.Name)
+		})
+
 		cluster := NewCluster(namespace.Name)
 
 		// Add initial args for server

tests/common_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -47,8 +48,6 @@ func NewVirtualClusterWithType(persistenceType v1beta1.PersistenceMode) *Virtual
 
 	namespace := NewNamespace()
 
-	By(fmt.Sprintf("Creating new virtual cluster in namespace %s", namespace.Name))
-
 	cluster := NewCluster(namespace.Name)
 	cluster.Spec.Persistence.Type = persistenceType
 
@@ -101,6 +100,11 @@ func NewNamespace() *v1.Namespace {
 func DeleteNamespaces(names ...string) {
 	GinkgoHelper()
 
+	if _, found := os.LookupEnv("KEEP_NAMESPACES"); found {
+		By(fmt.Sprintf("Keeping namespace %v", names))
+		return
+	}
+
 	wg := sync.WaitGroup{}
 	wg.Add(len(names))
 
@@ -151,14 +155,16 @@ func NewCluster(namespace string) *v1beta1.Cluster {
 func CreateCluster(cluster *v1beta1.Cluster) {
 	GinkgoHelper()
 
+	By(fmt.Sprintf("Creating new virtual cluster in namespace %s", cluster.Namespace))
+
 	ctx := context.Background()
 	err := k8sClient.Create(ctx, cluster)
 	Expect(err).To(Not(HaveOccurred()))
 
 	expectedServers := int(*cluster.Spec.Servers)
 	expectedAgents := int(*cluster.Spec.Agents)
 
-	By(fmt.Sprintf("Waiting for cluster to be ready. Expected servers: %d. Expected agents: %d", expectedServers, expectedAgents))
+	By(fmt.Sprintf("Waiting for cluster %s to be ready in namespace %s. Expected servers: %d. Expected agents: %d", cluster.Name, cluster.Namespace, expectedServers, expectedAgents))
 
 	// track the Eventually status to log for changes
 	prev := -1
@@ -189,7 +195,11 @@ func CreateCluster(cluster *v1beta1.Cluster) {
 		}
 
 		if prev != (serversReady + agentsReady) {
-			GinkgoLogr.Info("Waiting for pods to be Ready", "servers", serversReady, "agents", agentsReady, "time", time.Now().Format(time.DateTime))
+			GinkgoLogr.Info("Waiting for pods to be Ready",
+				"servers", serversReady, "agents", agentsReady,
+				"name", cluster.Name, "namespace", cluster.Namespace,
+				"time", time.Now().Format(time.DateTime),
+			)
 			prev = (serversReady + agentsReady)
 		}