Skip to content

chore: additional E2E test reliability improvements #1272

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jgwest wants to merge 1 commit into
base: master
Choose a base branch
from
Open

chore: additional E2E test reliability improvements #1272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions test/openshift/e2e/ginkgo/fixture/agent/fixture.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
argov1beta1api "github.com/argoproj-labs/argocd-operator/api/v1beta1"
"github.com/argoproj-labs/argocd-operator/common"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/certutil"
certutilFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/certutil"
k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
osFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/os"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
Expand Down Expand Up @@ -158,8 +158,8 @@ func CreateRequiredSecrets(cfg PrincipalSecretsConfig) {
}
Expect(k8sClient.Create(ctx, jwtSecret)).To(Succeed())

caKey, caCert, caCertPEM := certutil.GenerateCertificateAuthority(caSubject)
caKeyPEM := certutil.EncodePrivateKeyToPEM(caKey)
caKey, caCert, caCertPEM := certutilFixture.GenerateCertificateAuthority(caSubject)
caKeyPEM := certutilFixture.EncodePrivateKeyToPEM(caKey)

caSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -176,7 +176,7 @@ func CreateRequiredSecrets(cfg PrincipalSecretsConfig) {
Expect(k8sClient.Create(ctx, caSecret)).To(Succeed())

principalDNS, principalIPs := aggregateSANs(cfg.PrincipalNamespaceName, cfg.PrincipalServiceName, cfg.AdditionalPrincipalSANs)
principalCertPEM, principalKeyPEM := certutil.IssueCertificate(caCert, caKey, certutil.CertificateRequest{
principalCertPEM, principalKeyPEM := certutilFixture.IssueCertificate(caCert, caKey, certutilFixture.CertificateRequest{
CommonName: cfg.PrincipalServiceName,
DNSNames: principalDNS,
IPAddresses: principalIPs,
Expand All @@ -185,7 +185,7 @@ func CreateRequiredSecrets(cfg PrincipalSecretsConfig) {
createTLSSecret(ctx, k8sClient, cfg.PrincipalNamespaceName, cfg.PrincipalTLSSecretName, principalCertPEM, principalKeyPEM, caCertPEM)

resourceProxyDNS, resourceProxyIPs := aggregateSANs(cfg.PrincipalNamespaceName, cfg.ResourceProxyServiceName, cfg.AdditionalResourceProxySANs)
resourceProxyCertPEM, resourceProxyKeyPEM := certutil.IssueCertificate(caCert, caKey, certutil.CertificateRequest{
resourceProxyCertPEM, resourceProxyKeyPEM := certutilFixture.IssueCertificate(caCert, caKey, certutilFixture.CertificateRequest{
CommonName: cfg.ResourceProxyServiceName,
DNSNames: resourceProxyDNS,
IPAddresses: resourceProxyIPs,
Expand Down Expand Up @@ -218,7 +218,7 @@ func CreateRequiredAgentSecrets(cfg AgentSecretsConfig) {
caKey := parsePrivateKey(caKeyPEM)

clientDNS, clientIPs := aggregateClientSANs(cfg.ClientDNSNames)
clientCertPEM, clientKeyPEM := certutil.IssueCertificate(caCert, caKey, certutil.CertificateRequest{
clientCertPEM, clientKeyPEM := certutilFixture.IssueCertificate(caCert, caKey, certutilFixture.CertificateRequest{
CommonName: cfg.ClientCommonName,
DNSNames: clientDNS,
IPAddresses: clientIPs,
Expand Down
21 changes: 9 additions & 12 deletions test/openshift/e2e/ginkgo/fixture/argocd/fixture.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/retry"
"sigs.k8s.io/controller-runtime/pkg/client"

routeFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/route"
)

// Update will update an ArgoCD CR. Update will keep trying to update object until it succeeds, or times out.
Expand Down Expand Up @@ -267,26 +269,20 @@ func fetchArgoCD(f func(*argov1beta1api.ArgoCD) bool) matcher.GomegaMatcher {

}

// NOTE: this should only be called from sequential tests. If you call it from a parallel test, there is a risk that another test will login to a different Argo CD instance.
func LogInToDefaultArgoCDInstance() error {
k8sClient, _, err := utils.GetE2ETestKubeClientWithError()
if err != nil {
return err
}

var routeList routev1.RouteList
Expect(k8sClient.List(context.Background(), &routeList, client.InNamespace("openshift-gitops"))).To(Succeed())
route := &routev1.Route{ObjectMeta: metav1.ObjectMeta{Name: "openshift-gitops-server", Namespace: "openshift-gitops"}}

var route *routev1.Route
for idx := range routeList.Items {
idxRoute := routeList.Items[idx]
Eventually(func() error {
return k8sClient.Get(context.Background(), client.ObjectKeyFromObject(route), route)
}, "3m", "2s").Should(Succeed())

if idxRoute.Name == "openshift-gitops-server" {
route = &idxRoute
}
}
if route == nil {
return fmt.Errorf("unable to locate route")
}
Eventually(route, "3m", "2s").Should(routeFixture.HaveAdmittedIngress())

secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "openshift-gitops-cluster", Namespace: "openshift-gitops"}}
if err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(secret), secret); err != nil {
Expand All @@ -307,6 +303,7 @@ func LogInToDefaultArgoCDInstance() error {

}

// NOTE: this should only be called from sequential tests. If you call it from a parallel test, there is a risk that another test will login to a different Argo CD instance.
func RunArgoCDCLI(args ...string) (string, error) {

cmdArgs := append([]string{"argocd"}, args...)
Expand Down
21 changes: 16 additions & 5 deletions test/openshift/e2e/ginkgo/fixture/fixture.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ import (

"github.com/onsi/gomega/format"
gitopsoperatorv1alpha1 "github.com/redhat-developer/gitops-operator/api/v1alpha1"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
deploymentFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/deployment"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
osFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/os"
subscriptionFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/subscription"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
Expand Down Expand Up @@ -69,7 +69,7 @@ func EnsureParallelCleanSlate() {
err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(defaultOpenShiftGitOpsArgoCD), defaultOpenShiftGitOpsArgoCD)
Expect(err).ToNot(HaveOccurred())

Eventually(defaultOpenShiftGitOpsArgoCD, "5m", "5s").Should(argocd.BeAvailableWithCustomSleepTime(3 * time.Second))
Eventually(defaultOpenShiftGitOpsArgoCD, "5m", "5s").Should(argocdFixture.BeAvailableWithCustomSleepTime(3 * time.Second))
}
// Unlike sequential clean slate, parallel clean slate cannot assume that there are no other tests running. This limits our ability to clean up old test artifacts.
}
Expand Down Expand Up @@ -119,7 +119,7 @@ func EnsureSequentialCleanSlateWithError() error {
defaultOpenShiftGitOpsArgoCD := &argov1beta1api.ArgoCD{
ObjectMeta: metav1.ObjectMeta{Name: "openshift-gitops", Namespace: "openshift-gitops"},
}
Eventually(defaultOpenShiftGitOpsArgoCD, "3m", "5s").Should(k8s.ExistByName())
Eventually(defaultOpenShiftGitOpsArgoCD, "3m", "5s").Should(k8sFixture.ExistByName())

// Ensure that default state of ArgoCD CR in openshift-gitops is restored
if err := updateWithoutConflict(defaultOpenShiftGitOpsArgoCD, func(obj client.Object) {
Expand Down Expand Up @@ -201,7 +201,7 @@ func EnsureSequentialCleanSlateWithError() error {

// Finally, wait for default openshift-gitops instance to be ready.
failure := InterceptGomegaFailure(func() {
Eventually(defaultOpenShiftGitOpsArgoCD, "5m", "5s").Should(argocd.BeAvailable())
Eventually(defaultOpenShiftGitOpsArgoCD, "5m", "5s").Should(argocdFixture.BeAvailable())
})
// Output debug information on argo startup failure
if failure != nil {
Expand Down Expand Up @@ -931,6 +931,17 @@ func OutputDebug(namespaceParams ...any) {
GinkgoWriter.Println(kubectlOutput)
GinkgoWriter.Println("----------------------------------------------------------------")

kubectlOutput, err = osFixture.ExecCommandWithOutputParam(false, true, "kubectl", "get", "statefulsets", "-n", namespace)
if err != nil {
GinkgoWriter.Println("unable to get statefulsets for namespace", err, kubectlOutput)
} else {
GinkgoWriter.Println("")
GinkgoWriter.Println("----------------------------------------------------------------")
GinkgoWriter.Println("'kubectl get statefulsets -n " + namespace + ":")
GinkgoWriter.Println(kubectlOutput)
GinkgoWriter.Println("----------------------------------------------------------------")
}

kubectlOutput, err = osFixture.ExecCommandWithOutputParam(false, true, "kubectl", "get", "events", "-n", namespace)
if err != nil {
GinkgoWriter.Println("unable to get events for namespace", err, kubectlOutput)
Expand Down
6 changes: 3 additions & 3 deletions test/openshift/e2e/ginkgo/fixture/promoter/fixture.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"

"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/certutil"
certutilFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/certutil"
k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
)
Expand Down Expand Up @@ -49,7 +49,7 @@ func CreateAPIServerTLSSecrets(cfg PromoterAPIServerTLSSecretConfig) {

By("Creating API Server TLS secrets")

caKey, caCert, caCertPEM := certutil.GenerateCertificateAuthority(caSubject)
caKey, caCert, caCertPEM := certutilFixture.GenerateCertificateAuthority(caSubject)
caBundleSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: cfg.CABundleSecretName,
Expand All @@ -61,7 +61,7 @@ func CreateAPIServerTLSSecrets(cfg PromoterAPIServerTLSSecretConfig) {
}
Expect(k8sClient.Create(ctx, caBundleSecret)).To(Succeed())

apiServerCertPEM, apiServerKeyPEM := certutil.IssueCertificate(caCert, caKey, certutil.CertificateRequest{
apiServerCertPEM, apiServerKeyPEM := certutilFixture.IssueCertificate(caCert, caKey, certutilFixture.CertificateRequest{
CommonName: cfg.APIServerServiceName,
DNSNames: []string{
cfg.APIServerServiceName,
Expand Down
6 changes: 3 additions & 3 deletions test/openshift/e2e/ginkgo/parallel/1-009_validate-manage-other-namespace_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (
appFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/application"
argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/namespace"
namespaceFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/namespace"
secretFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/secret"
fixtureUtils "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -71,7 +71,7 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
Eventually(argoCDInRandomNS, "5m", "5s").Should(argocdFixture.BeAvailable())

By("modifying the labels of another namespace to add the argocd managed-by label")
namespace.Update(nsTest_1_9_custom, func(n *corev1.Namespace) {
namespaceFixture.Update(nsTest_1_9_custom, func(n *corev1.Namespace) {
n.Labels["argocd.argoproj.io/managed-by"] = argoCDInRandomNS.Namespace
})

Expand Down Expand Up @@ -113,7 +113,7 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
Eventually(app, "4m", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))

By("removing managed-by label from that other Namespace")
namespace.Update(nsTest_1_9_custom, func(n *corev1.Namespace) {
namespaceFixture.Update(nsTest_1_9_custom, func(n *corev1.Namespace) {
delete(n.Labels, "argocd.argoproj.io/managed-by")
})

Expand Down
9 changes: 7 additions & 2 deletions test/openshift/e2e/ginkgo/parallel/1-023_validate_repo_server_tls_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,9 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
ac.Spec.Repo.VerifyTLS = true
})

Eventually(func() bool {
// Check that the service has what we need, here.

checkArgoCDServer := func() bool {
depl := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "argocd-server", Namespace: nsTest_1_23_custom.Name}}
if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(depl), depl); err != nil {
GinkgoWriter.Println(err)
Expand Down Expand Up @@ -116,7 +118,10 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
"text",
})

}).Should(BeTrue())
}

Eventually(checkArgoCDServer).Should(BeTrue())
Consistently(checkArgoCDServer, "20s", "5s").Should(BeTrue())

Eventually(argoCDTest_1_23_custom, "5m", "5s").Should(argocdFixture.BeAvailable())

Expand Down
2 changes: 2 additions & 0 deletions test/openshift/e2e/ginkgo/parallel/1-030_validate_reencrypt_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,8 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
// Print the response body
GinkgoWriter.Println(string(body))

GinkgoWriter.Println(r.Status.Ingress, r.Spec.Host)

return strings.Contains(string(body), "Your browser does not support JavaScript.")

}, "90s", "5s").Should(BeTrue())
Expand Down
18 changes: 9 additions & 9 deletions test/openshift/e2e/ginkgo/parallel/1-034_validate_webhook_notifications_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ import (
. "github.com/onsi/gomega"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture"
argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/configmap"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/deployment"
configmapFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/configmap"
deploymentFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/deployment"
k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/notificationsconfiguration"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/os"
notificationsconfigurationFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/notificationsconfiguration"
osFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/os"
fixtureUtils "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -281,7 +281,7 @@ Xq+NinfrqOLJkIZ/u/PJu4KqN3M=
}

Expect(k8sClient.Create(ctx, depl)).To(Succeed())
Eventually(depl, "4m", "5s").Should(deployment.HaveAvailableReplicas(1))
Eventually(depl, "4m", "5s").Should(deploymentFixture.HaveAvailableReplicas(1))

argocd := &argov1beta1api.ArgoCD{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -350,7 +350,7 @@ UVwpFuaKz5vTCD36Gmmy/u8y
}
Eventually(nc).Should(k8sFixture.ExistByName())

notificationsconfiguration.Update(nc, func(nc *argov1alpha1api.NotificationsConfiguration) {
notificationsconfigurationFixture.Update(nc, func(nc *argov1alpha1api.NotificationsConfiguration) {

nc.Spec.Services = map[string]string{
"service.webhook.test-webhook": "url: https://webhook/hooks/example",
Expand All @@ -377,7 +377,7 @@ UVwpFuaKz5vTCD36Gmmy/u8y
},
}
Eventually(notifConfigMap).Should(k8sFixture.ExistByName())
Eventually(notifConfigMap).Should(configmap.HaveStringDataKeyValueContainsSubstring("template.test-app-created", `{"created":"{{.app.metadata.name}}","type":"{{(call .repo.GetAppDetails).Type}}"}`))
Eventually(notifConfigMap).Should(configmapFixture.HaveStringDataKeyValueContainsSubstring("template.test-app-created", `{"created":"{{.app.metadata.name}}","type":"{{(call .repo.GetAppDetails).Type}}"}`))

By("creating an Argo CD Application that contains a notification annotation, which will trigger the notifications controller")
app := &argocdv1alpha1.Application{
Expand All @@ -403,14 +403,14 @@ UVwpFuaKz5vTCD36Gmmy/u8y
}
Expect(k8sClient.Create(ctx, app)).To(Succeed())

out, err := os.ExecCommand("kubectl", "-n", ns.Name, "logs", "deployment.apps/argocd-notifications-controller")
out, err := osFixture.ExecCommand("kubectl", "-n", ns.Name, "logs", "deployment.apps/argocd-notifications-controller")
Expect(err).ToNot(HaveOccurred())
Expect(out).ToNot(ContainSubstring("x509"))

By("waiting for notifications controller to POST to the webhook workload, indicating that the workload event was successfully processed")
Eventually(func() bool {

out, err := os.ExecCommand("kubectl", "-n", ns.Name, "logs", "deployment.apps/webhook")
out, err := osFixture.ExecCommand("kubectl", "-n", ns.Name, "logs", "deployment.apps/webhook")
if err != nil {
GinkgoWriter.Println(err)
return false
Expand Down
4 changes: 2 additions & 2 deletions test/openshift/e2e/ginkgo/parallel/1-045_validate_repo_exec_timeout_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (
. "github.com/onsi/gomega"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture"
argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/deployment"
deploymentFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/deployment"
k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
fixtureUtils "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -74,7 +74,7 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
depl := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "argocd-repo-server", Namespace: ns.Name}}
Eventually(depl).Should(k8sFixture.ExistByName())

Eventually(depl).Should(deployment.HaveContainerWithEnvVar("ARGOCD_EXEC_TIMEOUT", "300s", 0))
Eventually(depl).Should(deploymentFixture.HaveContainerWithEnvVar("ARGOCD_EXEC_TIMEOUT", "300s", 0))
})

})
Expand Down
Loading
Loading