test(e2e): Nightly test setup

.github/workflows/e2e-nightly.yaml

@@ -0,0 +1,91 @@
+name: E2E (nightly)
+
+on:
+  # schedule:
+  # Nightly at 03:00 UTC.
+  # - cron: "0 3 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    # Test provisions kind cluster, cert-manager, Argo CD and Kargo, so the full run takes a while.
+    timeout-minutes: 180
+    steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
+      with:
+        egress-policy: audit
+
+    - name: Checkout code
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        # Use the e2e module's toolchain version.
+        go-version-file: hack/test/e2e/go.mod
+        cache-dependency-path: |
+          go.sum
+          hack/test/e2e/go.sum
+
+    # The test shells out to helm, kind, kargo and argocd; install them all into
+    # hack/bin, which is added to PATH for the subsequent steps.
+    - name: Add hack/bin to PATH
+      run: echo "${GITHUB_WORKSPACE}/hack/bin" >> "${GITHUB_PATH}"
+
+    - name: Install helm
+      run: make install-helm
+
+    - name: Build kind
+      run: go build -o hack/bin/kind sigs.k8s.io/kind
+
+    - name: Build kargo CLI
+      run: go build -o hack/bin/kargo ./cmd/cli
+
+    - name: Install argocd CLI
+      run: |
+        curl -fsSL -o hack/bin/argocd \
+          https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
+        chmod +x hack/bin/argocd
+
+    - name: Verify tools
+      run: |
+        go version
+        helm version
+        kind version
+        kargo version --client || true
+        argocd version --client
+
+    # nightly_config.yaml ships with an empty `context` section. Populate it from
+    # repository secrets so the git- and http-dependent suites can run. Each key
+    # is only written when its secret is set; suites whose context is missing
+    # skip or fail on their own.
+    - name: Configure e2e context
+      env:
+        E2E_KARGO_DEMO_GITOPS_REPO: ${{ secrets.E2E_KARGO_DEMO_GITOPS_REPO }}
+        E2E_GIT_PAT: ${{ secrets.E2E_GIT_PAT }}
+        ## TODO: We need to set up the endpoint to ebable that
+        # E2E_HTTP_ENDPOINT: ${{ secrets.E2E_HTTP_ENDPOINT }}
+      run: |
+        cfg=hack/test/e2e/envs/nightly_config.yaml
+        # Drop the existing (last) context section and re-add it from secrets.
+        sed -i '/^context:/,$d' "${cfg}"
+        {
+          echo "context:"
+          [ -n "${E2E_KARGO_DEMO_GITOPS_REPO}" ] && echo "  kargo_demo_gitops_repo: ${E2E_KARGO_DEMO_GITOPS_REPO}"
+          [ -n "${E2E_GIT_PAT}" ] && echo "  git_pat: ${E2E_GIT_PAT}"
+          [ -n "${E2E_HTTP_ENDPOINT}" ] && echo "  http_endpoint: ${E2E_HTTP_ENDPOINT}"
+        } >> "${cfg}"
+
+    - name: Build kargo image
+      run: |
+        make hack-build
+
+    - name: Run e2e tests
+      working-directory: hack/test/e2e
+      run: |
+        go test -tags=e2e,shared ./suites/shared -timeout 120m -args -env-file=nightly_config.yaml

hack/test/e2e/envs/kind_config.yaml

@@ -30,6 +30,11 @@ cert_manager:
   set: []
   values_files: []
 
+## argo-rollouts are required for AnalysisTemplate in kargo
+argo-rollouts:
+  namespace: argo-rollouts
+  timeout: 10m
+
 argocd:
   release_name: argocd
   namespace: argocd

hack/test/e2e/envs/nightly_config.yaml

@@ -0,0 +1,51 @@
+## Environment template used to configure nightly e2e runs.
+
+## It is based on kind_config.yaml and is going to use `kind` and `helm` to set up kargo and argocd instances.
+description: "Nightly test wth kind cluster"
+
+cluster:
+  name: kargo-e2e
+
+## cert-manager is a prerequisite for Kargo's self-signed certificates.
+cert_manager:
+  namespace: cert-manager
+  timeout: 10m
+
+argocd:
+  namespace: argocd
+  timeout: 10m
+  values_files: [
+    ../../values.argocd.test.yaml
+  ]
+
+argo-rollouts:
+  namespace: argo-rollouts
+  timeout: 10m
+
+kargo:
+  namespace: kargo
+  ## Using a chart directly from the repo
+  chart: ../../../../../charts/kargo
+  ## Using an image built with make hack-build
+  image: docker.io/library/kargo:dev
+  timeout: 10m
+  values_files: [
+    ../../values.test.yaml
+  ]
+
+kargo_cli:
+  ## Indicate that we need to run `kargo login` to login into a kargo instance in `kind` cluster
+  kargo_login:
+    ## Uste tmp directory to store kargo config
+    use_tmp_config_home: true
+
+argocd_cli:
+  ## Using system-local argocd config created by `argocd login`
+  config_file: ~/.config/argocd/config
+  ## Indicate that we need to run `argocd login` to login to an argocd instance in `kind` cluster
+  argocd_login: true
+
+context:
+  # kargo_demo_gitops_repo: <specify a repo here>
+  # git_pat: <specify a PAT here>
+  # http_endpoint: <URL of an HTTP endpoint that returns 2xx to a POST>

hack/test/e2e/framework/envfuncs/cluster.go

@@ -25,6 +25,7 @@ const ClusterNameKey ContextKey = "cluster_name"
 
 // KargoHostKey holds a hostname used in KargoLogin. Populated by InstallKargo
 const KargoHostKey ContextKey = "kargo_host"
+
 // KargoPasswordKey holds a password used in KargoLogin. Populated by InstallKargo
 const KargoPasswordKey ContextKey = "kargo_password"
 
@@ -39,6 +40,7 @@ func ClusterSetupFuncs() []env.Func {
 		CreateKindCluster,
 		InstallCertManager,
 		InstallArgoCD,
+		InstallArgoRollouts,
 		InstallKargo,
 	}
 }
@@ -70,7 +72,7 @@ func CreateKindCluster(ctx context.Context, cfg *envconf.Config) (context.Contex
 	}
 
 	provider := kind.NewProvider()
-	
+
 	tempdir := ctx.Value(TmpDirKey)
 	if tempdir == nil {
 		return ctx, fmt.Errorf("Temp dir is not set up. Cannot create kubeconfig")
@@ -178,7 +180,7 @@ func InstallArgoCD(ctx context.Context, cfg *envconf.Config) (context.Context, e
 	}
 	// Port from portForward above
 	ctx = context.WithValue(ctx, ArgocdHostKey, "localhost:8080")
-	// Auth info. 
+	// Auth info.
 	// FIXME: the values require setValues to have configs.secret.argocdServerAdminPassword set
 	// Currently set in values.argocd.test.yaml
 	ctx = context.WithValue(ctx, ArgocdUsernameKey, "admin")
@@ -239,17 +241,46 @@ func InstallKargo(ctx context.Context, cfg *envconf.Config) (context.Context, er
 	return ctx, nil
 }
 
+// InstallArgoRollouts installs the Argo Rollouts Helm chart.
+// It no-ops when no managed cluster is present in the context.
+func InstallArgoRollouts(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+	if _, ok := managedClusterName(ctx); !ok {
+		return ctx, nil
+	}
+	if _, err := GetEnvMap(ctx, []string{"argo-rollouts"}); err != nil {
+		fmt.Println("No `argo-rollouts` section in env; skipping argo rollouts installation")
+		return ctx, nil
+	}
+	chart := helmChart{
+		releaseName: optionalString(ctx, []string{"argo-rollouts", "release_name"}, "argo-rollouts"),
+		chart:       optionalString(ctx, []string{"argo-rollouts", "chart"}, "argo/argo-rollouts"),
+		namespace:   optionalString(ctx, []string{"argo-rollouts", "namespace"}, "argo-rollouts"),
+		version:     optionalString(ctx, []string{"argo-rollouts", "version"}, ""),
+		repoName:    optionalString(ctx, []string{"argo-rollouts", "chart_repo_name"}, "argo"),
+		repoURL:     optionalString(ctx, []string{"argo-rollouts", "chart_repo_url"}, "https://argoproj.github.io/argo-helm"),
+		timeout:     optionalString(ctx, []string{"argo-rollouts", "timeout"}, "10m"),
+		valuesFiles: expandHomeAll(optionalStringSlice(ctx, []string{"argo-rollouts", "values_files"})),
+		setValues:   optionalStringSlice(ctx, []string{"argo-rollouts", "set"}),
+	}
+	fmt.Println("Installing Argo Rollouts")
+	if err := chart.install(cfg.KubeconfigFile()); err != nil {
+		return ctx, fmt.Errorf("installing Argo Rollouts: %w", err)
+	}
+
+	return ctx, nil
+}
+
 func portForward(kubeconfig, namespace, service string, outport, inport int) error {
 	// Run port-forward in background.
 	// This is a simplified approach when we just run a background shell.
 	// There is no error handling here, it might fail silently.
 	// FIXME: replace that with goroutine and error channels?
 	// FIXME: implement forwarding to an non-predefined port
-	cmd := fmt.Sprintf("sh -c \"kubectl port-forward --kubeconfig %s --namespace %s %s %d:%d > /dev/null 2>&1 &\"", 
+	cmd := fmt.Sprintf("sh -c \"kubectl port-forward --kubeconfig %s --namespace %s %s %d:%d > /dev/null 2>&1 &\"",
 		kubeconfig, namespace, service, outport, inport)
 
-	fmt.Printf("Port forwarding %s to %d", service, outport)
-	
+	fmt.Printf("Port forwarding %s to %d\n", service, outport)
+
 	p := utils.RunCommand(cmd)
 	if p.Err() != nil {
 		outBytes, outErr := io.ReadAll(p.Out())

hack/test/e2e/framework/utils/features.go

@@ -0,0 +1,5 @@
+package utils
+
+import "sigs.k8s.io/e2e-framework/pkg/features"
+
+var TestFeatures []features.Feature