test(e2e): Nightly test setup

.github/workflows/e2e-nightly.yaml

@@ -0,0 +1,87 @@
+name: E2E (nightly)
+
+on:
+  # schedule:
+  # Nightly at 03:00 UTC.
+  # - cron: "0 3 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    # Test provisions kind cluster, cert-manager, Argo CD and Kargo, so the full run takes a while.
+    timeout-minutes: 180
+    steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
+      with:
+        egress-policy: audit
+
+    - name: Checkout code
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        # Use the e2e module's toolchain version.
+        go-version-file: hack/test/e2e/go.mod
+        cache-dependency-path: |
+          go.sum
+          hack/test/e2e/go.sum
+
+    # The test shells out to helm, kind, kargo and argocd; install them all into
+    # hack/bin, which is added to PATH for the subsequent steps.
+    - name: Add hack/bin to PATH
+      run: echo "${GITHUB_WORKSPACE}/hack/bin" >> "${GITHUB_PATH}"
+
+    - name: Install helm
+      run: make install-helm
+
+    - name: Build kind
+      run: go build -o hack/bin/kind sigs.k8s.io/kind
+
+    - name: Build kargo CLI
+      run: go build -o hack/bin/kargo ./cmd/cli
+
+    - name: Install argocd CLI
+      run: |
+        curl -fsSL -o hack/bin/argocd \
+          https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
+        chmod +x hack/bin/argocd
+
+    - name: Verify tools
+      run: |
+        go version
+        helm version
+        kind version
+        kargo version --client || true
+        argocd version --client
+
+    # nightly_config.yaml ships with an empty `context` section. Populate it from
+    # repository secrets so the git- and http-dependent suites can run. Each key
+    # is only written when its secret is set; suites whose context is missing
+    # skip or fail on their own.
+    - name: Configure e2e context
+      env:
+        E2E_KARGO_DEMO_GITOPS_REPO: ${{ secrets.E2E_KARGO_DEMO_GITOPS_REPO }}
+        E2E_GIT_PAT: ${{ secrets.E2E_GIT_PAT }}
+        ## TODO: We need to set up the endpoint to ebable that
+        # E2E_HTTP_ENDPOINT: ${{ secrets.E2E_HTTP_ENDPOINT }}
+      run: |
+        cfg=hack/test/e2e/envs/nightly_config.yaml
+        # Drop the existing (last) context section and re-add it from secrets.
+        sed -i '/^context:/,$d' "${cfg}"
+        {
+          echo "context:"
+          [ -n "${E2E_KARGO_DEMO_GITOPS_REPO}" ] && echo "  kargo_demo_gitops_repo: ${E2E_KARGO_DEMO_GITOPS_REPO}"
+          [ -n "${E2E_GIT_PAT}" ] && echo "  git_pat: ${E2E_GIT_PAT}"
+          [ -n "${E2E_HTTP_ENDPOINT}" ] && echo "  http_endpoint: ${E2E_HTTP_ENDPOINT}"
+        } >> "${cfg}"
+
+    - name: Run e2e tests
+      working-directory: hack/test/e2e
+      run: |
+        go test -tags=e2e,shared ./suites/shared -timeout 120m -args -env-file=nightly_config.yaml

hack/test/e2e/envs/nightly_config.yaml

@@ -0,0 +1,43 @@
+## Environment template used to configure nightly e2e runs.
+
+## It is based on kind_config.yaml and is going to use `kind` and `helm` to set up kargo and argocd instances.
+description: "Nightly test wth kind cluster"
+
+cluster:
+  name: kargo-e2e
+
+## cert-manager is a prerequisite for Kargo's self-signed certificates.
+cert_manager:
+  namespace: cert-manager
+  timeout: 10m
+
+argocd:
+  namespace: argocd
+  timeout: 10m
+  values_files: [
+    ../../values.argocd.test.yaml
+  ]
+
+kargo:
+  namespace: kargo
+  timeout: 10m
+  values_files: [
+    ../../values.test.yaml
+  ]
+
+kargo_cli:
+  ## Indicate that we need to run `kargo login` to login into a kargo instance in `kind` cluster
+  kargo_login:
+    ## Uste tmp directory to store kargo config
+    use_tmp_config_home: true
+
+argocd_cli:
+  ## Using system-local argocd config created by `argocd login`
+  config_file: ~/.config/argocd/config
+  ## Indicate that we need to run `argocd login` to login to an argocd instance in `kind` cluster
+  argocd_login: true
+
+context:
+  # kargo_demo_gitops_repo: <specify a repo here>
+  # git_pat: <specify a PAT here>
+  # http_endpoint: <URL of an HTTP endpoint that returns 2xx to a POST>

hack/test/e2e/framework/utils/features.go

@@ -0,0 +1,5 @@
+package utils
+
+import "sigs.k8s.io/e2e-framework/pkg/features"
+
+var TestFeatures []features.Feature

hack/test/e2e/framework/utils/kargo_fixtures.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
+	"io/fs"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -26,6 +26,8 @@ import (
 const groupKargo = "kargo"
 const KargoCLIKey envfuncs.ContextKey = "kargo_cli"
 const KargoCLIWatchKey envfuncs.ContextKey = "kargo_watch"
+const TestDataPath = "testdata"
+const TestDataKey envfuncs.ContextKey = "test_data"
 
 func SetupKargoClients(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 	ctx = SetupKargoApiClient(ctx, t, cfg)
@@ -113,23 +115,34 @@ func TeardownKargoFixtures(ctx context.Context, t *testing.T, cfg *envconf.Confi
 	return TeardownKargoFixturesWithOptions(ctx, t, cfg)
 }
 
+func TestData(testData fs.FS) features.Func {
+	return func(ctx context.Context, _ *testing.T, _ *envconf.Config) context.Context {
+		return context.WithValue(ctx, TestDataKey, testData)
+	}
+}
+
 func scanFixtures(
 	ctx context.Context,
 	group string,
 	sortFun func([]string) []string,
 	handlerFun decoder.HandlerFunc,
-	options ...decoder.DecodeOption) error {
+	options ...decoder.DecodeOption,
+) error {
+	testData, ok := ctx.Value(TestDataKey).(fs.FS)
+	if !ok {
+		return fmt.Errorf("unable to get testdata from context")
+	}
 
-	fixturesDir := filepath.Join("testdata", group)
-	files, err := filepath.Glob(filepath.Join(fixturesDir, "*.yaml"))
+	fixturesDir := filepath.Join(TestDataPath, group)
+	files, err := fs.Glob(testData, filepath.Join(fixturesDir, "*.yaml"))
 	if err != nil {
 		return err
 	}
 
 	files = sortFun(files)
 
 	for _, file := range files {
-		err := scanFile(ctx, file, handlerFun, options...)
+		err := scanFile(ctx, testData, file, handlerFun, options...)
 		if err != nil {
 			return err
 		}
@@ -140,11 +153,12 @@ func scanFixtures(
 
 func scanFile(
 	ctx context.Context,
+	testData fs.FS,
 	fileName string,
 	handlerFun decoder.HandlerFunc,
 	options ...decoder.DecodeOption,
 ) error {
-	f, err := os.Open(fileName)
+	f, err := testData.Open(fileName)
 	if err != nil {
 		return err
 	}
@@ -153,7 +167,7 @@ func scanFile(
 	if err != nil {
 		return err
 	}
-	return f.Close()
+	return nil
 }
 
 func sortDesc(sorted []string) []string {
@@ -180,7 +194,6 @@ func KargoCreateHandler() decoder.HandlerFunc {
 		}
 
 		manifest, err := yaml.Marshal(obj)
-		fmt.Printf("Creating resource: %v\n", obj.GetObjectKind())
 		if err != nil {
 			return fmt.Errorf("error encoding kargo resource manifest: %w", err)
 		}

hack/test/e2e/framework/utils/kargo_functions.go

@@ -3,7 +3,6 @@ package utils
 
 import (
 	"errors"
-	"fmt"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/e2e-framework/klient/decoder"
@@ -13,16 +12,13 @@ import (
 func UpdatePromotionTasksVar(name, key, val string) decoder.DecodeOption {
 	return MutateAsUnstructuredOptionFor("PromotionTask", name, func(unstr runtime.Unstructured) error {
 		data := unstr.UnstructuredContent()
-		fmt.Printf("Parsed data %v\n", data)
 		for _, tplVar := range data["spec"].(map[string]any)["vars"].([]any) {
 			tplVarMap := tplVar.(map[string]any)
 			if tplVarMap["name"] == key {
 				tplVarMap["value"] = val
 			}
 		}
 
-		fmt.Printf("Updated data %v\n", data)
-
 		unstr.SetUnstructuredContent(data)
 		return nil
 	})
@@ -31,7 +27,6 @@ func UpdatePromotionTasksVar(name, key, val string) decoder.DecodeOption {
 func UpdateWarehouseGitRepoURL(name, repoURL string) decoder.DecodeOption {
 	return MutateAsUnstructuredOptionFor("Warehouse", name, func(unstr runtime.Unstructured) error {
 		data := unstr.UnstructuredContent()
-		fmt.Printf("Parsed data %v\n", data)
 		for _, sub := range data["spec"].(map[string]any)["subscriptions"].([]any) {
 			subMap := sub.(map[string]any)
 
@@ -41,8 +36,6 @@ func UpdateWarehouseGitRepoURL(name, repoURL string) decoder.DecodeOption {
 			}
 		}
 
-		fmt.Printf("Updated data %v\n", data)
-
 		unstr.SetUnstructuredContent(data)
 		return nil
 	})