diff --git a/internal/api/v1/application/create.go b/internal/api/v1/application/create.go index 670ed0384c..d9e3f71ce6 100644 --- a/internal/api/v1/application/create.go +++ b/internal/api/v1/application/create.go @@ -94,6 +94,15 @@ func Create(c *gin.Context) apierror.APIErrors { return apierror.NewMultiError(theIssues) } + if createRequest.Configuration.Processes != nil { + for _, issue := range application.ValidateProcesses(*createRequest.Configuration.Processes) { + theIssues = append(theIssues, apierror.NewBadRequestError(issue.Error())) + } + if len(theIssues) > 0 { + return apierror.NewMultiError(theIssues) + } + } + var routes []string if createRequest.Configuration.Routes != nil { // Note: Routes can be empty here! @@ -115,6 +124,11 @@ func Create(c *gin.Context) apierror.APIErrors { } routes = []string{route} } + if createRequest.Configuration.Processes != nil { + if issue := application.ValidateProcessRoutes(*createRequest.Configuration.Processes, routes); issue != nil { + return apierror.NewBadRequestError(issue.Error()) + } + } apierr := validateRoutes(ctx, cluster, appRef.Name, appRef.Namespace, routes) if apierr != nil { @@ -139,7 +153,7 @@ func Create(c *gin.Context) apierror.APIErrors { // Arguments found OK, now we can modify the system state err = application.Create(ctx, cluster, appRef, username, routes, chart, - createRequest.Configuration.Settings) + createRequest.Configuration.Settings, processValue(createRequest.Configuration.Processes)) if err != nil { return apierror.InternalError(err) } @@ -172,6 +186,13 @@ func Create(c *gin.Context) apierror.APIErrors { return nil } +func processValue(processes *models.ApplicationProcesses) models.ApplicationProcesses { + if processes == nil { + return nil + } + return *processes +} + func validateRoutes(ctx context.Context, cluster *kubernetes.Cluster, appName, namespace string, desiredRoutes []string) apierror.APIErrors { desiredRoutesMap := map[string]struct{}{} for _, desiredRoute := range desiredRoutes { diff --git a/internal/api/v1/application/update.go b/internal/api/v1/application/update.go index 5b8f7b1b35..81da346c96 100644 --- a/internal/api/v1/application/update.go +++ b/internal/api/v1/application/update.go @@ -13,6 +13,7 @@ package application import ( "context" + "encoding/json" "fmt" "net/url" "strings" @@ -88,13 +89,36 @@ func Update(c *gin.Context) apierror.APIErrors { // nolint:gocyclo // simplifica len(updateRequest.Settings) == 0 && updateRequest.Configurations == nil && updateRequest.Routes == nil && - updateRequest.AppChart == "" { + updateRequest.AppChart == "" && + updateRequest.Processes == nil { log.Infow("updating app -- no changes") response.OK(c) return nil } + if updateRequest.Processes != nil { + if issues := application.ValidateProcesses(*updateRequest.Processes); len(issues) > 0 { + apiIssues := make([]apierror.APIError, 0, len(issues)) + for _, issue := range issues { + apiIssues = append(apiIssues, apierror.NewBadRequestError(issue.Error())) + } + return apierror.NewMultiError(apiIssues) + } + } + + effectiveProcesses := app.Configuration.Processes + if updateRequest.Processes != nil { + effectiveProcesses = *updateRequest.Processes + } + effectiveRoutes := app.Configuration.Routes + if updateRequest.Routes != nil { + effectiveRoutes = updateRequest.Routes + } + if issue := application.ValidateProcessRoutes(effectiveProcesses, effectiveRoutes); issue != nil { + return apierror.NewBadRequestError(issue.Error()) + } + if app.Workload != nil { // For a running application we have to validate changed custom chart values against // the configured app chart. It has to be done first, this ensures that there will @@ -199,6 +223,13 @@ func Update(c *gin.Context) apierror.APIErrors { // nolint:gocyclo // simplifica } } + if updateRequest.Processes != nil { + log.Infow("updating app", "processes", *updateRequest.Processes) + if err := updateProcesses(ctx, client, namespace, appName, *updateRequest.Processes); err != nil { + return apierror.InternalError(err) + } + } + // backward compatibility: if no flag provided then restart the app restart := updateRequest.Restart == nil || *updateRequest.Restart if restart { @@ -375,3 +406,19 @@ func updateChartValueSettings( _, err := client.Namespace(namespace).Patch(ctx, appName, types.JSONPatchType, []byte(patch), metav1.PatchOptions{}) return err } + +func updateProcesses( + ctx context.Context, + client dynamic.NamespaceableResourceInterface, + namespace string, + appName string, + processes models.ApplicationProcesses, +) error { + value, err := json.Marshal(processes) + if err != nil { + return err + } + patch := fmt.Sprintf(`[{"op":"add","path":"/spec/processes","value":%s}]`, value) + _, err = client.Namespace(namespace).Patch(ctx, appName, types.JSONPatchType, []byte(patch), metav1.PatchOptions{}) + return err +} diff --git a/internal/api/v1/deploy/deploy.go b/internal/api/v1/deploy/deploy.go index bae185ffcb..2d3630dcf0 100644 --- a/internal/api/v1/deploy/deploy.go +++ b/internal/api/v1/deploy/deploy.go @@ -153,6 +153,7 @@ func deployApp(ctx context.Context, cluster *kubernetes.Cluster, app models.AppR Domains: domains, Start: start, Settings: appObj.Configuration.Settings, + Processes: appObj.Configuration.Processes, } log.Infow("deploying app", "namespace", app.Namespace, "app", app.Name) diff --git a/internal/application/application.go b/internal/application/application.go index bd08d95856..d6869a5c4a 100644 --- a/internal/application/application.go +++ b/internal/application/application.go @@ -153,6 +153,7 @@ func Create( routes []string, chart string, settings models.ChartValueSettings, + processes models.ApplicationProcesses, ) error { client, err := cluster.ClientApp() if err != nil { @@ -178,6 +179,15 @@ func Create( if err != nil { return err } + if processes != nil { + processData, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&processes) + if err != nil { + return err + } + if err := unstructured.SetNestedMap(u, processData, "spec", "processes"); err != nil { + return err + } + } us := &unstructured.Unstructured{Object: u} us.SetAPIVersion("application.epinio.io/v1") us.SetKind("App") @@ -1028,6 +1038,28 @@ func Settings(app *unstructured.Unstructured) (models.ChartValueSettings, error) return settings, nil } +// Processes returns the release-scoped process definitions stored on the app. +// A missing field preserves the existing single-process application behavior. +func Processes(app *unstructured.Unstructured) (models.ApplicationProcesses, error) { + processData, found, err := unstructured.NestedMap( + app.UnstructuredContent(), + "spec", + "processes", + ) + if err != nil { + return nil, errors.Wrap(err, "processes should be an object") + } + if !found { + return nil, nil + } + + processes := models.ApplicationProcesses{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(processData, &processes); err != nil { + return nil, errors.Wrap(err, "decoding processes") + } + return processes, nil +} + /* StageID returns the stage ID of the last attempt at staging, if one exists. It returns an empty string otherwise. The information is pulled out of the @@ -1609,6 +1641,11 @@ func aggregate(ctx context.Context, return nil, errors.Wrap(err, "finding settings") } + processes, err := Processes(&appCR) + if err != nil { + return nil, errors.Wrap(err, "finding processes") + } + desiredRoutes, err := DesiredRoutes(&appCR) if err != nil { return nil, errors.Wrap(err, "finding desired routes") @@ -1628,6 +1665,7 @@ func aggregate(ctx context.Context, app.Configuration.Routes = desiredRoutes app.Configuration.AppChart = chartName app.Configuration.Settings = settings + app.Configuration.Processes = processes app.Origin = origin app.StageID = stageID app.ImageURL = imageURL @@ -1799,6 +1837,14 @@ func fetch(ctx context.Context, cluster *kubernetes.Cluster, app *models.App) er return err } + processes, err := Processes(applicationCR) + if err != nil { + err = errors.Wrap(err, "finding processes") + app.StatusMessage = err.Error() + app.Status = models.ApplicationError + return err + } + app.Meta.CreatedAt = applicationCR.GetCreationTimestamp() app.Configuration.Instances = &instances @@ -1809,6 +1855,7 @@ func fetch(ctx context.Context, cluster *kubernetes.Cluster, app *models.App) er app.Configuration.Routes = desiredRoutes app.Configuration.AppChart = chartName app.Configuration.Settings = settings + app.Configuration.Processes = processes app.Origin = origin app.StageID = stageID app.ImageURL = imageURL diff --git a/internal/application/processes.go b/internal/application/processes.go new file mode 100644 index 0000000000..91e3dc6b99 --- /dev/null +++ b/internal/application/processes.go @@ -0,0 +1,96 @@ +// Copyright © 2026 SUSE LLC +// Licensed under the Apache License, Version 2.0. + +package application + +import ( + "fmt" + + "github.com/epinio/epinio/pkg/api/core/v1/models" + "k8s.io/apimachinery/pkg/util/validation" +) + +// ValidateProcesses validates the deliberately small multi-process POC model. +// Kubernetes remains responsible for validating the cron schedule itself. +func ValidateProcesses(processes models.ApplicationProcesses) []error { + var issues []error + routeProcesses := 0 + releaseProcesses := 0 + + for name, process := range processes { + if problems := validation.IsDNS1123Label(name); len(problems) > 0 { + issues = append(issues, fmt.Errorf("process %q has an invalid name: %s", name, problems[0])) + } + if len(process.Command) == 0 { + issues = append(issues, fmt.Errorf("process %q must define a command", name)) + } + + kind := process.Kind + if kind == "" { + kind = models.ApplicationProcessDeployment + } + + switch kind { + case models.ApplicationProcessDeployment: + if process.Replicas != nil && *process.Replicas < 0 { + issues = append(issues, fmt.Errorf("process %q replicas must be zero or greater", name)) + } + if process.Schedule != "" { + issues = append(issues, fmt.Errorf("deployment process %q cannot define a schedule", name)) + } + case models.ApplicationProcessCron: + if process.Schedule == "" { + issues = append(issues, fmt.Errorf("cron process %q must define a schedule", name)) + } + if process.Replicas != nil { + issues = append(issues, fmt.Errorf("cron process %q cannot define replicas", name)) + } + if process.Routes { + issues = append(issues, fmt.Errorf("cron process %q cannot receive routes", name)) + } + case models.ApplicationProcessRelease: + releaseProcesses++ + if process.Replicas != nil { + issues = append(issues, fmt.Errorf("release process %q cannot define replicas", name)) + } + if process.Schedule != "" { + issues = append(issues, fmt.Errorf("release process %q cannot define a schedule", name)) + } + if process.Routes { + issues = append(issues, fmt.Errorf("release process %q cannot receive routes", name)) + } + default: + issues = append(issues, fmt.Errorf("process %q has unknown kind %q", name, process.Kind)) + } + + if process.Routes { + routeProcesses++ + } + } + + if routeProcesses > 1 { + issues = append(issues, fmt.Errorf("only one deployment process may receive application routes")) + } + if releaseProcesses > 1 { + issues = append(issues, fmt.Errorf("only one release process is supported")) + } + + return issues +} + +// ValidateProcessRoutes ensures that external application routes have a +// process Service to target. A nil process map is the legacy single-process +// model and remains valid for backward compatibility. +func ValidateProcessRoutes(processes models.ApplicationProcesses, routes []string) error { + if processes == nil || len(routes) == 0 { + return nil + } + + for _, process := range processes { + if process.Routes { + return nil + } + } + + return fmt.Errorf("application routes require one deployment process with routes enabled") +} diff --git a/internal/application/processes_test.go b/internal/application/processes_test.go new file mode 100644 index 0000000000..7d46b86681 --- /dev/null +++ b/internal/application/processes_test.go @@ -0,0 +1,87 @@ +// Copyright © 2026 SUSE LLC +// Licensed under the Apache License, Version 2.0. + +package application_test + +import ( + "github.com/epinio/epinio/internal/application" + "github.com/epinio/epinio/pkg/api/core/v1/models" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +var _ = Describe("application processes", func() { + It("accepts deployment, cron, and release processes", func() { + two := int32(2) + processes := models.ApplicationProcesses{ + "web": {Kind: models.ApplicationProcessDeployment, Command: []string{"web"}, Replicas: &two, Routes: true}, + "scheduler": {Kind: models.ApplicationProcessCron, Command: []string{"cron"}, Schedule: "*/5 * * * *"}, + "release": {Kind: models.ApplicationProcessRelease, Command: []string{"migrate"}}, + } + + Expect(application.ValidateProcesses(processes)).To(BeEmpty()) + }) + + It("accepts an explicit zero deployment replica count", func() { + zero := int32(0) + processes := models.ApplicationProcesses{ + "worker": {Command: []string{"worker"}, Replicas: &zero}, + } + + Expect(application.ValidateProcesses(processes)).To(BeEmpty()) + }) + + It("rejects incompatible process fields", func() { + one := int32(1) + processes := models.ApplicationProcesses{ + "bad cron": {Kind: models.ApplicationProcessCron, Command: []string{"cron"}, Replicas: &one}, + "release": {Kind: models.ApplicationProcessRelease}, + } + + Expect(application.ValidateProcesses(processes)).To(HaveLen(4)) + }) + + It("rejects multiple routed or release processes", func() { + processes := models.ApplicationProcesses{ + "web": {Command: []string{"web"}, Routes: true}, + "admin": {Command: []string{"admin"}, Routes: true}, + "release": {Kind: models.ApplicationProcessRelease, Command: []string{"migrate"}}, + "release2": {Kind: models.ApplicationProcessRelease, Command: []string{"migrate-again"}}, + } + + Expect(application.ValidateProcesses(processes)).To(HaveLen(2)) + }) + + It("requires a routed process when a process application has external routes", func() { + processes := models.ApplicationProcesses{ + "worker": {Command: []string{"worker"}}, + } + + Expect(application.ValidateProcessRoutes(processes, []string{"example.test"})).To(MatchError( + "application routes require one deployment process with routes enabled", + )) + Expect(application.ValidateProcessRoutes(processes, nil)).To(Succeed()) + Expect(application.ValidateProcessRoutes(nil, []string{"legacy.example.test"})).To(Succeed()) + }) + + It("decodes processes from the application CR", func() { + app := &unstructured.Unstructured{Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "processes": map[string]interface{}{ + "web": map[string]interface{}{ + "kind": "deployment", + "command": []interface{}{"python", "app.py"}, + "replicas": int64(2), + "routes": true, + }, + }, + }, + }} + + processes, err := application.Processes(app) + Expect(err).ToNot(HaveOccurred()) + Expect(processes["web"].Command).To(Equal([]string{"python", "app.py"})) + Expect(*processes["web"].Replicas).To(Equal(int32(2))) + }) +}) diff --git a/internal/helm/helm.go b/internal/helm/helm.go index 24636decea..6b6efeff95 100644 --- a/internal/helm/helm.go +++ b/internal/helm/helm.go @@ -77,6 +77,7 @@ type ChartParameters struct { Domains domain.DomainMap // Map of domains with secrets covering them Start *int64 // Nano-epoch of deployment. Optional. Used to force a restart, even when nothing else has changed. Settings models.ChartValueSettings + Processes models.ApplicationProcesses } func Values( @@ -322,19 +323,21 @@ type RouteParam struct { Secret string `yaml:"secret,omitempty"` // nolint:gosec // route secret for ingress, not credentials } type EpinioParam struct { - AppName string `yaml:"appName"` - Configurations []string `yaml:"configurations"` - ConfigPaths []ConfigParameter `yaml:"configpaths"` - Env []models.EnvVariable `yaml:"env"` - ImageUrl string `yaml:"imageURL"` - Ingress string `yaml:"ingress,omitempty"` - Gateway string `yaml:"gateway,omitempty"` - ReplicaCount int32 `yaml:"replicaCount"` - Routes []RouteParam `yaml:"routes"` - StageID string `yaml:"stageID"` - Start string `yaml:"start,omitempty"` - TlsIssuer string `yaml:"tlsIssuer"` - Username string `yaml:"username"` + AppName string `yaml:"appName"` + Configurations []string `yaml:"configurations"` + ConfigPaths []ConfigParameter `yaml:"configpaths"` + Env []models.EnvVariable `yaml:"env"` + ImageUrl string `yaml:"imageURL"` + Ingress string `yaml:"ingress,omitempty"` + Gateway string `yaml:"gateway,omitempty"` + ReplicaCount int32 `yaml:"replicaCount"` + Routes []RouteParam `yaml:"routes"` + StageID string `yaml:"stageID"` + Staged bool `yaml:"staged,omitempty"` + Start string `yaml:"start,omitempty"` + TlsIssuer string `yaml:"tlsIssuer"` + Username string `yaml:"username"` + Processes models.ApplicationProcesses `yaml:"processes,omitempty"` } type ChartParam struct { Epinio EpinioParam `yaml:"epinio"` @@ -717,8 +720,10 @@ func getValuesYAML(appChart *models.AppChartFull, parameters ChartParameters) (s Configurations: configurationNames, ConfigPaths: parameters.Configurations, StageID: parameters.StageID, + Staged: isStagedImage(parameters.ImageURL, parameters.StageID), TlsIssuer: viper.GetString("tls-issuer"), Username: parameters.Username, + Processes: parameters.Processes, // Ingress, Gateway, Start, Routes: see below }, // Chart, User: see below @@ -824,3 +829,11 @@ func getValuesYAML(appChart *models.AppChartFull, parameters ChartParameters) (s logger.Infow("deploy app, return values.yaml") return yamlString, nil } + +// isStagedImage identifies the image naming contract produced by Epinio's +// staging pipeline. Matching the complete tag avoids classifying an unrelated +// prebuilt image as staged merely because its repository or tag contains the +// stage ID as a substring. +func isStagedImage(imageURL, stageID string) bool { + return stageID != "" && strings.HasSuffix(imageURL, ":"+stageID) +} diff --git a/internal/helm/helm_test.go b/internal/helm/helm_test.go index 471923238e..465e9b2927 100644 --- a/internal/helm/helm_test.go +++ b/internal/helm/helm_test.go @@ -12,12 +12,60 @@ package helm import ( + "strings" + "github.com/epinio/epinio/pkg/api/core/v1/models" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) +var _ = Describe("multi-process values", func() { + It("passes structured process definitions under epinio.processes", func() { + two := int32(2) + values, err := getValuesYAML(&models.AppChartFull{}, ChartParameters{ + AppRef: models.NewAppRef("demo", "workspace"), + Processes: models.ApplicationProcesses{ + "web": { + Kind: models.ApplicationProcessDeployment, + Command: []string{"python", "app.py"}, + Replicas: &two, + Routes: true, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(values).To(ContainSubstring("processes:")) + Expect(values).To(ContainSubstring("command:")) + Expect(values).To(ContainSubstring("replicas: 2")) + Expect(strings.Count(values, "python")).To(Equal(1)) + }) + + It("marks a staged application image so charts preserve the CNB launcher", func() { + values, err := getValuesYAML(&models.AppChartFull{}, ChartParameters{ + AppRef: models.NewAppRef("demo", "workspace"), + ImageURL: "registry.example/apps/demo:stage-123", + StageID: "stage-123", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(values).To(ContainSubstring("staged: true")) + }) + + It("does not mark a prebuilt image containing the stage ID as staged", func() { + values, err := getValuesYAML(&models.AppChartFull{}, ChartParameters{ + AppRef: models.NewAppRef("demo", "workspace"), + ImageURL: "registry.example/stage-123/demo:not-stage-123-suffix", + StageID: "stage-123", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(values).ToNot(ContainSubstring("staged: true")) + }) + + It("requires a non-empty stage ID", func() { + Expect(isStagedImage("registry.example/demo:", "")).To(BeFalse()) + }) +}) + var _ = Describe("ValidateField()", func() { It("is ok for unconstrained integer", func() { diff --git a/internal/manifest/manifest_test.go b/internal/manifest/manifest_test.go index 976260fd86..c2b94f380e 100644 --- a/internal/manifest/manifest_test.go +++ b/internal/manifest/manifest_test.go @@ -104,6 +104,12 @@ configuration: environment: CREDO: up DOGMA: "no" + processes: + web: + kind: deployment + command: ["python", "app.py", "web"] + replicas: 0 + routes: true `), 0600) Expect(err).ToNot(HaveOccurred()) }) @@ -128,6 +134,14 @@ configuration: "DOGMA": "no", "CREDO": "up", }, + Processes: models.ApplicationProcesses{ + "web": { + Kind: models.ApplicationProcessDeployment, + Command: []string{"python", "app.py", "web"}, + Replicas: func() *int32 { value := int32(0); return &value }(), + Routes: true, + }, + }, }, Self: path.Join(workdir, "goodyaml.yml"), Origin: models.ApplicationOrigin{ diff --git a/pkg/api/core/v1/models/models.go b/pkg/api/core/v1/models/models.go index 2047115cb1..d544004ecc 100644 --- a/pkg/api/core/v1/models/models.go +++ b/pkg/api/core/v1/models/models.go @@ -151,8 +151,32 @@ type ApplicationConfiguration struct { AppChart string `json:"appchart,omitempty" yaml:"appchart,omitempty"` Settings ChartValueSettings `json:"settings,omitempty" yaml:"settings,omitempty"` Ignore []string `json:"ignore,omitempty" yaml:"ignore,omitempty"` + Processes ApplicationProcesses `json:"processes,omitempty" yaml:"processes,omitempty"` } +// ApplicationProcessKind describes the Kubernetes workload rendered for an +// application process. All process kinds use the application's staged image. +type ApplicationProcessKind string + +const ( + ApplicationProcessDeployment ApplicationProcessKind = "deployment" + ApplicationProcessCron ApplicationProcessKind = "cron" + ApplicationProcessRelease ApplicationProcessKind = "release" +) + +// ApplicationProcess describes one named, release-scoped process of an app. +// Replicas only applies to deployments, Schedule only applies to cron +// processes, and Routes marks the deployment targeted by the app's routes. +type ApplicationProcess struct { + Kind ApplicationProcessKind `json:"kind,omitempty" yaml:"kind,omitempty"` + Command []string `json:"command" yaml:"command"` + Replicas *int32 `json:"replicas,omitempty" yaml:"replicas,omitempty"` + Schedule string `json:"schedule,omitempty" yaml:"schedule,omitempty"` + Routes bool `json:"routes,omitempty" yaml:"routes,omitempty"` +} + +type ApplicationProcesses map[string]ApplicationProcess + // ApplicationOrigin is the part of the manifest describing the origin of the application // (sources). At most one of the fields may be specified / not empty. type ApplicationOrigin struct { @@ -219,19 +243,20 @@ type ApplicationCreateRequest struct { // Note: Instances is a pointer to give us a nil value separate from // actual integers, as means of communicating `default`/`no change`. type ApplicationUpdateRequest struct { - Restart *bool `json:"restart,omitempty"` - Instances *int32 `json:"instances" yaml:"instances,omitempty"` - Configurations []string `json:"configurations" yaml:"configurations,omitempty"` - Environment EnvVariableMap `json:"environment" yaml:"environment,omitempty"` - ReplaceEnv *bool `json:"replace_env,omitempty" yaml:"replace_env,omitempty"` - Routes []string `json:"routes" yaml:"routes,omitempty"` - AppChart string `json:"appchart,omitempty" yaml:"appchart,omitempty"` - Settings ChartValueSettings `json:"settings,omitempty" yaml:"settings,omitempty"` + Restart *bool `json:"restart,omitempty"` + Instances *int32 `json:"instances" yaml:"instances,omitempty"` + Configurations []string `json:"configurations" yaml:"configurations,omitempty"` + Environment EnvVariableMap `json:"environment" yaml:"environment,omitempty"` + ReplaceEnv *bool `json:"replace_env,omitempty" yaml:"replace_env,omitempty"` + Routes []string `json:"routes" yaml:"routes,omitempty"` + AppChart string `json:"appchart,omitempty" yaml:"appchart,omitempty"` + Settings ChartValueSettings `json:"settings,omitempty" yaml:"settings,omitempty"` + Processes *ApplicationProcesses `json:"processes,omitempty" yaml:"processes,omitempty"` } func NewApplicationUpdateRequest(manifest ApplicationManifest) ApplicationUpdateRequest { manifestConfig := manifest.Configuration - return ApplicationUpdateRequest{ + request := ApplicationUpdateRequest{ Instances: manifestConfig.Instances, Configurations: manifestConfig.Configurations, Environment: manifestConfig.Environment, @@ -240,6 +265,10 @@ func NewApplicationUpdateRequest(manifest ApplicationManifest) ApplicationUpdate AppChart: manifestConfig.AppChart, Settings: manifestConfig.Settings, } + if manifestConfig.Processes != nil { + request.Processes = &manifestConfig.Processes + } + return request } type ImportGitResponse struct { diff --git a/pkg/api/core/v1/models/models_test.go b/pkg/api/core/v1/models/models_test.go index 7397b2233e..4c62675741 100644 --- a/pkg/api/core/v1/models/models_test.go +++ b/pkg/api/core/v1/models/models_test.go @@ -124,5 +124,25 @@ var _ = Describe("ApplicationOrigin String()", func() { update := models.NewApplicationUpdateRequest(m) Expect(update.ReplaceEnv).To(BeNil()) }) + + It("copies process definitions while preserving an explicit zero replica count", func() { + zero := int32(0) + m := models.ApplicationManifest{ + Configuration: models.ApplicationConfiguration{ + Processes: models.ApplicationProcesses{ + "worker": {Command: []string{"worker"}, Replicas: &zero}, + }, + }, + } + + update := models.NewApplicationUpdateRequest(m) + Expect(update.Processes).ToNot(BeNil()) + Expect(*(*update.Processes)["worker"].Replicas).To(Equal(int32(0))) + }) + + It("keeps process definitions nil when unset", func() { + update := models.NewApplicationUpdateRequest(models.ApplicationManifest{}) + Expect(update.Processes).To(BeNil()) + }) }) }) diff --git a/poc/multiprocess/FOLLOWUP-AUDIT.md b/poc/multiprocess/FOLLOWUP-AUDIT.md new file mode 100644 index 0000000000..a87e3d18d4 --- /dev/null +++ b/poc/multiprocess/FOLLOWUP-AUDIT.md @@ -0,0 +1,227 @@ +# Follow-up audit: Epinio multi-process POC + +Audit date: 2026-08-06 UTC + +Source commit: `f84b932e866e442490cf134ef72e5d90adf8a01f` + +## Verdict + +The corrected POC demonstrates that one Epinio Helm release can render and coordinate web, worker, cron, and release processes from one prebuilt or Epinio-staged image. Failed release hooks prevent ordinary workloads from changing, and an unhealthy worker causes Helm atomic rollback to restore the prior Deployment and CronJob specifications. + +The production assessment remains **feasible but invasive**. The Helm boundary works; Epinio's desired-state persistence, application-wide pod discovery, global scaling, status, logs, exec, restart, and missing one-off command API do not yet provide first-class process semantics. + +This audit does not certify the POC as production-ready. In particular, migration side effects and CronJob executions are not rollbackable, Epinio's App CR diverges from the deployed Helm revision after failure, and the authoritative CRD module/schema was not versioned or migrated. + +## Independent clean reproduction + +On 2026-08-07 UTC, fork branch `poc/multiprocess-apps` at +`0c5438d974f0a37ee5f5a7763a72c14df69a3f15` was cloned into a new +`epinio-trial-02.exe.xyz` VM copied from baseline revision `2026-08-06.4`. +Nothing was copied from the original Epinio POC VM. + +The README installation produced Epinio chart `1.14.1`, installed the committed +server and CRD changes, and installed POC AppChart `0.2.1` with a verified chart +archive checksum. The final-tree static script passed all focused Go tests, +Helm lint/render assertions, zero-replica rendering, Kubernetes client-side dry +runs, and `git diff --check`. + +The fresh `multiprocess-repro` application passed the seven-revision live +matrix: initial v1, v2 upgrade with worker scaling from two to three, successful +migrations and controller-created cron executions, failed migration with exit +42 and atomic rollback, unhealthy-worker timeout and atomic rollback with the +candidate fully scaled down, and direct source staging/deployment. Revision 7 +finished with staged web `2/2`, worker `3/3`, a staged CronJob and successful +release Job. Direct HTTPS ingress returned +`web version=staged-v5 process=web`, plain HTTP returned 301, and neither a +port-forward nor local TLS bridge was active. Cilium, its operator, Envoy, and +Hubble Relay remained healthy. + +The only reproduction-instruction correction was to export +`KUBECONFIG=/etc/rancher/k3s/k3s.yaml` for Helm and the scripts on a baseline +copy. Raw outputs remain only on the validation VM and were not added to the +fork or this trial directory. + +### Post-reproduction port compatibility + +Chart `0.2.3` restores the standard Epinio `appListeningPort` setting that the +initial multi-process chart had accidentally replaced with a hardcoded 8080. +The chart keeps 8080 as its backward-compatible default and accepts port 80 for +Pretix. Uncached focused Go tests and the static Helm suite passed both renders, +Kubernetes client-side dry-runs, chart packaging, and AppChart server-side +dry-run validation. Live port-80 application behavior is intentionally left to +the Pretix trial. + +## Audit method + +The audit treated the earlier README as unverified and inspected: + +- the complete tracked and untracked git diff; +- App models, create/update/deploy handlers, staging and image selection, Helm value construction, workload discovery, status, logs, exec, scaling, restart, routes, services, and bindings; +- the live App CRD schema and AppChart object; +- the chart archive actually served by the cluster, not only the working-tree templates; +- rendered prebuilt, staged, and zero-replica manifests; +- Helm history, values, hooks, and failed revisions; +- live Deployments, ReplicaSets, Services, Ingresses, Certificates, CronJobs, Jobs, Pods, events, commands, images, replica counts, and logs; +- a clean seven-revision live test matrix and a separate rollback-convergence capture. + +## Requirement and deliverable audit + +| Requirement | Result | Direct evidence or qualification | +|---|---|---| +| Trace current Epinio architecture | Met | README architecture summary plus source locations listed below. | +| Add named web, worker, cron, and release processes | Met for POC | `models.go`, CRD patch, API persistence, Helm values, and chart templates. Exported live manifest contains all four definitions. | +| One coordinated application release | Met | All ordinary resources are owned by one hashed Helm release; history is in `evidence/live-audit3/*/helm-history.json`. Hook Jobs are Helm hooks and deliberately retained outside the ordinary manifest lifecycle. | +| Same staged image, different commands | Met | Revision 7 uses one runtime image in both Deployments, CronJob, and release Job; commands use the CNB launcher plus process-specific args. | +| Process-specific replicas | Met | v1 web/worker `2/2`; v2 and staged web `2/2`, worker `3/3`. Zero replicas is render/unit tested but was not deployed live. | +| Web Deployment + Service/route | Met after infrastructure follow-up | The matrix proved the Deployment, Service, TLS Certificate, Ingress, and Service response. After the baseline listeners moved from 8000/8443 to 80/443, direct standard-port HTTPS returned the staged web response after Cilium component restarts with no local bridge. | +| Worker Deployment | Met | Healthy and deliberately crashing worker revisions were deployed. Kubernetes events record all three v4 worker pods in `BackOff`. | +| Scheduler CronJob | Met | The Kubernetes CronJob controller created Jobs for revisions 1, 2, 5, and 7. Captured v1, v2, and staged logs report the expected versions. | +| Successful release/migration | Met | Revision-named hook Jobs for v1, v2, v4, and staged images completed and have retained logs. | +| Failed release blocks deployment | Met | Revision 3 failed with `BackoffLimitExceeded`; retained pod exit code is 42. Revision 4 rolled back to revision 2 without changing ordinary workloads. | +| Unhealthy process fails whole release | Met | Revision 5 timed out after 198.77s; Helm revision 6 rolled back. Captured events show all three crashing worker pods in BackOff. | +| Rollback restores every prior ordinary process/image/replica state | Met with qualification | The final harness asserted web `2/2` v2, worker `3/3` v2, CronJob v2, Service response v2, zero candidate Deployment pods, and zero nonzero candidate ReplicaSets before snapshotting. Independent convergence evidence records the same result. Hook Jobs, CronJob-created Jobs, and external migration effects are not rollback targets. | +| Services remain independent | Source-confirmed only | No service code was changed and service instances still use separate Helm releases. The chart mounts Epinio binding/configuration Secrets into each process pod. No live database/service instance was created in this follow-up. | +| One-off CLI commands | Not implemented | This is a missing first-class capability, not a passing deliverable. It requires a revision-aware ephemeral Job API/CLI. | +| Reproduction instructions | Met after correction | README uses repeatable install, static verification, and live audit scripts and explains the historical baseline port mismatch and its correction. | +| Existing assumptions needing work | Met | Status, readiness, logs, exec, scale, restart, routes, stage tracking, bindings, hooks, and desired/deployed state are documented. | +| Production effort and assessment | Met | README retains an 8-12 engineer-week estimate and “feasible but invasive” assessment. | + +## Findings and fixes + +### Correctness and evidence issues fixed + +1. **The cluster served a stale chart.** The live ConfigMap still contained `replicas: {{ default 1 ... }}` while the working tree preserved zero correctly. `scripts/install-chart.sh` now packages the current chart, updates the ConfigMap, restarts the subPath-mounted chart server, updates the AppChart cache-busting URL, and compares package and ConfigMap SHA-256 values. + +2. **The earlier staged success was overstated.** The original source push created a staged image but failed revisions 7 and 9; revision 11 succeeded only after `app restart`, leaving the App origin at v2. The clean audit performed a direct source push with the final chart. It succeeded as revision 7 and persisted the source path origin. + +3. **Failed hook evidence was overwritten.** A constant hook Job name was deleted before rollback or the next upgrade. Hook Jobs now include `.Release.Revision`, retaining the failed revision-3 pod, log, and exit code. + +4. **The routed chart omitted Epinio's normal TLS behavior.** The chart now renders a cert-manager Certificate when no matching route secret is provided and always configures Ingress TLS. The live Certificate is Ready. + +5. **Staged-image detection used substring matching.** A prebuilt image could be misclassified if its path or tag happened to contain the stage ID. Detection now requires the complete image tag to equal the stage ID and has positive, negative, and empty-stage unit coverage. It remains a POC heuristic. + +6. **Routes could have no process target.** Create/update validation now rejects external routes for a non-legacy process map unless one deployment has `routes: true`. Legacy process-less apps remain valid. + +7. **Manifest/API round-trip coverage was incomplete.** Tests now cover manifest YAML decoding, update request propagation, CR decoding, validation, explicit zero replicas, multiple routed/release processes, and Helm values. + +8. **There was no reproducible evidence harness.** Added `verify-static.sh`, `install-chart.sh`, and `run-live-audit.sh`. The live script refuses to overwrite an existing App and asserts state before writing each snapshot. + +9. **Rollback convergence was initially recorded too early.** The first post-rollback snapshot contained terminating candidate web pods. The audit script now waits specifically for candidate Deployment pods and ReplicaSets to reach zero while excluding retained hook/Cron history. `evidence/rollback-converged/summary.txt` is the separate converged capture. + +10. **The previous acceptance explanation was not directly captured.** A fresh API acceptance run failed in `SynchronizedBeforeSuite` with 404 on `/.well-known/openid-configuration` because Dex is disabled. It ran zero specs; the exact output is retained. + +11. **Service and Ingress backend naming were not generated from one helper.** They now share the same bounded Service name. Chart `0.2.1` containing that fix was installed by checksum and used for the final seven-revision run. + +12. **The live harness did not preserve all direct failure facts.** Every snapshot now captures application-related events, and a failed release hook must expose exit code 42 in terminated-container state before the test can pass. + +13. **Composed resource names are not safe at maximum input lengths.** An exploratory `0.2.2` normal-name upgrade passed, but a 63-character application probe failed atomically because its CronJob name exceeded Kubernetes' 52-character limit. The incomplete hashed-name experiment was not retained in the final tree. Collision-safe, kind-specific naming remains a documented gap; evidence is in `evidence/chart-0.2.2-live/` and `evidence/long-name-live/push.log`. + +### Newly established semantics + +- Helm's automatic atomic rollback did **not** run the prior revision's release hook. Jobs `release-r4` and `release-r6` were not created. +- A successful candidate release Job and a candidate CronJob execution can remain after ordinary resources roll back. +- Epinio reports the staged application as `5/1`, lists historical hooks as instances, and showed an empty `Running StageId` because workload metadata is taken from the first application-wide pod. +- `epinio app logs` aggregates current processes and retained release history, including failed and superseded migrations. +- After failed revisions, `spec.imageurl` and `spec.processes` represent failed desired intent while `origin` and live Helm resources remain at the last successful v2 release. + +## Commands and results + +### Static and Go checks + +```bash +poc/multiprocess/scripts/verify-static.sh poc/multiprocess/evidence/static +``` + +Result when run: pass. It runs focused Go tests, Helm lint, exact prebuilt/staged/zero-replica render assertions, Kubernetes client-side dry runs, and `git diff --check`. The retained `evidence/static/verification.log` was later overwritten by the experimental `0.2.2` naming run and is therefore not claimed as an exact final-tree rerun. The complete live matrix used final-tree chart `0.2.1`; after the naming experiment was reverted, only `git diff --check` was rerun at the user's direction. + +```bash +go list ./... | rg -v '/acceptance($|/)' | xargs go test -count=1 +``` + +Result: pass for every non-acceptance package. See `evidence/non-acceptance-go-test.log`. + +```bash +go test ./acceptance/api/v1 -run TestAPI -count=1 \ + -ginkgo.label-filter=application +``` + +Result: fail before specs. OIDC discovery returned 404; 0 of 253 specs ran. See `evidence/acceptance/api-v1-blocker.log`. + +### Live matrix + +```bash +EPINIO_TEST_APP=multiprocess-audit3 \ +EPINIO_EVIDENCE_DIR="$PWD/poc/multiprocess/evidence/live-audit3" \ + poc/multiprocess/scripts/run-live-audit.sh +``` + +Result: pass against chart `0.2.1`. The release was `multiprocess-2c659e0ad9a452a685ebfc317e797f78991ebeaf`. + +| Revision | Helm status | Evidence | +|---:|---|---| +| 1 | superseded after later upgrade | v1 initial install; migration, scheduled cron, Service response, web `2/2`, worker `2/2`. | +| 2 | superseded | v2 upgrade; migration, scheduled cron, Service response, web `2/2`, worker `3/3`. | +| 3 | failed | Migration `fail=True`, exit 42, `BackoffLimitExceeded`. | +| 4 | superseded after later upgrade | Atomic rollback to revision 2; no rollback hook Job. | +| 5 | failed | v4 migration succeeded; worker pods entered BackOff; timeout. | +| 6 | superseded after staged upgrade | Atomic rollback to revision 4/v2 content; no rollback hook Job. | +| 7 | deployed | Direct staged push; source origin persisted; all process pods use stage `c52c51a39859a471`. | + +The separate `multiprocess-audit` rollback capture ended at revision 6 and records fully converged v2 state in `evidence/rollback-converged/summary.txt`. + +## Evidence map + +- `evidence/live-audit3/summary.txt`: chart/revision history, exact timings, release exit code, BackOff events, stage ID, and revision-7 process state. +- `evidence/live-audit3/01-initial-v1/`: push log, revision history/values/manifest/hooks, full resources/events, migration log, controller-created cron log, and web response. +- `evidence/live-audit3/02-upgrade-v2/`: corresponding v2 evidence and replica change. +- `evidence/live-audit3/03-failed-release/`: failed values/hook, exit-code pod JSON, failed migration log, rollback history, and restored resources. +- `evidence/live-audit3/04-unhealthy-worker/`: failed values/manifest, v4 migration log, Kubernetes BackOff events, rollback history, candidate/rollback resources, and v2 web response. +- `evidence/live-audit3/05-staged-direct/`: direct build/push log, source-origin App CR, staged resources, migration/cron logs, and Service response. +- `evidence/live-audit3/exported-manifest.yml`: API/CLI manifest round trip containing all process definitions. +- `evidence/live-audit3/app-show.txt`: direct evidence of status and instance-model breakage. +- `evidence/live-audit3/app-logs.txt`: direct evidence of application-wide log aggregation. +- `evidence/rollback-converged/`: independent unhealthy rollback ending at the restored v2 release. +- `evidence/invalid-route-validation.log` and `evidence/invalid-route-update-validation.log`: live create/update rejection and preservation of the existing process map. +- `evidence/chart-0.2.2-live/`: successful normal-name live upgrade after the complete matrix; this experimental chart is not the final working-tree chart. +- `evidence/long-name-live/push.log`: atomic failure proving the unresolved CronJob 52-character name limit. + +## Remaining gaps + +- No first-class one-off command API or CLI. +- No process-aware status/readiness model, log selector, exec selector, scaling, or restart. +- No transactional desired/deployed revision state; a failed push leaves the App CR ahead of Helm. +- No rollback of migration side effects or CronJob-created work. +- No production CRD/API version migration, generated client/OpenAPI update, compatibility matrix, or upgrade tests. +- No live bound database/service test in this follow-up. +- Cilium ingress was validated after the audit rather than inside the seven-revision matrix; the matrix itself captured only Service traffic. +- No UI work, generalized multi-image model, per-process health schema, route-to-process mapping beyond one target, or hook retention policy. +- No collision-safe, per-kind resource naming for maximum-length application/process names; CronJobs require a 52-character bound. +- Acceptance specs remain blocked by this cluster's intentional Dex-disabled installation. + +## Current cluster state + +- `multiprocess-audit3`: Helm revision 8 deployed with experimental chart `0.2.2`, staged web `2/2`, worker `3/3`, CronJob active, Certificate Ready. Chart `0.2.1` was the fully matrix-tested retained tree at audit time; current chart `0.2.3` adds only restored listening-port configurability and related static coverage. +- `multiprocess-audit2`: earlier chart `0.2.0` audit release retained for comparison. +- `multiprocess-audit`: Helm revision 6 deployed at restored v2 state; its App CR still contains failed v4 desired intent for inspection. +- `multiprocess-poc`: original development release remains deployed for historical comparison. +- The 63-character long-name probe left an App CR with failed desired v1 intent; its atomic initial install was uninstalled and produced no successful application release. +- The Epinio server is the rebuilt `v0.0.0-dev` POC binary. The original audit cluster's served chart and AppChart cache key remain experimental `0.2.2`/`?v=audit-3`; running the current `scripts/install-chart.sh` installs chart `0.2.3` with cache key `?v=0.2.3`. + +All repository changes are intentionally uncommitted. + +## Git status + +```text + M internal/api/v1/application/create.go + M internal/api/v1/application/update.go + M internal/api/v1/deploy/deploy.go + M internal/application/application.go + M internal/helm/helm.go + M internal/helm/helm_test.go + M internal/manifest/manifest_test.go + M pkg/api/core/v1/models/models.go + M pkg/api/core/v1/models/models_test.go +?? internal/application/processes.go +?? internal/application/processes_test.go +?? poc/ +``` diff --git a/poc/multiprocess/README.md b/poc/multiprocess/README.md new file mode 100644 index 0000000000..4149cabed1 --- /dev/null +++ b/poc/multiprocess/README.md @@ -0,0 +1,281 @@ +# Epinio multi-process POC + +This POC adds a deliberately small, typed process model to Epinio and passes it to a custom application chart. The live tests were run on 2026-08-06 against: + +- Epinio source commit `f84b932e866e442490cf134ef72e5d90adf8a01f` (2026-07-30) +- Epinio Helm chart `1.14.1` +- k3s/Kubernetes `v1.36.3+k3s1` +- Helm `v3.21.3` +- cert-manager `v1.21.1` + +The result is **feasible but invasive**. Rendering and coordinating multiple workloads in one Helm release is a clean fit. Epinio's management APIs and workload model, however, assume one scalable process strongly enough that a production implementation requires meaningful cross-cutting work. + +## Current Epinio deployment path + +1. `epinio push` creates or updates the passive `application.epinio.io/v1 App` CR. Environment, scale, configuration bindings, and service bindings are stored in associated Secrets. +2. Source pushes upload an archive to Epinio's S3 store. A staging Job runs Cloud Native Buildpacks and pushes one image to the registry. The App CR records `stageid`, builder, blob ID, and image URL. +3. The deploy endpoint writes `spec.imageurl` before attempting the workload deployment, then `internal/api/v1/deploy` loads the App CR, scale Secret, environment, bindings, routes, domains, and selected AppChart. +4. `internal/helm.getValuesYAML` constructs the chart contract under `epinio.*`, plus operator `chartConfig` and validated `userConfig`. +5. Epinio installs or upgrades one hashed Helm release with `Wait: true`, `Atomic: true`, `ReuseValues: true`, and a three-minute deployment timeout. +6. Only after Helm succeeds does Epinio persist the new source origin and clean older staging artifacts. +7. Status, logs, and exec discover pods by application-wide labels. Scale remains a single integer in one Secret. Services use separate service Helm releases; bindings feed configuration Secrets into the app release. + +## POC model and resources + +The manifest/API model accepts: + +```yaml +configuration: + processes: + web: + kind: deployment + command: ["python", "app.py", "web"] + replicas: 2 + routes: true + worker: + kind: deployment + command: ["python", "app.py", "worker"] + replicas: 3 + scheduler: + kind: cron + command: ["python", "app.py", "cron"] + schedule: "* * * * *" + release: + kind: release + command: ["python", "app.py", "migrate"] +``` + +The POC persists this under `App.spec.processes`, returns it in application manifests/API responses, and passes it as structured `epinio.processes` Helm values. + +The chart renders: + +- `deployment` -> one Deployment per named process +- the one deployment with `routes: true` -> Service, TLS Certificate, and application Ingress +- `cron` -> CronJob +- `release` -> a revision-named `pre-install,pre-upgrade` Helm hook Job with `backoffLimit: 0` + +Every process pod receives the same `epinio.imageURL`, environment, stage ID, configuration mounts, and application identity labels. Process selectors add `epinio.io/process-name` so Deployments do not overlap. Resources also carry `epinio.io/release-revision` for auditability. + +For an Epinio-staged CNB image whose complete image tag equals the stage ID, Epinio adds `epinio.staged: true`. The chart explicitly invokes `/cnb/lifecycle/launcher -- ` so buildpack `exec.d` environment setup is retained. For a prebuilt container image, the chart uses the process command directly. This is still an execution-mode heuristic and should become explicit API state in production. + +## Code layout + +- Core model/API/CR plumbing: `pkg/api/core/v1/models/models.go`, `internal/application/application.go`, `internal/application/processes.go`, and the create/update/deploy handlers. +- Helm contract: `internal/helm/helm.go`. +- CRD schema POC patch: `crd-processes-json-patch.json`. +- Application chart and chart server: `chart/`, `chart-server.yaml`, and `appchart.yaml`. +- Test app and manifests: `test-app/` and the `epinio-*.yml` files. +- Repeatable checks: `scripts/verify-static.sh`, `scripts/install-chart.sh`, and `scripts/run-live-audit.sh`. +- Captured outputs: `evidence/static/`, `evidence/live-audit3/`, `evidence/rollback-converged/`, and `evidence/acceptance/`. + +The upstream App CRD types live in the separate `github.com/epinio/application` module, which this checkout pins to a 2023 pseudo-version. The POC avoids changing that dependency and supplies an explicit CRD JSON patch. A production change must update the authoritative CRD/type source and add conversion and upgrade coverage. + +## Reproduction + +The commands below assume a working Kubernetes context, Docker, Helm, and the Epinio source checkout. Replace the domain/IP for another cluster. + +On a VM copied from `paas-k3s-base.exe.xyz`, point Helm and the scripts at the +K3s kubeconfig first: + +```bash +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +``` + +The chart retains Epinio's standard application-port setting and defaults to +8080. For an image such as Pretix that listens on port 80, include this in the +application manifest: + +```yaml +configuration: + settings: + appListeningPort: 80 +``` + +Install cert-manager first so its CRDs exist before the Epinio and POC Certificate resources are applied: + +```bash +helm repo add jetstack https://charts.jetstack.io +helm repo add epinio https://epinio.github.io/helm-charts +helm repo update + +helm upgrade --install cert-manager jetstack/cert-manager \ + --namespace cert-manager --create-namespace \ + --set crds.enabled=true --wait + +helm upgrade --install epinio epinio/epinio \ + --namespace epinio --create-namespace \ + --set global.domain=10.42.0.42.sslip.io \ + --set certManager.install=false \ + --set ingress.ingressClassName=cilium \ + --set server.ingressClassName=cilium \ + --set global.dex.enabled=false \ + --wait --timeout 12m +``` + +Build and install the modified server binary and CRD schema: + +```bash +kubectl patch crd apps.application.epinio.io --type=json \ + --patch-file=poc/multiprocess/crd-processes-json-patch.json + +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -o dist/epinio-linux-amd64 . + +EPINIO_BINARY_TAG=multiprocess-poc \ +EPINIO_BINARY_PATH=dist/epinio-linux-amd64 \ +EPINIO_NAMESPACE=epinio \ + ./scripts/patch-epinio-deployment.sh +``` + +Package and expose the custom AppChart to the Epinio API pod. The script restarts the chart server after updating its ConfigMap and verifies that the cluster archive checksum matches the package; both steps prevent testing a stale subPath-mounted chart: + +```bash +poc/multiprocess/scripts/install-chart.sh +``` + +For the prebuilt-image cases, build and load all tags into the single-node k3s image store: + +```bash +for version in v1 v2 v3 v4; do + docker build \ + -t docker.io/epinio-poc/multiprocess:$version \ + --build-arg IMAGE_VERSION=$version \ + poc/multiprocess/test-app +done + +docker save \ + docker.io/epinio-poc/multiprocess:v1 \ + docker.io/epinio-poc/multiprocess:v2 \ + docker.io/epinio-poc/multiprocess:v3 \ + docker.io/epinio-poc/multiprocess:v4 \ + | sudo k3s ctr images import - +``` + +The original audit used baseline revision `2026-08-06.3`, whose Cilium +host-network ingress listeners used ports 8000/8443 while Epinio generated +portless URLs for the standard ports 80/443. The audit's Service port-forward +and local TLS bridge worked around that port mismatch; the shared ingress +Service being `ClusterIP` was not the underlying problem. Baseline revision +`2026-08-06.4` serves ingress directly on 80/443 and needs no workaround. On a +different cluster, ensure the Epinio wildcard domain resolves to ingress that +is reachable on those standard ports. + +Log in and select a workspace: + +```bash +./dist/epinio-linux-amd64 login -u admin -p password --trust-ca \ + https://epinio.10.42.0.42.sslip.io +./dist/epinio-linux-amd64 namespace create poc +./dist/epinio-linux-amd64 target poc +``` + +Run the static checks and the complete live matrix: + +```bash +poc/multiprocess/scripts/verify-static.sh poc/multiprocess/evidence/static + +EPINIO_TEST_APP=multiprocess-audit3 \ +EPINIO_EVIDENCE_DIR="$PWD/poc/multiprocess/evidence/live-audit3" \ + poc/multiprocess/scripts/run-live-audit.sh +``` + +The live script deliberately refuses to overwrite an existing test App. It verifies every assertion before recording a case snapshot. The four prebuilt image tags must already be present on the node as shown above. + +## Live test results + +The final complete matrix used chart `0.2.1`, a fresh application named `multiprocess-audit3`, and Helm release `multiprocess-2c659e0ad9a452a685ebfc317e797f78991ebeaf`. The full captured record is under `evidence/live-audit3/`; `summary.txt` records the exact history, timings, failed container exit code, BackOff events, stage ID, and revision-7 resources. + +| Case | Observed result | +|---|---| +| Initial v1 | Revision 1 installed in 16.43s. Web `2/2`, worker `2/2`, migration Job completed with v1, a controller-created CronJob Job printed v1, and a Service port-forward returned v1. | +| Upgrade v2 | Revision 2 completed in 18.20s. Web stayed `2/2`, worker became `3/3`, migration and a new controller-created cron Job printed v2, and Service traffic returned v2. | +| Failed migration | Revision 3 returned 255 in 12.14s with `BackoffLimitExceeded`. The retained migration pod logged `fail=True`; the harness read exit code 42 from its terminated container. Atomic revision 4 rolled back to revision 2; web `2/2`, worker `3/3`, CronJob image/command/schedule, Service, Ingress, and Certificate remained/restored to v2 state. | +| Unhealthy worker | Revision 5's migration succeeded with v4. Captured Kubernetes events record all three v4 worker pods in `BackOff`; a v4 CronJob-created Job also completed while the candidate was pending. The push returned 255 after 198.77s and revision 6 rolled back to revision 4. Before snapshotting, the harness asserted zero remaining v4 Deployment pods and zero nonzero v4 ReplicaSets, plus web `2/2`, worker `3/3`, CronJob v2, and Service traffic v2. | +| Direct staged push | A single source `push` built stage `c52c51a39859a471` and deployed revision 7 in 54.09s. The App origin changed to the source path. Web `2/2`, worker `3/3`, CronJob, migration Job, and a controller-created cron Job used the same internal image and `/cnb/lifecycle/launcher`; all outputs reported `staged-v5`. | + +The final staged command is `/cnb/lifecycle/launcher` with `--` plus the declared process command in `args`. Earlier development revisions 7 and 9 of the original `multiprocess-poc` release failed with two incomplete CNB command strategies; their Helm hook manifests remain inspectable, but the deleted pod logs do not. They are therefore development history, not part of the passing evidence matrix. + +After that matrix, an experimental chart `0.2.2` added hashed overlength resource names. A normal-name live upgrade to revision 8 passed in 13.85s, including the release hook, web/worker replicas, CronJob, Certificate, and Service response (`evidence/chart-0.2.2-live/`). A separate 63-character application-name probe then failed atomically because Kubernetes limits CronJob names to 52 characters (`evidence/long-name-live/push.log`). The incomplete long-name change is not in the retained tree; chart `0.2.1` is the version used for the complete behavioral matrix and long process/application-name collision handling remains an explicit gap. Current chart `0.2.3` builds on that retained tree and restores the standard Epinio `appListeningPort` setting for images such as Pretix that do not listen on 8080. The cluster used for the original audit still contains experimental revision 8 for inspection. + +## Verification + +All non-acceptance Go packages pass: + +```bash +go list ./... | rg -v '/acceptance($|/)' | xargs go test -count=1 +``` + +`scripts/verify-static.sh` passed focused Go tests, Helm lint, exact prebuilt/staged resource assertions, Kubernetes client-side dry runs, and `git diff --check`; rendering explicitly covered `replicas: 0`. The retained `evidence/static/verification.log` was subsequently overwritten by the experimental `0.2.2` naming run, so it is not claimed as an exact final-tree rerun. The application Go code did not change afterward, and the complete live matrix used chart `0.2.1`. For chart `0.2.3`, the focused Go tests were rerun uncached and the static suite additionally rendered, asserted, and client-side dry-ran both the default port 8080 and overridden port 80 cases. The chart packaged successfully and the AppChart passed Kubernetes server-side dry-run validation. The broader historical Go run is in `evidence/non-acceptance-go-test.log`. + +At audit completion, `multiprocess-audit3` revision 8 remained deployed with the experimental chart `0.2.2`, staged web `2/2`, worker `3/3`, a staged CronJob, successful staged migration Job, ready Certificate, and TLS Ingress. The complete behavioral matrix immediately preceding it used the final-tree chart `0.2.1`. During that matrix, application traffic was proven through a Kubernetes Service port-forward because the then-current baseline listened on 8000/8443. A subsequent infrastructure validation moved Cilium to 80/443, restarted the operator, agent, and Envoy, and verified both direct Epinio CLI access and the staged application response over standard HTTPS without a port-forward or TLS bridge. The earlier limitation was therefore baseline port configuration, not the chart's route or Cilium's `ClusterIP` ingress Service. + +The API acceptance suite was rerun with its application label filter. Its `SynchronizedBeforeSuite` received 404 for `/.well-known/openid-configuration` because this cluster deliberately has Dex/OIDC disabled. It ran 0 of 253 specs. The exact output is `evidence/acceptance/api-v1-blocker.log`; no acceptance result is claimed. + +### Atomicity qualifications + +- A failed pre-upgrade migration hook runs before normal resources change, so the old release stays intact. +- If the release hook succeeds and a Deployment later fails, Helm rolls back Kubernetes release resources, but it cannot undo the migration's external effects. +- Automatic atomic rollback did not execute the prior revision's release hook: Jobs `release-r4` and `release-r6` were not created. This avoids rerunning the old migration, but rollback migrations are likewise not available. +- Hook Jobs and CronJob-created Jobs are history, not ordinary rollback targets. The successful v4 migration Job and a completed v4 scheduled Job remained after ordinary resources rolled back to v2. +- Helm returned rollback success while candidate pods were still terminating. The separate `evidence/rollback-converged/summary.txt` capture waited until candidate Deployment pods and ReplicaSet counts were zero. Atomicity does not mean simultaneous pod replacement. +- Most importantly, Epinio writes process configuration and `spec.imageurl` before Helm. After both failed upgrades, the App CR pointed at the failed v3/v4 intent while the live Helm release was v2. `origin` remained v2 because Epinio only updates it after success. A production design needs desired-vs-deployed revisions or transactional reconciliation. + +## Existing assumptions that break + +### Status and readiness + +Epinio aggregates all pods labeled as the application and compares ready pods with one global desired instance count. The audited staged app showed `5/1`, while historical hook and current cron pods appeared as non-ready entries in the instance table. Because workload metadata comes from the first unsorted application pod, `Running StageId` was blank even though all current Deployments used the staged image. Status needs a process-indexed workload model and per-kind readiness and revision rules. Helm waits for Deployments and hook Jobs, but not for a CronJob to execute. + +### Logs + +Existing logs aggregate web, worker, cron, and every retained release hook, including failed and superseded revisions. `evidence/live-audit3/app-logs.txt` directly contains all four process categories. There is no process selector, so production UX needs `--process`, sensible defaults, bounded history, and explicit job handling. + +### Exec and one-off commands + +Exec selects from one application-wide pod list and derives a container name from application workload data. Without `--instance`, it uses the first unsorted pod and can choose the wrong process or a completed Job. `--instance` can target an exact pod, but there is no stable process selector. The POC uses the same container name in every pod only to avoid an additional failure mode. + +One-off commands are not implemented as a first-class API in this POC. They need an endpoint/CLI that creates an ephemeral Job from a specific deployed application revision, uses the same bindings and CNB launcher semantics, streams exit status/logs, and applies retention policy. Existing interactive exec is not an adequate substitute. + +### Scaling + +Scale is one integer in one Secret and the CLI/API accept only application-wide `instances`. The POC chart intentionally uses per-process replicas, so the old scale value cannot represent or update them. Production needs `scale APP --process worker N`, per-process persistence, events, validation, and a backward-compatible default-web mapping. + +### Restart + +Restart redeploys the whole Helm release and cannot target a process. More seriously, after atomic rollback it reads the failed desired image/processes from the App CR and can retry the broken release. Restart must distinguish desired and last-deployed revisions. + +### Routes + +Current application routes converge on one Service. The POC permits exactly one routed deployment and now rejects external routes when a non-legacy process map has no routed target. Multiple independently routed web processes would require route-to-process mapping and collision validation. The audit verified Service traffic and ready TLS resources, not VM-to-Cilium ingress traffic. + +### Stage/image tracking + +One staged image and stage ID work cleanly for all processes. The audit also exposed that application-wide pod discovery can report the stage ID from an old hook pod. The CNB launcher distinction and deployed revision must become explicit execution state rather than tag and pod-order heuristics. Multi-image composition was intentionally out of scope. + +### Service bindings + +Source inspection confirms service instances remain separate Helm releases, and the chart mounts Epinio's binding/configuration Secrets into every process pod. The follow-up live matrix did not create a PostgreSQL/Redis-style service instance, so no live service-lifecycle claim is made. Production may want process-specific binding visibility, but services should remain independent from application rollback. + +### Release jobs + +The Helm hook is a good POC fit because failure blocks install/upgrade immediately. Revision-named Jobs retain direct success/failure evidence, but also worsen application-wide status/log noise and survive as hook history. Production work must define retries, timeouts, deletion/history, concurrent deploy behavior, idempotency, hook log retention, and whether rollback migrations are ever supported. + +### Resource naming + +The POC validates process names as DNS labels but does not yet define collision-safe composed names for maximum-length application/process pairs. Kubernetes also imposes a stricter 52-character limit on CronJob names. The failed long-name probe proves this is unresolved; production naming needs deterministic per-kind length bounds and hash suffixes with dedicated tests. + +## Production effort estimate + +For one engineer familiar with Epinio: roughly **8-12 engineering weeks** to reach a maintainable backend/CLI implementation, excluding polished UI work. A realistic breakdown is: + +- 1-2 weeks: versioned schema/CRD migration, validation, desired/deployed revision model +- 2-3 weeks: production chart contract, hooks, CNB/container execution semantics, upgrades +- 2-3 weeks: process-aware status/workload inventory and reconciliation +- 2-3 weeks: logs, exec, scale, restart, routes, and one-off CLI/API behavior +- 1-2 weeks: acceptance tests, compatibility/upgrade coverage, docs and operational hardening + +Two engineers could likely deliver a reviewed first production slice in 5-7 calendar weeks, with follow-up hardening. + +## Assessment + +**Feasible but invasive.** Epinio's Helm/AppChart boundary is an excellent extension point: one application release can naturally render multiple resources, successful upgrades are coordinated, hook failures block deployment, and atomic rollback restores ordinary workloads together. The difficult work is above and beside Helm—Epinio's App CR state transitions, global scale/status model, pod discovery, and process-agnostic operational commands. This is not fighting the fundamental deployment architecture, but it is substantially more than a chart-only feature. diff --git a/poc/multiprocess/appchart.yaml b/poc/multiprocess/appchart.yaml new file mode 100644 index 0000000000..1cc2b2521b --- /dev/null +++ b/poc/multiprocess/appchart.yaml @@ -0,0 +1,18 @@ +apiVersion: application.epinio.io/v1 +kind: AppChart +metadata: + name: multiprocess-poc + namespace: epinio + labels: + app.kubernetes.io/component: epinio + app.kubernetes.io/managed-by: epinio + app.kubernetes.io/name: epinio-multiprocess-poc-app-chart + app.kubernetes.io/part-of: epinio +spec: + shortDescription: Multi-process application POC + description: Deployment, cron, and release processes in one Epinio Helm release + helmChart: http://multiprocess-chart.epinio.svc.cluster.local/chart.tgz?v=0.2.3 + settings: + appListeningPort: + type: integer + minimum: "0" diff --git a/poc/multiprocess/chart-server.yaml b/poc/multiprocess/chart-server.yaml new file mode 100644 index 0000000000..a5807b7552 --- /dev/null +++ b/poc/multiprocess/chart-server.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: multiprocess-chart-server + namespace: epinio +spec: + replicas: 1 + selector: + matchLabels: + app: multiprocess-chart-server + template: + metadata: + labels: + app: multiprocess-chart-server + spec: + containers: + - name: nginx + image: nginx:1.29-alpine + ports: + - name: http + containerPort: 80 + volumeMounts: + - name: chart + mountPath: /usr/share/nginx/html/chart.tgz + subPath: chart.tgz + readOnly: true + volumes: + - name: chart + configMap: + name: multiprocess-chart +--- +apiVersion: v1 +kind: Service +metadata: + name: multiprocess-chart + namespace: epinio +spec: + selector: + app: multiprocess-chart-server + ports: + - name: http + port: 80 + targetPort: http diff --git a/poc/multiprocess/chart/Chart.yaml b/poc/multiprocess/chart/Chart.yaml new file mode 100644 index 0000000000..9d0886e31d --- /dev/null +++ b/poc/multiprocess/chart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: epinio-multiprocess-poc +description: Multi-process Epinio application chart proof of concept +type: application +version: 0.2.3 +appVersion: "0.2.3" diff --git a/poc/multiprocess/chart/ci/test-values.yaml b/poc/multiprocess/chart/ci/test-values.yaml new file mode 100644 index 0000000000..cb67756af1 --- /dev/null +++ b/poc/multiprocess/chart/ci/test-values.yaml @@ -0,0 +1,29 @@ +epinio: + appName: multiprocess-poc + imageURL: docker.io/epinio-poc/multiprocess:v1 + stageID: "" + staged: false + tlsIssuer: epinio-ca + ingress: cilium + username: poc + routes: + - id: multiprocess-poc + domain: multiprocess-poc.example.test + path: / + processes: + web: + kind: deployment + command: ["python", "/app/app.py", "web"] + replicas: 2 + routes: true + worker: + kind: deployment + command: ["python", "/app/app.py", "worker"] + replicas: 3 + scheduler: + kind: cron + command: ["python", "/app/app.py", "cron"] + schedule: "* * * * *" + release: + kind: release + command: ["python", "/app/app.py", "migrate"] diff --git a/poc/multiprocess/chart/templates/_helpers.tpl b/poc/multiprocess/chart/templates/_helpers.tpl new file mode 100644 index 0000000000..13a9315bd2 --- /dev/null +++ b/poc/multiprocess/chart/templates/_helpers.tpl @@ -0,0 +1,123 @@ +{{- define "epinio-multiprocess.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "epinio-multiprocess.resourceName" -}} +{{- printf "%s-%s" .root.Values.epinio.appName .process | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "epinio-multiprocess.containerName" -}} +{{- .Values.epinio.appName | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "epinio-multiprocess.serviceName" -}} +{{- .Values.epinio.appName | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Application listening port. Keep the standard Epinio chart's 8080 default and +its appListeningPort customization contract. +*/}} +{{- define "epinio-multiprocess.appListeningPort" -}} +{{ default 8080 (default (dict "appListeningPort" "8080") .Values.userConfig).appListeningPort }} +{{- end -}} + +{{- define "epinio-multiprocess.labels" -}} +app.kubernetes.io/managed-by: epinio +app.kubernetes.io/part-of: {{ .root.Release.Namespace | quote }} +app.kubernetes.io/name: {{ .root.Values.epinio.appName | quote }} +app.kubernetes.io/component: application +epinio.io/process-name: {{ .process | quote }} +epinio.io/release-revision: {{ .root.Release.Revision | quote }} +helm.sh/chart: {{ include "epinio-multiprocess.chart" .root }} +{{- end -}} + +{{- define "epinio-multiprocess.selectorLabels" -}} +app.kubernetes.io/name: {{ .root.Values.epinio.appName | quote }} +app.kubernetes.io/component: application +epinio.io/process-name: {{ .process | quote }} +{{- end -}} + +{{- define "epinio-multiprocess.podMetadata" -}} +annotations: + epinio.io/created-by: {{ .root.Values.epinio.username | quote }} + {{- with .root.Values.epinio.start }} + epinio.io/start: {{ . | quote }} + {{- end }} +labels: + {{- include "epinio-multiprocess.labels" . | nindent 2 }} + epinio.io/stage-id: {{ .root.Values.epinio.stageID | quote }} + epinio.io/app-container: {{ include "epinio-multiprocess.containerName" .root | quote }} +{{- end -}} + +{{- define "epinio-multiprocess.podSpecPrefix" -}} +serviceAccountName: {{ .Release.Namespace }} +automountServiceAccountToken: true +{{- with .Values.epinio.configurations }} +volumes: +{{- range . }} +- name: {{ . }} + secret: + defaultMode: 420 + secretName: {{ . }} +{{- end }} +{{- end }} +{{- end -}} + +{{- define "epinio-multiprocess.container" -}} +name: {{ include "epinio-multiprocess.containerName" .root }} +image: {{ .root.Values.epinio.imageURL }} +imagePullPolicy: {{ .root.Values.image.pullPolicy }} +{{- if .root.Values.epinio.staged }} +command: +- "/cnb/lifecycle/launcher" +args: +- "--" +{{- range .process.command }} +- {{ . | quote }} +{{- end }} +{{- else }} +command: + {{- toYaml .process.command | nindent 2 }} +{{- end }} +env: +- name: PORT + value: {{ include "epinio-multiprocess.appListeningPort" .root | quote }} +- name: EPINIO_PROCESS + value: {{ .name | quote }} +{{- range .root.Values.epinio.env }} +- name: {{ .name | quote }} + value: {{ .value | quote }} +{{- end }} +{{- with .root.Values.epinio.configpaths }} +volumeMounts: +{{- range . }} +- mountPath: /configurations/{{ .path }} + name: {{ .name }} + readOnly: true +{{- end }} +{{- end }} +{{- with .root.Values.resources }} +resources: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end -}} + +{{- define "epinio-multiprocess.podScheduling" -}} +{{- with .Values.imagePullSecrets }} +imagePullSecrets: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.nodeSelector }} +nodeSelector: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.tolerations }} +tolerations: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end -}} diff --git a/poc/multiprocess/chart/templates/cronjobs.yaml b/poc/multiprocess/chart/templates/cronjobs.yaml new file mode 100644 index 0000000000..10e9b5398c --- /dev/null +++ b/poc/multiprocess/chart/templates/cronjobs.yaml @@ -0,0 +1,31 @@ +{{- range $name, $process := .Values.epinio.processes }} +{{- if eq (default "deployment" $process.kind) "cron" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "epinio-multiprocess.resourceName" (dict "root" $ "process" $name) }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "epinio-multiprocess.labels" (dict "root" $ "process" $name) | nindent 4 }} +spec: + schedule: {{ $process.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + metadata: + labels: + {{- include "epinio-multiprocess.labels" (dict "root" $ "process" $name) | nindent 8 }} + spec: + template: + metadata: + {{- include "epinio-multiprocess.podMetadata" (dict "root" $ "process" $name) | nindent 10 }} + spec: + {{- include "epinio-multiprocess.podSpecPrefix" $ | nindent 10 }} + containers: + - {{ include "epinio-multiprocess.container" (dict "root" $ "process" $process "name" $name) | nindent 12 | trim }} + restartPolicy: Never + {{- include "epinio-multiprocess.podScheduling" $ | nindent 10 }} +{{- end }} +{{- end }} diff --git a/poc/multiprocess/chart/templates/deployments.yaml b/poc/multiprocess/chart/templates/deployments.yaml new file mode 100644 index 0000000000..3a4fb43f0f --- /dev/null +++ b/poc/multiprocess/chart/templates/deployments.yaml @@ -0,0 +1,39 @@ +{{- range $name, $process := .Values.epinio.processes }} +{{- if or (not $process.kind) (eq $process.kind "deployment") }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "epinio-multiprocess.resourceName" (dict "root" $ "process" $name) }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "epinio-multiprocess.labels" (dict "root" $ "process" $name) | nindent 4 }} + annotations: + epinio.io/created-by: {{ $.Values.epinio.username | quote }} +spec: + replicas: {{ if hasKey $process "replicas" }}{{ $process.replicas }}{{ else }}1{{ end }} + selector: + matchLabels: + {{- include "epinio-multiprocess.selectorLabels" (dict "root" $ "process" $name) | nindent 6 }} + template: + metadata: + {{- include "epinio-multiprocess.podMetadata" (dict "root" $ "process" $name) | nindent 6 }} + spec: + {{- include "epinio-multiprocess.podSpecPrefix" $ | nindent 6 }} + containers: + - {{ include "epinio-multiprocess.container" (dict "root" $ "process" $process "name" $name) | nindent 8 | trim }} + {{- if $process.routes }} + ports: + - name: http + containerPort: {{ include "epinio-multiprocess.appListeningPort" $ }} + protocol: TCP + readinessProbe: + tcpSocket: + port: http + initialDelaySeconds: 1 + periodSeconds: 2 + {{- end }} + restartPolicy: Always + {{- include "epinio-multiprocess.podScheduling" $ | nindent 6 }} +{{- end }} +{{- end }} diff --git a/poc/multiprocess/chart/templates/ingress.yaml b/poc/multiprocess/chart/templates/ingress.yaml new file mode 100644 index 0000000000..f968f0bf62 --- /dev/null +++ b/poc/multiprocess/chart/templates/ingress.yaml @@ -0,0 +1,60 @@ +{{- $routeProcess := "" -}} +{{- range $name, $process := .Values.epinio.processes -}} + {{- if and (or (not $process.kind) (eq $process.kind "deployment")) $process.routes -}} + {{- $routeProcess = $name -}} + {{- end -}} +{{- end -}} +{{- if $routeProcess }} +{{- range .Values.epinio.routes }} +{{- $routeName := printf "%s-%s" $.Values.epinio.appName .id | trunc 63 | trimSuffix "-" }} +{{- $tlsSecret := printf "%s-%s-tls" $.Values.epinio.appName .id | trunc 63 | trimSuffix "-" }} +{{- if not .secret }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ $routeName }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "epinio-multiprocess.labels" (dict "root" $ "process" $routeProcess) | nindent 4 }} + annotations: + epinio.io/created-by: {{ $.Values.epinio.username | quote }} +spec: + secretName: {{ $tlsSecret }} + dnsNames: + - {{ .domain | quote }} + issuerRef: + name: {{ $.Values.epinio.tlsIssuer | quote }} + kind: ClusterIssuer +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $routeName }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "epinio-multiprocess.labels" (dict "root" $ "process" $routeProcess) | nindent 4 }} + annotations: + epinio.io/created-by: {{ $.Values.epinio.username | quote }} +spec: + {{- with $.Values.epinio.ingress }} + ingressClassName: {{ . | quote }} + {{- end }} + rules: + - host: {{ .domain | quote }} + http: + paths: + - path: {{ .path | quote }} + pathType: ImplementationSpecific + backend: + service: + name: {{ include "epinio-multiprocess.serviceName" $ }} + port: + number: 8080 + tls: + - hosts: + - {{ .domain | quote }} + secretName: {{ default $tlsSecret .secret | quote }} +{{- end }} +{{- end }} diff --git a/poc/multiprocess/chart/templates/release-job.yaml b/poc/multiprocess/chart/templates/release-job.yaml new file mode 100644 index 0000000000..af2102a292 --- /dev/null +++ b/poc/multiprocess/chart/templates/release-job.yaml @@ -0,0 +1,28 @@ +{{- range $name, $process := .Values.epinio.processes }} +{{- if eq (default "deployment" $process.kind) "release" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s-r%d" $.Values.epinio.appName $name $.Release.Revision | trunc 63 | trimSuffix "-" }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "epinio-multiprocess.labels" (dict "root" $ "process" $name) | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: before-hook-creation + epinio.io/created-by: {{ $.Values.epinio.username | quote }} +spec: + backoffLimit: 0 + template: + metadata: + {{- include "epinio-multiprocess.podMetadata" (dict "root" $ "process" $name) | nindent 6 }} + spec: + {{- include "epinio-multiprocess.podSpecPrefix" $ | nindent 6 }} + containers: + - {{ include "epinio-multiprocess.container" (dict "root" $ "process" $process "name" $name) | nindent 8 | trim }} + restartPolicy: Never + {{- include "epinio-multiprocess.podScheduling" $ | nindent 6 }} +{{- end }} +{{- end }} diff --git a/poc/multiprocess/chart/templates/service.yaml b/poc/multiprocess/chart/templates/service.yaml new file mode 100644 index 0000000000..8831e5818b --- /dev/null +++ b/poc/multiprocess/chart/templates/service.yaml @@ -0,0 +1,23 @@ +{{- range $name, $process := .Values.epinio.processes }} +{{- if and (or (not $process.kind) (eq $process.kind "deployment")) $process.routes }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "epinio-multiprocess.serviceName" $ }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "epinio-multiprocess.labels" (dict "root" $ "process" $name) | nindent 4 }} + annotations: + epinio.io/created-by: {{ $.Values.epinio.username | quote }} +spec: + type: ClusterIP + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: http + selector: + {{- include "epinio-multiprocess.selectorLabels" (dict "root" $ "process" $name) | nindent 4 }} +{{- end }} +{{- end }} diff --git a/poc/multiprocess/chart/values.yaml b/poc/multiprocess/chart/values.yaml new file mode 100644 index 0000000000..6ee1cc5c5b --- /dev/null +++ b/poc/multiprocess/chart/values.yaml @@ -0,0 +1,22 @@ +image: + pullPolicy: IfNotPresent + +epinio: + appName: placeholder + imageURL: "" + stageID: "" + staged: false + tlsIssuer: "" + username: "" + routes: [] + env: [] + configurations: [] + configpaths: [] + processes: {} + +resources: {} +imagePullSecrets: [] +nodeSelector: {} +tolerations: [] +affinity: {} +userConfig: {} diff --git a/poc/multiprocess/crd-processes-json-patch.json b/poc/multiprocess/crd-processes-json-patch.json new file mode 100644 index 0000000000..98fb975984 --- /dev/null +++ b/poc/multiprocess/crd-processes-json-patch.json @@ -0,0 +1,32 @@ +[ + { + "op": "add", + "path": "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/processes", + "value": { + "description": "Named release-scoped application processes (multi-process POC).", + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["command"], + "properties": { + "kind": { + "type": "string", + "enum": ["deployment", "cron", "release"] + }, + "command": { + "type": "array", + "minItems": 1, + "items": {"type": "string"} + }, + "replicas": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "schedule": {"type": "string"}, + "routes": {"type": "boolean"} + } + } + } + } +] diff --git a/poc/multiprocess/epinio-invalid-no-route-target.yml b/poc/multiprocess/epinio-invalid-no-route-target.yml new file mode 100644 index 0000000000..21ce709e50 --- /dev/null +++ b/poc/multiprocess/epinio-invalid-no-route-target.yml @@ -0,0 +1,12 @@ +name: multiprocess-invalid-route +configuration: + appchart: multiprocess-poc + routes: + - multiprocess-invalid-route.poc.10.42.0.42.sslip.io + processes: + worker: + kind: deployment + command: ["python", "/app/app.py", "worker"] + replicas: 1 +origin: + container: docker.io/epinio-poc/multiprocess:v1 diff --git a/poc/multiprocess/epinio-migration-fail.yml b/poc/multiprocess/epinio-migration-fail.yml new file mode 100644 index 0000000000..8aa470adac --- /dev/null +++ b/poc/multiprocess/epinio-migration-fail.yml @@ -0,0 +1,24 @@ +name: multiprocess-poc +configuration: + appchart: multiprocess-poc + routes: + - multiprocess-poc.poc.10.42.0.42.sslip.io + processes: + web: + kind: deployment + command: ["python", "/app/app.py", "web"] + replicas: 2 + routes: true + worker: + kind: deployment + command: ["python", "/app/app.py", "worker"] + replicas: 3 + scheduler: + kind: cron + command: ["python", "/app/app.py", "cron"] + schedule: "* * * * *" + release: + kind: release + command: ["python", "/app/app.py", "migrate", "fail"] +origin: + container: docker.io/epinio-poc/multiprocess:v3 diff --git a/poc/multiprocess/epinio-staged.yml b/poc/multiprocess/epinio-staged.yml new file mode 100644 index 0000000000..b11939f245 --- /dev/null +++ b/poc/multiprocess/epinio-staged.yml @@ -0,0 +1,26 @@ +name: multiprocess-poc +configuration: + appchart: multiprocess-poc + environment: + IMAGE_VERSION: staged-v5 + routes: + - multiprocess-poc.poc.10.42.0.42.sslip.io + processes: + web: + kind: deployment + command: ["python", "app.py", "web"] + replicas: 2 + routes: true + worker: + kind: deployment + command: ["python", "app.py", "worker"] + replicas: 3 + scheduler: + kind: cron + command: ["python", "app.py", "cron"] + schedule: "* * * * *" + release: + kind: release + command: ["python", "app.py", "migrate"] +origin: + path: test-app diff --git a/poc/multiprocess/epinio-unhealthy.yml b/poc/multiprocess/epinio-unhealthy.yml new file mode 100644 index 0000000000..d72a88b4de --- /dev/null +++ b/poc/multiprocess/epinio-unhealthy.yml @@ -0,0 +1,24 @@ +name: multiprocess-poc +configuration: + appchart: multiprocess-poc + routes: + - multiprocess-poc.poc.10.42.0.42.sslip.io + processes: + web: + kind: deployment + command: ["python", "/app/app.py", "web"] + replicas: 2 + routes: true + worker: + kind: deployment + command: ["python", "/app/app.py", "crash"] + replicas: 3 + scheduler: + kind: cron + command: ["python", "/app/app.py", "cron"] + schedule: "* * * * *" + release: + kind: release + command: ["python", "/app/app.py", "migrate"] +origin: + container: docker.io/epinio-poc/multiprocess:v4 diff --git a/poc/multiprocess/epinio-v1.yml b/poc/multiprocess/epinio-v1.yml new file mode 100644 index 0000000000..0e8b2513ea --- /dev/null +++ b/poc/multiprocess/epinio-v1.yml @@ -0,0 +1,24 @@ +name: multiprocess-poc +configuration: + appchart: multiprocess-poc + routes: + - multiprocess-poc.poc.10.42.0.42.sslip.io + processes: + web: + kind: deployment + command: ["python", "/app/app.py", "web"] + replicas: 2 + routes: true + worker: + kind: deployment + command: ["python", "/app/app.py", "worker"] + replicas: 2 + scheduler: + kind: cron + command: ["python", "/app/app.py", "cron"] + schedule: "* * * * *" + release: + kind: release + command: ["python", "/app/app.py", "migrate"] +origin: + container: docker.io/epinio-poc/multiprocess:v1 diff --git a/poc/multiprocess/epinio-v2.yml b/poc/multiprocess/epinio-v2.yml new file mode 100644 index 0000000000..3c85c5ef8b --- /dev/null +++ b/poc/multiprocess/epinio-v2.yml @@ -0,0 +1,24 @@ +name: multiprocess-poc +configuration: + appchart: multiprocess-poc + routes: + - multiprocess-poc.poc.10.42.0.42.sslip.io + processes: + web: + kind: deployment + command: ["python", "/app/app.py", "web"] + replicas: 2 + routes: true + worker: + kind: deployment + command: ["python", "/app/app.py", "worker"] + replicas: 3 + scheduler: + kind: cron + command: ["python", "/app/app.py", "cron"] + schedule: "* * * * *" + release: + kind: release + command: ["python", "/app/app.py", "migrate"] +origin: + container: docker.io/epinio-poc/multiprocess:v2 diff --git a/poc/multiprocess/scripts/install-chart.sh b/poc/multiprocess/scripts/install-chart.sh new file mode 100755 index 0000000000..cd7ebb6635 --- /dev/null +++ b/poc/multiprocess/scripts/install-chart.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +namespace=${EPINIO_NAMESPACE:-epinio} +package_dir=$(mktemp -d) +trap 'rm -rf "$package_dir"' EXIT + +cd "$repo_root" +helm package poc/multiprocess/chart --destination "$package_dir" +chart_archive="$package_dir/epinio-multiprocess-poc-0.2.3.tgz" + +kubectl -n "$namespace" create configmap multiprocess-chart \ + --from-file=chart.tgz="$chart_archive" \ + --dry-run=client -o yaml | kubectl apply -f - +kubectl apply -f poc/multiprocess/chart-server.yaml +kubectl rollout restart deployment/multiprocess-chart-server -n "$namespace" +kubectl rollout status deployment/multiprocess-chart-server -n "$namespace" --timeout=2m +kubectl apply -f poc/multiprocess/appchart.yaml + +local_sha=$(sha256sum "$chart_archive" | awk '{print $1}') +cluster_sha=$(kubectl -n "$namespace" get configmap multiprocess-chart \ + -o jsonpath='{.binaryData.chart\.tgz}' | base64 -d | sha256sum | awk '{print $1}') + +if [[ "$local_sha" != "$cluster_sha" ]]; then + echo "ASSERTION FAILED: packaged chart and ConfigMap archive differ" >&2 + exit 1 +fi + +echo "Installed chart archive sha256=$cluster_sha" +kubectl -n "$namespace" get appchart multiprocess-poc \ + -o jsonpath='{.spec.helmChart}{"\n"}' diff --git a/poc/multiprocess/scripts/run-live-audit.sh b/poc/multiprocess/scripts/run-live-audit.sh new file mode 100755 index 0000000000..c6b9143624 --- /dev/null +++ b/poc/multiprocess/scripts/run-live-audit.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +cli=${EPINIO_CLI:-$repo_root/dist/epinio-linux-amd64} +namespace=${EPINIO_TEST_NAMESPACE:-poc} +app=${EPINIO_TEST_APP:-multiprocess-audit} +domain=${EPINIO_TEST_DOMAIN:-10.42.0.42.sslip.io} +route="$app.$namespace.$domain" +evidence_dir=${EPINIO_EVIDENCE_DIR:-$repo_root/poc/multiprocess/evidence/live} +selector="app.kubernetes.io/name=$app" +release="" + +mkdir -p "$evidence_dir" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_eq() { + local expected=$1 + local actual=$2 + local description=$3 + if [[ "$actual" != "$expected" ]]; then + fail "$description: expected [$expected], got [$actual]" + fi +} + +assert_contains() { + local expected=$1 + local actual=$2 + local description=$3 + if [[ "$actual" != *"$expected"* ]]; then + fail "$description: expected [$actual] to contain [$expected]" + fi +} + +current_revision() { + helm status "$release" -n "$namespace" -o json | jq -r '.version' +} + +revision_status() { + local revision=$1 + helm history "$release" -n "$namespace" -o json | + jq -r --argjson revision "$revision" '.[] | select(.revision == $revision) | .status' +} + +snapshot() { + local case_name=$1 + local case_dir="$evidence_dir/$case_name" + mkdir -p "$case_dir" + + kubectl -n "$namespace" get app "$app" -o yaml >"$case_dir/app.yaml" + kubectl -n "$namespace" get deployment,replicaset,service,ingress,cronjob,job,pod,certificate \ + -l "$selector" -o yaml >"$case_dir/resources.yaml" + kubectl -n "$namespace" get deployment,replicaset,service,ingress,cronjob,job,pod,certificate \ + -l "$selector" -o wide >"$case_dir/resources.txt" + helm history "$release" -n "$namespace" -o json >"$case_dir/helm-history.json" + helm get values "$release" -n "$namespace" -o yaml >"$case_dir/helm-values.yaml" + helm get manifest "$release" -n "$namespace" >"$case_dir/helm-manifest.yaml" + helm get hooks "$release" -n "$namespace" >"$case_dir/helm-hooks.yaml" + kubectl -n "$namespace" get events --sort-by=.metadata.creationTimestamp | \ + rg "$app" >"$case_dir/events.txt" || true +} + +run_push() { + local case_name=$1 + local manifest=$2 + local expectation=$3 + local case_dir="$evidence_dir/$case_name" + mkdir -p "$case_dir" + + echo "[$case_name] pushing $manifest (expect $expectation)" + set +e + /usr/bin/time -f 'elapsed=%e exit=%x' \ + "$cli" push --name "$app" --route "$route" "$repo_root/poc/multiprocess/$manifest" \ + >"$case_dir/push.log" 2>&1 + push_rc=$? + set -e + + echo "[$case_name] push exit=$push_rc" + if [[ "$expectation" == success && "$push_rc" -ne 0 ]]; then + tail -80 "$case_dir/push.log" >&2 + fail "$case_name push unexpectedly failed" + fi + if [[ "$expectation" == failure && "$push_rc" -eq 0 ]]; then + fail "$case_name push unexpectedly succeeded" + fi +} + +assert_deployment() { + local process=$1 + local replicas=$2 + local image=$3 + local expected_command_json=$4 + local name="$app-$process" + + kubectl -n "$namespace" rollout status "deployment/$name" --timeout=2m >/dev/null + assert_eq "$replicas" "$(kubectl -n "$namespace" get deployment "$name" -o jsonpath='{.spec.replicas}')" "$name replicas" + assert_eq "$replicas" "$(kubectl -n "$namespace" get deployment "$name" -o jsonpath='{.status.readyReplicas}')" "$name ready replicas" + assert_eq "$image" "$(kubectl -n "$namespace" get deployment "$name" -o jsonpath='{.spec.template.spec.containers[0].image}')" "$name image" + assert_eq "$expected_command_json" "$(kubectl -n "$namespace" get deployment "$name" -o json | jq -c '.spec.template.spec.containers[0].command')" "$name command" +} + +assert_cronjob() { + local image=$1 + local expected_command_json=$2 + local name="$app-scheduler" + + assert_eq '* * * * *' "$(kubectl -n "$namespace" get cronjob "$name" -o jsonpath='{.spec.schedule}')" "$name schedule" + assert_eq "$image" "$(kubectl -n "$namespace" get cronjob "$name" -o jsonpath='{.spec.jobTemplate.spec.template.spec.containers[0].image}')" "$name image" + assert_eq "$expected_command_json" "$(kubectl -n "$namespace" get cronjob "$name" -o json | jq -c '.spec.jobTemplate.spec.template.spec.containers[0].command')" "$name command" +} + +assert_release_job() { + local revision=$1 + local expected_condition=$2 + local image=$3 + local log_pattern=$4 + local case_dir=$5 + local name="$app-release-r$revision" + + kubectl -n "$namespace" get job "$name" -o yaml >"$case_dir/$name.yaml" + assert_eq "$image" "$(kubectl -n "$namespace" get job "$name" -o jsonpath='{.spec.template.spec.containers[0].image}')" "$name image" + + if [[ "$expected_condition" == complete ]]; then + kubectl -n "$namespace" wait --for=condition=complete "job/$name" --timeout=30s >/dev/null + else + kubectl -n "$namespace" wait --for=condition=failed "job/$name" --timeout=30s >/dev/null + fi + + kubectl -n "$namespace" logs "job/$name" >"$case_dir/$name.log" + assert_contains "$log_pattern" "$(cat "$case_dir/$name.log")" "$name logs" + + kubectl -n "$namespace" get pod \ + -l "$selector,epinio.io/process-name=release,epinio.io/release-revision=$revision" \ + -o json >"$case_dir/$name-pods.json" + + if [[ "$expected_condition" == failed ]]; then + assert_eq 42 "$(jq -r '[.items[].status.containerStatuses[]?.state.terminated.exitCode] | unique | .[0]' \ + "$case_dir/$name-pods.json")" "$name container exit code" + fi +} + +assert_no_release_job() { + local revision=$1 + local case_dir=$2 + local name="$app-release-r$revision" + + if kubectl -n "$namespace" get job "$name" >"$case_dir/$name-lookup.txt" 2>&1; then + fail "$name exists, but Helm's automatic atomic rollback was expected to skip hooks" + fi + echo "not created: automatic atomic rollback skipped release hooks" >"$case_dir/$name-lookup.txt" +} + +assert_scheduled_cron_run() { + local revision=$1 + local image=$2 + local log_pattern=$3 + local case_dir=$4 + local deadline=$((SECONDS + 100)) + local job="" + + echo "[cron] waiting for controller-created revision $revision job" + while (( SECONDS < deadline )); do + job=$(kubectl -n "$namespace" get job \ + -l "$selector,epinio.io/process-name=scheduler,epinio.io/release-revision=$revision" \ + -o json | jq -r --arg image "$image" ' + [.items[] + | select(.metadata.ownerReferences[]?.kind == "CronJob") + | select(.spec.template.spec.containers[0].image == $image)] + | sort_by(.metadata.creationTimestamp) + | last + | .metadata.name // empty') + if [[ -n "$job" ]]; then + break + fi + sleep 2 + done + + [[ -n "$job" ]] || fail "no controller-created cron Job for revision $revision and image $image" + kubectl -n "$namespace" wait --for=condition=complete "job/$job" --timeout=30s >/dev/null + kubectl -n "$namespace" get job "$job" -o yaml >"$case_dir/$job.yaml" + kubectl -n "$namespace" logs "job/$job" >"$case_dir/$job.log" + assert_contains "$log_pattern" "$(cat "$case_dir/$job.log")" "$job logs" + echo "[cron] observed $job: $(cat "$case_dir/$job.log")" +} + +assert_route_and_service() { + local expected_version=$1 + local case_dir=$2 + local ingress_tls_secret + local certificate_name + local response="" + local port=18082 + + assert_eq web "$(kubectl -n "$namespace" get service "$app" -o jsonpath='{.spec.selector.epinio\.io/process-name}')" "service route process" + assert_eq "$route" "$(kubectl -n "$namespace" get ingress -l "$selector" -o jsonpath='{.items[0].spec.rules[0].host}')" "ingress host" + ingress_tls_secret=$(kubectl -n "$namespace" get ingress -l "$selector" -o jsonpath='{.items[0].spec.tls[0].secretName}') + [[ -n "$ingress_tls_secret" ]] || fail "Ingress TLS secret is empty" + certificate_name=$(kubectl -n "$namespace" get certificate -l "$selector" -o jsonpath='{.items[0].metadata.name}') + [[ -n "$certificate_name" ]] || fail "Certificate was not rendered" + kubectl -n "$namespace" wait --for=condition=ready "certificate/$certificate_name" --timeout=90s >/dev/null + kubectl -n "$namespace" get secret "$ingress_tls_secret" >/dev/null + + kubectl -n "$namespace" port-forward "service/$app" "$port:8080" >"$case_dir/port-forward.log" 2>&1 & + local forward_pid=$! + for _ in 1 2 3 4 5 6 7 8 9 10; do + if response=$(curl --silent --show-error --max-time 3 "http://127.0.0.1:$port"); then + break + fi + sleep 1 + done + kill "$forward_pid" 2>/dev/null || true + wait "$forward_pid" 2>/dev/null || true + + printf '%s\n' "$response" >"$case_dir/web-response.txt" + assert_contains "web version=$expected_version process=web" "$response" "web response" +} + +assert_prebuilt_state() { + local version=$1 + local worker_replicas=$2 + local case_dir=$3 + local image="docker.io/epinio-poc/multiprocess:$version" + + assert_deployment web 2 "$image" '["python","/app/app.py","web"]' + assert_deployment worker "$worker_replicas" "$image" '["python","/app/app.py","worker"]' + assert_cronjob "$image" '["python","/app/app.py","cron"]' + assert_route_and_service "$version" "$case_dir" +} + +assert_candidate_fully_scaled_down() { + local image=$1 + local deadline=$((SECONDS + 120)) + local pod_count + local nonzero_rs + + echo "[rollback] waiting for candidate pods and ReplicaSets to converge to zero" + while (( SECONDS < deadline )); do + pod_count=$(kubectl -n "$namespace" get pod -l "$selector" -o json | + jq -r --arg image "$image" '[.items[] + | select(.metadata.ownerReferences[]?.kind == "ReplicaSet") + | select(.spec.containers[0].image == $image)] | length') + nonzero_rs=$(kubectl -n "$namespace" get replicaset -l "$selector" -o json | + jq -r --arg image "$image" '[.items[] + | select(.spec.template.spec.containers[0].image == $image) + | select((.spec.replicas // 0) != 0 or (.status.replicas // 0) != 0)] | length') + if [[ "$pod_count" == 0 && "$nonzero_rs" == 0 ]]; then + echo "[rollback] candidate image fully scaled down" + return 0 + fi + sleep 2 + done + + fail "candidate image $image still has pods or nonzero ReplicaSets after rollback" +} + +cd "$repo_root" + +if [[ ! -x "$cli" ]]; then + fail "Epinio CLI not executable: $cli" +fi +if kubectl -n "$namespace" get app "$app" >/dev/null 2>&1; then + fail "test application $namespace/$app already exists; refusing to overwrite evidence" +fi + +{ + date --utc --iso-8601=seconds + git rev-parse HEAD + kubectl version -o json | jq -c '{clientVersion:.clientVersion.gitVersion,serverVersion:.serverVersion.gitVersion}' + helm version --short + "$cli" info +} >"$evidence_dir/environment.txt" 2>&1 + +run_push 01-initial-v1 epinio-v1.yml success +release=$(kubectl -n "$namespace" get deployment "$app-web" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}') +[[ -n "$release" ]] || fail "could not discover Helm release" +revision=$(current_revision) +assert_eq 1 "$revision" "initial Helm revision" +assert_eq deployed "$(revision_status "$revision")" "initial Helm status" +assert_prebuilt_state v1 2 "$evidence_dir/01-initial-v1" +assert_release_job "$revision" complete docker.io/epinio-poc/multiprocess:v1 'migration version=v1 fail=False' "$evidence_dir/01-initial-v1" +assert_scheduled_cron_run "$revision" docker.io/epinio-poc/multiprocess:v1 'cron ran version=v1' "$evidence_dir/01-initial-v1" +snapshot 01-initial-v1 + +run_push 02-upgrade-v2 epinio-v2.yml success +revision=$(current_revision) +assert_eq 2 "$revision" "upgrade Helm revision" +assert_eq deployed "$(revision_status "$revision")" "upgrade Helm status" +assert_prebuilt_state v2 3 "$evidence_dir/02-upgrade-v2" +assert_release_job "$revision" complete docker.io/epinio-poc/multiprocess:v2 'migration version=v2 fail=False' "$evidence_dir/02-upgrade-v2" +assert_scheduled_cron_run "$revision" docker.io/epinio-poc/multiprocess:v2 'cron ran version=v2' "$evidence_dir/02-upgrade-v2" +snapshot 02-upgrade-v2 + +run_push 03-failed-release epinio-migration-fail.yml failure +assert_eq 4 "$(current_revision)" "failed-release rollback revision" +assert_eq failed "$(revision_status 3)" "failed-release candidate status" +assert_eq deployed "$(revision_status 4)" "failed-release rollback status" +helm get values "$release" -n "$namespace" --revision 3 -o yaml >"$evidence_dir/03-failed-release/failed-values.yaml" +helm get hooks "$release" -n "$namespace" --revision 3 >"$evidence_dir/03-failed-release/failed-hooks.yaml" +assert_release_job 3 failed docker.io/epinio-poc/multiprocess:v3 'migration version=v3 fail=True' "$evidence_dir/03-failed-release" +assert_no_release_job 4 "$evidence_dir/03-failed-release" +assert_prebuilt_state v2 3 "$evidence_dir/03-failed-release" +assert_eq docker.io/epinio-poc/multiprocess:v3 "$(kubectl -n "$namespace" get app "$app" -o jsonpath='{.spec.imageurl}')" "failed-release desired App image" +assert_eq docker.io/epinio-poc/multiprocess:v2 "$(kubectl -n "$namespace" get app "$app" -o jsonpath='{.spec.origin.container}')" "failed-release persisted origin" +snapshot 03-failed-release + +run_push 04-unhealthy-worker epinio-unhealthy.yml failure +assert_eq 6 "$(current_revision)" "unhealthy rollback revision" +assert_eq failed "$(revision_status 5)" "unhealthy candidate status" +assert_eq deployed "$(revision_status 6)" "unhealthy rollback status" +helm get values "$release" -n "$namespace" --revision 5 -o yaml >"$evidence_dir/04-unhealthy-worker/failed-values.yaml" +helm get manifest "$release" -n "$namespace" --revision 5 >"$evidence_dir/04-unhealthy-worker/failed-manifest.yaml" +assert_release_job 5 complete docker.io/epinio-poc/multiprocess:v4 'migration version=v4 fail=False' "$evidence_dir/04-unhealthy-worker" +assert_no_release_job 6 "$evidence_dir/04-unhealthy-worker" +assert_prebuilt_state v2 3 "$evidence_dir/04-unhealthy-worker" +candidate_rs=$(kubectl -n "$namespace" get replicaset -l "$selector,epinio.io/process-name=worker" -o json | + jq -r '[.items[] | select(.spec.template.spec.containers[0].image == "docker.io/epinio-poc/multiprocess:v4")] | length') +[[ "$candidate_rs" -gt 0 ]] || fail "no v4 worker ReplicaSet remained as evidence of candidate rollout" +assert_candidate_fully_scaled_down docker.io/epinio-poc/multiprocess:v4 +assert_eq docker.io/epinio-poc/multiprocess:v4 "$(kubectl -n "$namespace" get app "$app" -o jsonpath='{.spec.imageurl}')" "unhealthy desired App image" +assert_eq docker.io/epinio-poc/multiprocess:v2 "$(kubectl -n "$namespace" get app "$app" -o jsonpath='{.spec.origin.container}')" "unhealthy persisted origin" +snapshot 04-unhealthy-worker + +run_push 05-staged-direct epinio-staged.yml success +revision=$(current_revision) +assert_eq 7 "$revision" "staged Helm revision" +assert_eq deployed "$(revision_status "$revision")" "staged Helm status" +stage_id=$(kubectl -n "$namespace" get app "$app" -o jsonpath='{.spec.stageid}') +app_image=$(kubectl -n "$namespace" get app "$app" -o jsonpath='{.spec.imageurl}') +[[ -n "$stage_id" ]] || fail "staged App stage ID is empty" +[[ "$app_image" == *":$stage_id" ]] || fail "staged App image does not use stage ID as its complete tag" +[[ -n "$(kubectl -n "$namespace" get app "$app" -o jsonpath='{.spec.origin.path}')" ]] || fail "successful staged push did not persist path origin" +runtime_image=$(kubectl -n "$namespace" get deployment "$app-web" -o jsonpath='{.spec.template.spec.containers[0].image}') +assert_deployment web 2 "$runtime_image" '["/cnb/lifecycle/launcher"]' +assert_deployment worker 3 "$runtime_image" '["/cnb/lifecycle/launcher"]' +assert_cronjob "$runtime_image" '["/cnb/lifecycle/launcher"]' +assert_eq '["--","python","app.py","web"]' "$(kubectl -n "$namespace" get deployment "$app-web" -o json | jq -c '.spec.template.spec.containers[0].args')" "staged web args" +assert_eq '["--","python","app.py","worker"]' "$(kubectl -n "$namespace" get deployment "$app-worker" -o json | jq -c '.spec.template.spec.containers[0].args')" "staged worker args" +assert_eq "$runtime_image" "$(kubectl -n "$namespace" get cronjob "$app-scheduler" -o jsonpath='{.spec.jobTemplate.spec.template.spec.containers[0].image}')" "staged cron image" +assert_release_job "$revision" complete "$runtime_image" 'migration version=staged-v5 fail=False' "$evidence_dir/05-staged-direct" +assert_scheduled_cron_run "$revision" "$runtime_image" 'cron ran version=staged-v5' "$evidence_dir/05-staged-direct" +assert_route_and_service staged-v5 "$evidence_dir/05-staged-direct" +snapshot 05-staged-direct + +echo "$release" >"$evidence_dir/helm-release.txt" +echo "PASS: live initial, upgrade, cron, release failure, unhealthy rollback, and direct staged push matrix" diff --git a/poc/multiprocess/scripts/verify-static.sh b/poc/multiprocess/scripts/verify-static.sh new file mode 100755 index 0000000000..b194b06d3f --- /dev/null +++ b/poc/multiprocess/scripts/verify-static.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +chart="$repo_root/poc/multiprocess/chart" +values="$chart/ci/test-values.yaml" +output_dir=${1:-$(mktemp -d)} +mkdir -p "$output_dir" + +prebuilt="$output_dir/prebuilt.yaml" +staged="$output_dir/staged.yaml" +zero="$output_dir/zero-replicas.yaml" +port_80="$output_dir/listening-port-80.yaml" + +assert_contains() { + local pattern=$1 + local file=$2 + if ! rg --quiet --multiline "$pattern" "$file"; then + echo "ASSERTION FAILED: pattern [$pattern] not found in $file" >&2 + return 1 + fi +} + +assert_count() { + local expected=$1 + local pattern=$2 + local file=$3 + local actual + actual=$(rg --count "$pattern" "$file" || true) + if [[ "$actual" != "$expected" ]]; then + echo "ASSERTION FAILED: expected $expected matches for [$pattern] in $file, got $actual" >&2 + return 1 + fi +} + +assert_not_contains() { + local pattern=$1 + local file=$2 + if rg --quiet --multiline "$pattern" "$file"; then + echo "ASSERTION FAILED: unexpected pattern [$pattern] found in $file" >&2 + return 1 + fi +} + +cd "$repo_root" + +echo "[go] focused packages" +go test -count=1 \ + ./pkg/api/core/v1/models \ + ./internal/manifest \ + ./internal/application \ + ./internal/helm \ + ./internal/api/v1/application + +echo "[helm] lint and render" +helm lint "$chart" -f "$values" +helm template audit "$chart" --namespace poc -f "$values" >"$prebuilt" +helm template audit "$chart" --namespace poc -f "$values" --set epinio.staged=true >"$staged" +helm template audit "$chart" --namespace poc -f "$values" --set epinio.processes.worker.replicas=0 >"$zero" +helm template audit "$chart" --namespace poc -f "$values" --set userConfig.appListeningPort=80 >"$port_80" + +echo "[render] resource and semantic assertions" +assert_count 2 '^kind: Deployment$' "$prebuilt" +assert_count 1 '^kind: Service$' "$prebuilt" +assert_count 1 '^kind: Ingress$' "$prebuilt" +assert_count 1 '^kind: Certificate$' "$prebuilt" +assert_count 1 '^kind: CronJob$' "$prebuilt" +assert_count 1 '^kind: Job$' "$prebuilt" +assert_contains 'name: multiprocess-poc-web.*\n(?:.*\n){0,15}spec:\n replicas: 2' "$prebuilt" +assert_contains 'name: multiprocess-poc-worker.*\n(?:.*\n){0,15}spec:\n replicas: 3' "$prebuilt" +assert_contains 'name: multiprocess-poc-worker.*\n(?:.*\n){0,15}spec:\n replicas: 0' "$zero" +assert_contains 'schedule: "\* \* \* \* \*"' "$prebuilt" +assert_contains 'name: multiprocess-poc-release-r1' "$prebuilt" +assert_contains 'helm.sh/hook: pre-install,pre-upgrade' "$prebuilt" +assert_contains 'secretName: "multiprocess-poc-multiprocess-poc-tls"' "$prebuilt" +assert_contains 'image: docker.io/epinio-poc/multiprocess:v1.*\n imagePullPolicy: IfNotPresent.*\n command:\n - python' "$prebuilt" +assert_count 4 '"/cnb/lifecycle/launcher"' "$staged" +assert_count 4 '^[[:space:]]+- "--"$' "$staged" +assert_contains 'containerPort: 8080' "$prebuilt" +assert_count 4 'value: "8080"' "$prebuilt" +assert_contains 'containerPort: 80' "$port_80" +assert_not_contains 'containerPort: 8080' "$port_80" +assert_count 4 'value: "80"' "$port_80" +assert_contains 'port: 8080\n[[:space:]]+protocol: TCP\n[[:space:]]+targetPort: http' "$port_80" + +echo "[kubernetes] client-side schema dry runs" +kubectl create --dry-run=client -f "$prebuilt" -o name >"$output_dir/prebuilt-dry-run.txt" +kubectl create --dry-run=client -f "$staged" -o name >"$output_dir/staged-dry-run.txt" +kubectl create --dry-run=client -f "$port_80" -o name >"$output_dir/listening-port-80-dry-run.txt" + +echo "[git] whitespace check" +git diff --check + +echo "PASS: focused Go tests, Helm lint/render assertions, Kubernetes dry runs, and diff check" +echo "Rendered evidence: $output_dir" diff --git a/poc/multiprocess/test-app/Dockerfile b/poc/multiprocess/test-app/Dockerfile new file mode 100644 index 0000000000..185245946b --- /dev/null +++ b/poc/multiprocess/test-app/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.13-alpine +ARG IMAGE_VERSION=dev +ENV IMAGE_VERSION=$IMAGE_VERSION +WORKDIR /app +COPY app.py /app/app.py +EXPOSE 8080 diff --git a/poc/multiprocess/test-app/Procfile b/poc/multiprocess/test-app/Procfile new file mode 100644 index 0000000000..e9834d5568 --- /dev/null +++ b/poc/multiprocess/test-app/Procfile @@ -0,0 +1 @@ +web: python app.py web diff --git a/poc/multiprocess/test-app/app.py b/poc/multiprocess/test-app/app.py new file mode 100644 index 0000000000..042bf84d33 --- /dev/null +++ b/poc/multiprocess/test-app/app.py @@ -0,0 +1,40 @@ +import http.server +import os +import sys +import time + + +version = os.environ.get("IMAGE_VERSION", "unknown") +process = sys.argv[1] if len(sys.argv) > 1 else "web" + +if process == "web": + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = f"web version={version} process={os.environ.get('EPINIO_PROCESS')}\n".encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, message, *args): + print(f"web version={version}: {message % args}", flush=True) + + print(f"web starting version={version}", flush=True) + http.server.ThreadingHTTPServer(("0.0.0.0", 8080), Handler).serve_forever() +elif process == "worker": + print(f"worker starting version={version}", flush=True) + while True: + print(f"worker heartbeat version={version}", flush=True) + time.sleep(5) +elif process == "cron": + print(f"cron ran version={version}", flush=True) +elif process == "migrate": + should_fail = len(sys.argv) > 2 and sys.argv[2] == "fail" + print(f"migration version={version} fail={should_fail}", flush=True) + sys.exit(42 if should_fail else 0) +elif process == "crash": + print(f"deliberate process failure version={version}", flush=True) + sys.exit(23) +else: + raise SystemExit(f"unknown process {process}") diff --git a/poc/multiprocess/test-app/requirements.txt b/poc/multiprocess/test-app/requirements.txt new file mode 100644 index 0000000000..841300749c --- /dev/null +++ b/poc/multiprocess/test-app/requirements.txt @@ -0,0 +1 @@ +# The POC uses only the Python standard library.