diff --git a/README.md b/README.md index 83cb41e..fbb98aa 100644 --- a/README.md +++ b/README.md @@ -38,16 +38,17 @@ The features provided by this plugin are: > in the object store, restore is currently tested only with full backup recovery > to the latest backup. Reports on more advanced recovery attempts are welcome. -This plugin is currently only compatible with S3 object storage. +This plugin is compatible with S3 and Azure Blob object storage. The following storage solutions have been tested and confirmed to work with this implementation: - [MinIO](https://min.io/) – An S3-compatible object storage solution. +- [Azure Blob Storage](https://azure.microsoft.com/products/storage/blobs). Known missing features: -- support for other object storage solutions (GCS, Azure), +- support for other object storage solutions (GCS), - backups from replicas, - proper support for private certificate authorities. @@ -194,6 +195,47 @@ spec: cpu: "2" ``` +For Azure Blob storage, use the `azureCredentials` field instead of +`s3Credentials`. The `bucket` field is used as the Azure container name: + +```yaml +apiVersion: pgbackrest.cnpg.opera.com/v1 +kind: Archive +metadata: + name: azure-store +spec: + configuration: + repositories: + - destinationPath: / + bucket: backups + azureCredentials: + # keyType defaults to "shared". Use "sas" to pass a SAS token as the key. + account: + name: azure + key: AZURE_STORAGE_ACCOUNT + key: + name: azure + key: AZURE_STORAGE_KEY + compression: zst +``` + +When targeting an Azure-compatible endpoint (for example the +[Azurite](https://github.com/Azure/Azurite) emulator), set an explicit +`endpointURL` and use path-style addressing via `uriStyle: path`, since these +endpoints expose the storage account name in the URL path rather than the host: + +```yaml + azureCredentials: + uriStyle: path + account: + name: azure + key: AZURE_STORAGE_ACCOUNT + key: + name: azure + key: AZURE_STORAGE_KEY + endpointURL: azurite:10000 +``` + > [!IMPORTANT] > Unlike Barman, pgBackRest requires object storage to be accessible over HTTPS. While > it's possible to disable key verification and use self-signed keys, using HTTP diff --git a/config/crd/bases/pgbackrest.cnpg.opera.com_archives.yaml b/config/crd/bases/pgbackrest.cnpg.opera.com_archives.yaml index f9b60dd..827451a 100644 --- a/config/crd/bases/pgbackrest.cnpg.opera.com_archives.yaml +++ b/config/crd/bases/pgbackrest.cnpg.opera.com_archives.yaml @@ -143,6 +143,56 @@ spec: repository, including all data needed to properly connect and authenticate with a selected object store. properties: + azureCredentials: + description: The credentials to use to upload data to Azure + Blob Storage + properties: + account: + description: The reference to the secret containing + the storage account name + properties: + key: + description: The key to select + type: string + name: + description: Name of the referent. + type: string + required: + - key + - name + type: object + key: + description: The reference to the secret containing + the account shared key or SAS token + properties: + key: + description: The key to select + type: string + name: + description: Name of the referent. + type: string + required: + - key + - name + type: object + keyType: + default: shared + description: KeyType specifies the type of key used, + either "shared" (default) or "sas" + enum: + - shared + - sas + type: string + uriStyle: + description: |- + Azure Repository URI style, either "host" (default) or "path". + The "path" style is required when targeting Azure-compatible + endpoints such as the Azurite emulator. + enum: + - host + - path + type: string + type: object bucket: minLength: 1 type: string @@ -315,6 +365,9 @@ spec: uriStyle: description: S3 Repository URI style, either "host" (default) or "path". + enum: + - host + - path type: string type: object required: diff --git a/internal/cnpgi/operator/specs/secrets.go b/internal/cnpgi/operator/specs/secrets.go index 15c6ce6..4bb5784 100644 --- a/internal/cnpgi/operator/specs/secrets.go +++ b/internal/cnpgi/operator/specs/secrets.go @@ -25,6 +25,13 @@ import ( // CollectSecretNamesFromCredentials collects the names of the secrets func CollectSecretNamesFromCredentials(pgbackrestCredentials *pgbackrestApi.PgbackrestCredentials) []string { + // A repository with both credential types set is an invalid configuration + // that is rejected upstream when building the pgBackRest command/env vars; + // don't grant RBAC access to secrets from an ambiguous configuration here. + if pgbackrestCredentials.HasConflictingCloudProviders() { + return nil + } + var references []*machineryapi.SecretKeySelector if pgbackrestCredentials.AWS != nil { references = append( @@ -33,6 +40,13 @@ func CollectSecretNamesFromCredentials(pgbackrestCredentials *pgbackrestApi.Pgba pgbackrestCredentials.AWS.SecretAccessKeyReference, ) } + if pgbackrestCredentials.Azure != nil { + references = append( + references, + pgbackrestCredentials.Azure.Account, + pgbackrestCredentials.Azure.Key, + ) + } result := make([]string, 0, len(references)) for _, reference := range references { diff --git a/internal/pgbackrest/api/config.go b/internal/pgbackrest/api/config.go index e56423f..ff96527 100644 --- a/internal/pgbackrest/api/config.go +++ b/internal/pgbackrest/api/config.go @@ -97,8 +97,43 @@ type S3Credentials struct { Region string `json:"region,omitempty"` // S3 Repository URI style, either "host" (default) or "path". - // TODO: Enforce values via Enum like iin compression. // +optional + // +kubebuilder:validation:Enum=host;path + URIStyle string `json:"uriStyle,omitempty"` +} + +// AzureKeyType is the type of key used for Azure credentials +type AzureKeyType string + +const ( + // AzureKeyTypeShared uses a storage account shared key + AzureKeyTypeShared = AzureKeyType("shared") + // AzureKeyTypeSAS uses a shared access signature token + AzureKeyTypeSAS = AzureKeyType("sas") +) + +// AzureCredentials is the type for the credentials to be used to upload +// files to Azure Blob Storage. +type AzureCredentials struct { + // KeyType specifies the type of key used, either "shared" (default) or "sas" + // +optional + // +kubebuilder:default:=shared + // +kubebuilder:validation:Enum=shared;sas + KeyType AzureKeyType `json:"keyType,omitempty"` + + // The reference to the secret containing the storage account name + // +optional + Account *machineryapi.SecretKeySelector `json:"account,omitempty"` + + // The reference to the secret containing the account shared key or SAS token + // +optional + Key *machineryapi.SecretKeySelector `json:"key,omitempty"` + + // Azure Repository URI style, either "host" (default) or "path". + // The "path" style is required when targeting Azure-compatible + // endpoints such as the Azurite emulator. + // +optional + // +kubebuilder:validation:Enum=host;path URIStyle string `json:"uriStyle,omitempty"` } @@ -107,6 +142,17 @@ type PgbackrestCredentials struct { // The credentials to use to upload data to S3 // +optional AWS *S3Credentials `json:"s3Credentials,omitempty"` + + // The credentials to use to upload data to Azure Blob Storage + // +optional + Azure *AzureCredentials `json:"azureCredentials,omitempty"` +} + +// HasConflictingCloudProviders reports whether more than one cloud provider +// credential block is configured. A single pgBackRest repository can only +// target one storage type, so having both set is an invalid configuration. +func (c PgbackrestCredentials) HasConflictingCloudProviders() bool { + return c.AWS != nil && c.Azure != nil } // PgbackrestRetention an object containing the backup retention time for all backup @@ -422,8 +468,8 @@ func (c *PgbackrestConfiguration) ShouldCreateStanzaOnBackup() bool { // ArePopulated checks if the passed set of credentials contains // something -func (credentials PgbackrestCredentials) ArePopulated() bool { - return credentials.AWS != nil +func (c PgbackrestCredentials) ArePopulated() bool { + return c.AWS != nil || c.Azure != nil } // AppendAdditionalRestoreCommandArgs adds custom arguments as pgbackrest restore command-line options diff --git a/internal/pgbackrest/api/config_test.go b/internal/pgbackrest/api/config_test.go index cc13014..9be7759 100644 --- a/internal/pgbackrest/api/config_test.go +++ b/internal/pgbackrest/api/config_test.go @@ -157,6 +157,12 @@ var _ = Describe("Pgbackrest credentials", func() { AWS: &S3Credentials{}, }.ArePopulated()).To(BeTrue()) }) + + It("can check when Azure credentials are set", func() { + Expect(PgbackrestCredentials{ + Azure: &AzureCredentials{}, + }.ArePopulated()).To(BeTrue()) + }) }) var _ = Describe("Pgbackrest retention", func() { diff --git a/internal/pgbackrest/api/zz_generated.deepcopy.go b/internal/pgbackrest/api/zz_generated.deepcopy.go index 7866706..ce208e4 100644 --- a/internal/pgbackrest/api/zz_generated.deepcopy.go +++ b/internal/pgbackrest/api/zz_generated.deepcopy.go @@ -24,6 +24,31 @@ import ( pkgapi "github.com/cloudnative-pg/machinery/pkg/api" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureCredentials) DeepCopyInto(out *AzureCredentials) { + *out = *in + if in.Account != nil { + in, out := &in.Account, &out.Account + *out = new(pkgapi.SecretKeySelector) + **out = **in + } + if in.Key != nil { + in, out := &in.Key, &out.Key + *out = new(pkgapi.SecretKeySelector) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureCredentials. +func (in *AzureCredentials) DeepCopy() *AzureCredentials { + if in == nil { + return nil + } + out := new(AzureCredentials) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataBackupConfiguration) DeepCopyInto(out *DataBackupConfiguration) { *out = *in @@ -146,6 +171,11 @@ func (in *PgbackrestCredentials) DeepCopyInto(out *PgbackrestCredentials) { *out = new(S3Credentials) (*in).DeepCopyInto(*out) } + if in.Azure != nil { + in, out := &in.Azure, &out.Azure + *out = new(AzureCredentials) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PgbackrestCredentials. diff --git a/internal/pgbackrest/command/commandbuilder.go b/internal/pgbackrest/command/commandbuilder.go index e4402f6..8fd3bb9 100644 --- a/internal/pgbackrest/command/commandbuilder.go +++ b/internal/pgbackrest/command/commandbuilder.go @@ -161,7 +161,20 @@ func appendCloudProviderOptions( options []string, repoIndex int, repository pgbackrestApi.PgbackrestRepository, -) ([]string, error) { // nolint: unparam +) ([]string, error) { + // A single pgBackRest repository can only target one storage type. If both + // credential blocks are set the configuration is ambiguous, so fail fast + // instead of silently letting one provider win. + if repository.HasConflictingCloudProviders() { + return nil, fmt.Errorf( + "repository \"repo%d\" has both s3Credentials and azureCredentials set; "+ + "exactly one storage type must be configured per repository", + repoIndex+1, + ) + } + if repository.Azure != nil { + return appendAzureOptions(options, repoIndex, repository), nil + } options = append( options, utils.FormatRepoFlag(repoIndex, "type"), @@ -192,6 +205,38 @@ func appendCloudProviderOptions( return options, nil } +// appendAzureOptions adds the pgbackrest options required to use an Azure Blob Storage repository +func appendAzureOptions( + options []string, + repoIndex int, + repository pgbackrestApi.PgbackrestRepository, +) []string { + options = append( + options, + utils.FormatRepoFlag(repoIndex, "type"), + "azure") + // The azure-endpoint override is intentionally not passed on the command line: + // pgBackRest rejects "repoN-azure-endpoint" as a CLI option (it could expose + // secrets in the process list). It is provided via the PGBACKREST_REPON_AZURE_ENDPOINT + // environment variable instead (see the credentials package). + if repository.DisableVerifyTLS { + options = append( + options, + utils.FormatRepoFlag(repoIndex, "storage-verify-tls=n")) + } + options = append(options, + utils.FormatRepoFlag(repoIndex, "azure-container"), repository.Bucket, + utils.FormatRepoFlag(repoIndex, "path"), repository.DestinationPath, + ) + if repository.Azure != nil && len(repository.Azure.URIStyle) > 0 { + options = append( + options, + utils.FormatRepoFlag(repoIndex, "azure-uri-style"), + repository.Azure.URIStyle) + } + return options +} + // AppendStanzaOptionsFromConfiguration takes an options array and adds the necessary // stanza-specific options required for all operations connecting to the database func AppendStanzaOptionsFromConfiguration( diff --git a/internal/pgbackrest/command/commandbuilder_test.go b/internal/pgbackrest/command/commandbuilder_test.go index 3928d4b..0b6ff3e 100644 --- a/internal/pgbackrest/command/commandbuilder_test.go +++ b/internal/pgbackrest/command/commandbuilder_test.go @@ -136,6 +136,59 @@ var _ = Describe("appendLogOptions", func() { ) }) +var _ = Describe("pgbackrestWalRestoreOptions with Azure repository", func() { + var storageConf *pgbackrestApi.PgbackrestConfiguration + BeforeEach(func() { + storageConf = &pgbackrestApi.PgbackrestConfiguration{ + Repositories: []pgbackrestApi.PgbackrestRepository{ + { + PgbackrestCredentials: pgbackrestApi.PgbackrestCredentials{ + Azure: &pgbackrestApi.AzureCredentials{}, + }, + Bucket: "container-name", + DestinationPath: "/", + }, + }, + } + }) + + It("should generate correct arguments for an Azure repository", func(ctx SpecContext) { + options, err := CloudWalRestoreOptions(ctx, storageConf, "test-cluster", "/var/lib/postgres/pgdata") + Expect(err).ToNot(HaveOccurred()) + Expect(strings.Join(options, " ")). + To( + Equal( + "--repo1-type azure --repo1-azure-container container-name --repo1-path / " + + "--pg1-path /var/lib/postgres/pgdata --log-level-stderr warn --log-level-console off --stanza test-cluster", + )) + }) + + It("should generate correct arguments for an Azure-compatible endpoint", func(ctx SpecContext) { + storageConf.Repositories[0].EndpointURL = "azurite:10000" + storageConf.Repositories[0].DisableVerifyTLS = true + storageConf.Repositories[0].Azure.URIStyle = "path" + options, err := CloudWalRestoreOptions(ctx, storageConf, "test-cluster", "/var/lib/postgres/pgdata") + Expect(err).ToNot(HaveOccurred()) + // The azure-endpoint override is not emitted on the command line; pgBackRest + // rejects it there and it is passed via PGBACKREST_REPO1_AZURE_ENDPOINT instead. + Expect(strings.Join(options, " ")). + To( + Equal( + "--repo1-type azure --repo1-storage-verify-tls=n " + + "--repo1-azure-container container-name --repo1-path / --repo1-azure-uri-style path " + + "--pg1-path /var/lib/postgres/pgdata --log-level-stderr warn --log-level-console off --stanza test-cluster", + )) + Expect(options).ToNot(ContainElement(ContainSubstring("azure-endpoint"))) + }) + + It("should fail fast when both s3 and azure credentials are set", func(ctx SpecContext) { + storageConf.Repositories[0].AWS = &pgbackrestApi.S3Credentials{} + _, err := CloudWalRestoreOptions(ctx, storageConf, "test-cluster", "/var/lib/postgres/pgdata") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("both s3Credentials and azureCredentials set")) + }) +}) + var _ = Describe("PgbackrestRetention", func() { var config *pgbackrestApi.PgbackrestConfiguration var history int32 = 8 diff --git a/internal/pgbackrest/credentials/credentials.go b/internal/pgbackrest/credentials/credentials.go index feb0917..294aeda 100644 --- a/internal/pgbackrest/credentials/credentials.go +++ b/internal/pgbackrest/credentials/credentials.go @@ -107,12 +107,28 @@ func envSetCloudCredentials( env []string, ) (envs []string, err error) { for index, repo := range configuration.Repositories { + // A single pgBackRest repository can only target one storage type. + // Reject configurations that set multiple credential blocks so we + // don't inject conflicting env vars for the same repo. + if repo.HasConflictingCloudProviders() { + return nil, fmt.Errorf( + "repository \"repo%d\" has both s3Credentials and azureCredentials set; "+ + "exactly one storage type must be configured per repository", + index+1, + ) + } if repo.AWS != nil { env, err = envSetAWSCredentials(ctx, c, namespace, repo.AWS, index, env) if err != nil { return nil, err } } + if repo.Azure != nil { + env, err = envSetAzureCredentials(ctx, c, namespace, repo.Azure, repo.EndpointURL, index, env) + if err != nil { + return nil, err + } + } if len(repo.Encryption) != 0 { env, err = envSetEncryptionCredentials(ctx, c, repo.Encryption, repo.EncryptionKey, namespace, index, env) if err != nil { @@ -178,6 +194,48 @@ func envSetAWSCredentials( return env, nil } +// envSetAzureCredentials sets the Azure environment variables given the configuration +// inside the cluster +func envSetAzureCredentials( + ctx context.Context, + client client.Client, + namespace string, + azureCredentials *pgbackrestApi.AzureCredentials, + endpointURL string, + repoIndex int, + env []string, +) ([]string, error) { + if azureCredentials.Account == nil { + return nil, fmt.Errorf("missing Azure storage account") + } + account, err := extractValueFromSecret(ctx, client, azureCredentials.Account, namespace) + if err != nil { + return nil, err + } + + if azureCredentials.Key == nil { + return nil, fmt.Errorf("missing Azure account key") + } + key, err := extractValueFromSecret(ctx, client, azureCredentials.Key, namespace) + if err != nil { + return nil, err + } + + env = append(env, utils.FormatRepoEnv(repoIndex, "AZURE_ACCOUNT", string(account))) + env = append(env, utils.FormatRepoEnv(repoIndex, "AZURE_KEY", string(key))) + env = append(env, utils.FormatRepoEnv(repoIndex, "AZURE_KEY_TYPE", string(azureCredentials.KeyType))) + + // The azure-endpoint override cannot be passed on the command line (pgBackRest + // rejects it to avoid exposing secrets in the process list), so it is provided + // as an environment variable. Only set when an endpoint override is configured; + // real Azure Blob Storage relies on endpoint auto-discovery. + if len(endpointURL) > 0 { + env = append(env, utils.FormatRepoEnv(repoIndex, "AZURE_ENDPOINT", endpointURL)) + } + + return env, nil +} + // envSetEncryptionCredentials sets the pgbackrest encryption environment variables given // the configuration inside the cluster func envSetEncryptionCredentials( diff --git a/internal/pgbackrest/credentials/credentials_test.go b/internal/pgbackrest/credentials/credentials_test.go new file mode 100644 index 0000000..4ccf755 --- /dev/null +++ b/internal/pgbackrest/credentials/credentials_test.go @@ -0,0 +1,138 @@ +/* +Copyright The CloudNativePG Contributors +Copyright 2025, Opera Norway AS + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package credentials + +import ( + machineryapi "github.com/cloudnative-pg/machinery/pkg/api" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + pgbackrestApi "github.com/operasoftware/cnpg-plugin-pgbackrest/internal/pgbackrest/api" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("envSetAzureCredentials", func() { + const namespace = "default" + + var ( + cl client.Client + credentials *pgbackrestApi.AzureCredentials + ) + + buildClient := func(objects ...client.Object) client.Client { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + } + + BeforeEach(func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "azure-secret", + }, + Data: map[string][]byte{ + "AZURE_STORAGE_ACCOUNT": []byte("storageaccountname"), + "AZURE_STORAGE_KEY": []byte("c3RvcmFnZWFjY291bnRrZXk="), + }, + } + cl = buildClient(secret) + credentials = &pgbackrestApi.AzureCredentials{ + KeyType: pgbackrestApi.AzureKeyTypeShared, + Account: &machineryapi.SecretKeySelector{ + LocalObjectReference: machineryapi.LocalObjectReference{Name: "azure-secret"}, + Key: "AZURE_STORAGE_ACCOUNT", + }, + Key: &machineryapi.SecretKeySelector{ + LocalObjectReference: machineryapi.LocalObjectReference{Name: "azure-secret"}, + Key: "AZURE_STORAGE_KEY", + }, + } + }) + + It("exports the Azure environment variables from the referenced secret", func(ctx SpecContext) { + env, err := envSetAzureCredentials(ctx, cl, namespace, credentials, "", 0, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(env).To(ConsistOf( + "PGBACKREST_REPO1_AZURE_ACCOUNT=storageaccountname", + "PGBACKREST_REPO1_AZURE_KEY=c3RvcmFnZWFjY291bnRrZXk=", + "PGBACKREST_REPO1_AZURE_KEY_TYPE=shared", + )) + }) + + It("exports the endpoint override as an environment variable when configured", func(ctx SpecContext) { + env, err := envSetAzureCredentials(ctx, cl, namespace, credentials, "azurite:10000", 0, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(env).To(ContainElement("PGBACKREST_REPO1_AZURE_ENDPOINT=azurite:10000")) + }) + + It("does not export an endpoint variable when no override is configured", func(ctx SpecContext) { + env, err := envSetAzureCredentials(ctx, cl, namespace, credentials, "", 0, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(env).ToNot(ContainElement(ContainSubstring("AZURE_ENDPOINT"))) + }) + + It("fails when the storage account reference is missing", func(ctx SpecContext) { + credentials.Account = nil + _, err := envSetAzureCredentials(ctx, cl, namespace, credentials, "", 0, nil) + Expect(err).To(MatchError(ContainSubstring("missing Azure storage account"))) + }) + + It("fails when the account key reference is missing", func(ctx SpecContext) { + credentials.Key = nil + _, err := envSetAzureCredentials(ctx, cl, namespace, credentials, "", 0, nil) + Expect(err).To(MatchError(ContainSubstring("missing Azure account key"))) + }) + + It("fails when the referenced secret key does not exist", func(ctx SpecContext) { + credentials.Key.Key = "MISSING_KEY" + _, err := envSetAzureCredentials(ctx, cl, namespace, credentials, "", 0, nil) + Expect(err).To(MatchError(ContainSubstring("missing key MISSING_KEY"))) + }) +}) + +var _ = Describe("envSetCloudCredentials", func() { + const namespace = "default" + + buildClient := func(objects ...client.Object) client.Client { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + } + + It("fails fast when a repository sets both s3 and azure credentials", func(ctx SpecContext) { + cl := buildClient() + configuration := &pgbackrestApi.PgbackrestConfiguration{ + Repositories: []pgbackrestApi.PgbackrestRepository{ + { + PgbackrestCredentials: pgbackrestApi.PgbackrestCredentials{ + AWS: &pgbackrestApi.S3Credentials{}, + Azure: &pgbackrestApi.AzureCredentials{}, + }, + }, + }, + } + _, err := envSetCloudCredentials(ctx, cl, namespace, configuration, nil) + Expect(err).To(MatchError(ContainSubstring("both s3Credentials and azureCredentials set"))) + }) +}) diff --git a/internal/pgbackrest/credentials/suite_test.go b/internal/pgbackrest/credentials/suite_test.go new file mode 100644 index 0000000..6991d27 --- /dev/null +++ b/internal/pgbackrest/credentials/suite_test.go @@ -0,0 +1,30 @@ +/* +Copyright The CloudNativePG Contributors +Copyright 2025, Opera Norway AS + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package credentials + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCredentials(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Credentials Suite") +} diff --git a/manifest.yaml b/manifest.yaml index 7ad05bb..1b01826 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -142,6 +142,56 @@ spec: repository, including all data needed to properly connect and authenticate with a selected object store. properties: + azureCredentials: + description: The credentials to use to upload data to Azure + Blob Storage + properties: + account: + description: The reference to the secret containing + the storage account name + properties: + key: + description: The key to select + type: string + name: + description: Name of the referent. + type: string + required: + - key + - name + type: object + key: + description: The reference to the secret containing + the account shared key or SAS token + properties: + key: + description: The key to select + type: string + name: + description: Name of the referent. + type: string + required: + - key + - name + type: object + keyType: + default: shared + description: KeyType specifies the type of key used, + either "shared" (default) or "sas" + enum: + - shared + - sas + type: string + uriStyle: + description: |- + Azure Repository URI style, either "host" (default) or "path". + The "path" style is required when targeting Azure-compatible + endpoints such as the Azurite emulator. + enum: + - host + - path + type: string + type: object bucket: minLength: 1 type: string @@ -314,6 +364,9 @@ spec: uriStyle: description: S3 Repository URI style, either "host" (default) or "path". + enum: + - host + - path type: string type: object required: diff --git a/test/e2e/internal/objectstore/azurite.go b/test/e2e/internal/objectstore/azurite.go new file mode 100644 index 0000000..718ae64 --- /dev/null +++ b/test/e2e/internal/objectstore/azurite.go @@ -0,0 +1,362 @@ +/* +Copyright 2024, The CloudNativePG Contributors +Copyright 2025, Opera Norway AS + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "fmt" + "net" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + "github.com/cloudnative-pg/machinery/pkg/api" + pluginPgbackrestV1 "github.com/operasoftware/cnpg-plugin-pgbackrest/api/v1" + pgbackrestApi "github.com/operasoftware/cnpg-plugin-pgbackrest/internal/pgbackrest/api" +) + +const ( + // azuriteAccount is the Azurite storage account name used for testing. + azuriteAccount = "storageaccountname" + // azuriteKey is the Azurite storage account key (base64 encoded) used for testing. + azuriteKey = "c3RvcmFnZWFjY291bnRrZXk=" + // azuriteContainer is the Azure Blob container (pgBackRest bucket) created for backups. + azuriteContainer = "backups" + + // AzuriteAccountKey is the secret key holding the Azure storage account name. + AzuriteAccountKey = "AZURE_STORAGE_ACCOUNT" + // AzuriteKeyKey is the secret key holding the Azure storage account key. + AzuriteKeyKey = "AZURE_STORAGE_KEY" +) + +// NewAzuriteObjectStoreResources creates the resources required to create an Azurite object store. +func NewAzuriteObjectStoreResources(namespace, name string) *Resources { + return &Resources{ + Deployment: newAzuriteDeployment(namespace, name), + ProvisioningJob: newAzuriteProvisioningJob(namespace, name), + Service: newAzuriteService(namespace, name), + PVC: newAzuritePVC(namespace, name), + Secret: newAzuriteSecret(namespace, name), + } +} + +func newAzuriteDeployment(namespace, name string) *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": name, + }, + }, + Spec: corev1.PodSpec{ + // Pgbackrest only allows HTTPS connections to object store endpoints, + // so Azurite must be started with a self-signed certificate. + InitContainers: []corev1.Container{ + { + Name: "generate-certs", + Image: "alpine/openssl:latest", + Args: []string{ + "req", + "-x509", + "-newkey", + "rsa:4096", + "-keyout", + "/certs/private.key", + "-out", + "/certs/public.crt", + "-sha256", + "-days", + "3650", + "-nodes", + "-subj", + fmt.Sprintf("/CN=%s", name)}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "certs", + MountPath: "/certs", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: name, + // TODO: renovate the image + Image: "mcr.microsoft.com/azure-storage/azurite:latest", + Args: []string{ + "azurite-blob", + "--blobHost", + "0.0.0.0", + "--location", + "/data", + "--skipApiVersionCheck", + "--disableProductStyleUrl", + "--cert", + "/certs/public.crt", + "--key", + "/certs/private.key", + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: 10000, + Name: name, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "AZURITE_ACCOUNTS", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: name, + }, + Key: "AZURITE_ACCOUNTS", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/data", + }, + { + Name: "certs", + MountPath: "/certs", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: name, + }, + }, + }, + { + Name: "certs", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + }, + } +} + +func newAzuriteProvisioningJob(namespace, name string) *batchv1.Job { + // Pgbackrest requires the Azure Blob container to exist but Azurite doesn't + // provision it automatically, so we create it with the Azure CLI. + // The connection string points at the self-signed HTTPS endpoint, hence + // TLS verification is disabled for the CLI as well. + connectionString := fmt.Sprintf( + "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;BlobEndpoint=https://%s:10000/%s;", + azuriteAccount, azuriteKey, name, azuriteAccount, + ) + return &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "batch/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name + "-provisioning", + Namespace: namespace, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: ptr.To(int32(10)), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": name, + }, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + Containers: []corev1.Container{ + { + Name: name + "-provisioner", + Image: "mcr.microsoft.com/azure-cli:latest", + Command: []string{"bash"}, + Args: []string{ + "-c", + "az storage container create --name " + azuriteContainer + + " --connection-string \"$AZURE_STORAGE_CONNECTION_STRING\"", + }, + TerminationMessagePolicy: "FallbackToLogsOnError", + Env: []corev1.EnvVar{ + { + Name: "AZURE_STORAGE_CONNECTION_STRING", + Value: connectionString, + }, + { + Name: "AZURE_CLI_DISABLE_CONNECTION_VERIFICATION", + Value: "1", + }, + { + Name: "ADAL_PYTHON_SSL_NO_VERIFY", + Value: "1", + }, + }, + }, + }, + }, + }, + }, + } +} + +func newAzuriteService(namespace, name string) *corev1.Service { + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": name, + }, + Ports: []corev1.ServicePort{ + { + Port: 10000, + TargetPort: intstr.FromInt32(10000), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func newAzuriteSecret(namespace, name string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string][]byte{ + "AZURITE_ACCOUNTS": []byte(fmt.Sprintf("%s:%s", azuriteAccount, azuriteKey)), + AzuriteAccountKey: []byte(azuriteAccount), + AzuriteKeyKey: []byte(azuriteKey), + }, + } +} + +func newAzuritePVC(namespace, name string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(DefaultSize), + }, + }, + }, + } +} + +// NewAzuriteArchive creates a new Archive configured to use the Azurite object store. +func NewAzuriteArchive(namespace, name, azuriteOSName string, maxParallel int) *pluginPgbackrestV1.Archive { + return &pluginPgbackrestV1.Archive{ + TypeMeta: metav1.TypeMeta{ + Kind: "Archive", + APIVersion: "pgbackrest.cnpg.opera.com/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: pluginPgbackrestV1.ArchiveSpec{ + Configuration: pgbackrestApi.PgbackrestConfiguration{ + Wal: &pgbackrestApi.WalBackupConfiguration{ + MaxParallel: maxParallel, + }, + Repositories: []pgbackrestApi.PgbackrestRepository{ + { + PgbackrestCredentials: pgbackrestApi.PgbackrestCredentials{ + Azure: &pgbackrestApi.AzureCredentials{ + KeyType: pgbackrestApi.AzureKeyTypeShared, + Account: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: azuriteOSName, + }, + Key: AzuriteAccountKey, + }, + Key: &api.SecretKeySelector{ + LocalObjectReference: api.LocalObjectReference{ + Name: azuriteOSName, + }, + Key: AzuriteKeyKey, + }, + // Azurite exposes the account name in the URL path, + // while pgBackRest defaults to host-style addressing. + URIStyle: "path", + }, + }, + EndpointURL: net.JoinHostPort(azuriteOSName, "10000"), + // Pgbackrest enforces HTTPS connections and there is only + // a self-signed certificate available. + DisableVerifyTLS: true, + DestinationPath: "/", + Bucket: azuriteContainer, + }, + }, + }, + }, + } +} diff --git a/test/e2e/internal/objectstore/objectstore.go b/test/e2e/internal/objectstore/objectstore.go index f69d2ea..53eac53 100644 --- a/test/e2e/internal/objectstore/objectstore.go +++ b/test/e2e/internal/objectstore/objectstore.go @@ -20,10 +20,13 @@ package objectstore import ( "context" "fmt" + "time" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -63,11 +66,53 @@ func (osr Resources) Create(ctx context.Context, cl client.Client) error { return fmt.Errorf("failed to create service: %w", err) } } + if osr.Deployment != nil { + if err := waitForDeploymentAvailable(ctx, cl, osr.Deployment); err != nil { + return fmt.Errorf("failed waiting for deployment: %w", err) + } + } if osr.ProvisioningJob != nil { if err := cl.Create(ctx, osr.ProvisioningJob); err != nil { return fmt.Errorf("failed to create provisioning job: %w", err) } + if err := waitForJobComplete(ctx, cl, osr.ProvisioningJob); err != nil { + return fmt.Errorf("failed waiting for provisioning job: %w", err) + } } return nil } + +func waitForDeploymentAvailable(ctx context.Context, cl client.Client, deployment *appsv1.Deployment) error { + key := types.NamespacedName{Name: deployment.Name, Namespace: deployment.Namespace} + return wait.PollUntilContextTimeout(ctx, 2*time.Second, 10*time.Minute, true, + func(ctx context.Context) (bool, error) { + current := &appsv1.Deployment{} + if err := cl.Get(ctx, key, current); err != nil { + return false, err + } + + return current.Status.AvailableReplicas >= 1, nil + }) +} + +func waitForJobComplete(ctx context.Context, cl client.Client, job *batchv1.Job) error { + key := types.NamespacedName{Name: job.Name, Namespace: job.Namespace} + return wait.PollUntilContextTimeout(ctx, 2*time.Second, 5*time.Minute, true, + func(ctx context.Context) (bool, error) { + current := &batchv1.Job{} + if err := cl.Get(ctx, key, current); err != nil { + return false, err + } + for _, condition := range current.Status.Conditions { + if condition.Type == batchv1.JobFailed && condition.Status == corev1.ConditionTrue { + return false, fmt.Errorf("job %s failed", key) + } + if condition.Type == batchv1.JobComplete && condition.Status == corev1.ConditionTrue { + return true, nil + } + } + + return false, nil + }) +} diff --git a/test/e2e/internal/tests/backup/backup_restore.go b/test/e2e/internal/tests/backup/backup_restore.go index 37c9ce5..4ed8be0 100644 --- a/test/e2e/internal/tests/backup/backup_restore.go +++ b/test/e2e/internal/tests/backup/backup_restore.go @@ -177,6 +177,10 @@ var _ = Describe("Backup and restore", func() { "using the plugin for backup and restore on S3", &s3BackupPluginBackupPluginRestore{}, ), + Entry( + "using the plugin for backup and restore on Azure Blob (Azurite)", + &azureBackupPluginBackupPluginRestore{}, + ), ) DescribeTable("should perform point-in-time recovery", @@ -374,5 +378,9 @@ var _ = Describe("Backup and restore", func() { "using TargetTime with plugin", &s3BackupPluginTargetTimeRestore{}, ), + Entry( + "using TargetTime with plugin on Azure Blob (Azurite)", + &azureBackupPluginTargetTimeRestore{}, + ), ) }) diff --git a/test/e2e/internal/tests/backup/fixtures.go b/test/e2e/internal/tests/backup/fixtures.go index d9bdc76..cf4b45f 100644 --- a/test/e2e/internal/tests/backup/fixtures.go +++ b/test/e2e/internal/tests/backup/fixtures.go @@ -27,7 +27,8 @@ import ( ) const ( - minio = "minio" + minio = "minio" + azurite = "azurite" // Size of the PVCs for the object stores and the cluster instances. size = "1Gi" srcClusterName = "source" @@ -77,6 +78,34 @@ func (s s3BackupPluginBackupPluginRestore) createBackupRestoreTestResources( return result } +type azureBackupPluginBackupPluginRestore struct{} + +type azureBackupPluginTargetTimeRestore struct { + azureBackupPluginBackupPluginRestore +} + +func (s azureBackupPluginBackupPluginRestore) createBackupRestoreTestResources( + namespace string, +) backupRestoreTestResources { + result := backupRestoreTestResources{} + + result.ObjectStoreResources = objectstore.NewAzuriteObjectStoreResources(namespace, azurite) + result.Archive = objectstore.NewAzuriteArchive(namespace, archiveName, azurite, 1) + result.SrcCluster = newSrcClusterWithPlugin(namespace) + result.SrcBackup = newSrcPluginBackup(namespace) + result.DstCluster = newDstClusterWithPlugin(namespace) + result.DstBackup = newDstPluginBackup(namespace) + + return result +} + +func (s azureBackupPluginTargetTimeRestore) createPITRCluster( + namespace string, + targetTime string, +) *cloudnativepgv1.Cluster { + return s3BackupPluginTargetTimeRestore{}.createPITRCluster(namespace, targetTime) +} + func (s s3BackupPluginTargetTimeRestore) createPITRCluster( namespace string, targetTime string,