diff --git a/.chloggen/postgresqlreceiver-connect-database.yaml b/.chloggen/postgresqlreceiver-connect-database.yaml new file mode 100644 index 0000000000000..ee10dfa09872e --- /dev/null +++ b/.chloggen/postgresqlreceiver-connect-database.yaml @@ -0,0 +1,11 @@ +change_type: enhancement + +component: receiver/postgresql + +note: Add `connect_database` to configure which database the receiver connects to for cluster-wide statistics collection. + +issues: [50920] + +subtext: + +change_logs: [user] diff --git a/receiver/postgresqlreceiver/README.md b/receiver/postgresqlreceiver/README.md index 123691c5abe0d..c4f41badc9c18 100644 --- a/receiver/postgresqlreceiver/README.md +++ b/receiver/postgresqlreceiver/README.md @@ -97,7 +97,14 @@ The following settings are optional: - `databases` (default = `[]`): The list of databases for which the receiver will attempt to collect statistics. If an empty list is provided, the receiver will attempt to collect statistics for all non-template databases. This list applies to metrics only; the query sample and top query collectors ignore it and are filtered solely by `exclude_databases`. -- `exclude_databases` (default = `[]`): List of databases excluded from statistics, query samples, and top queries. Excluded databases are filtered out of every collection query and the receiver opens no per-database connection to them. Exception: the receiver always connects to the default `postgres` database for discovery and server-level queries, even if it is listed here. +- `exclude_databases` (default = `[]`): List of databases excluded from statistics, query samples, and top queries. Excluded databases are filtered out of every collection query and the receiver opens no per-database connection to them. Exception: the receiver always connects to the configured `connect_database` (default `postgres`) for discovery and server-level queries, even if it is listed here. + +- `connect_database` (default = `postgres`): The database the receiver connects to for discovery and server-level queries, including `pg_stat_statements`. Independent of `databases` — `pg_stat_statements` is tracked cluster-wide, so any database with the extension installed works as the connection target, regardless of which databases are being monitored. Use this if `pg_stat_statements` lives outside `postgres`, or if you connect through a dedicated monitoring-only database: + ```yaml + connect_database: "mon" # extension lives here + databases: + - "landonline" # database being monitored + ``` > [!NOTE] > Managed PostgreSQL services create internal databases that no customer credential can connect to. The receiver discovers them like any other database and logs a connection error on every scrape. If you use one of these services, add its internal databases to `exclude_databases`: diff --git a/receiver/postgresqlreceiver/config.go b/receiver/postgresqlreceiver/config.go index 7d6147b1bdaa3..8f9adf4f8cc83 100644 --- a/receiver/postgresqlreceiver/config.go +++ b/receiver/postgresqlreceiver/config.go @@ -50,18 +50,21 @@ type QuerySampleCollection struct { } type Config struct { - ControllerConfig scraperhelper.ControllerConfig `mapstructure:",squash"` - Username string `mapstructure:"username"` - Password configopaque.String `mapstructure:"password"` - Databases []string `mapstructure:"databases"` - ExcludeDatabases []string `mapstructure:"exclude_databases"` - AddrConfig confignet.AddrConfig `mapstructure:",squash"` // provides Endpoint and Transport - ClientConfig configtls.ClientConfig `mapstructure:"tls,omitempty"` // provides SSL details - ConnectionPool ConnectionPool `mapstructure:"connection_pool,omitempty"` - MetricsBuilderConfig metadata.MetricsBuilderConfig `mapstructure:",squash"` - LogsBuilderConfig metadata.LogsBuilderConfig `mapstructure:",squash"` - QuerySampleCollection QuerySampleCollection `mapstructure:"query_sample_collection,omitempty"` - TopQueryCollection TopQueryCollection `mapstructure:"top_query_collection,omitempty"` + ControllerConfig scraperhelper.ControllerConfig `mapstructure:",squash"` + Username string `mapstructure:"username"` + Password configopaque.String `mapstructure:"password"` + Databases []string `mapstructure:"databases"` + ExcludeDatabases []string `mapstructure:"exclude_databases"` + // ConnectDatabase is the connection target for cluster-wide queries. + // Defaults to "postgres". Independent of Databases (the reporting scope). + ConnectDatabase string `mapstructure:"connect_database,omitempty"` + AddrConfig confignet.AddrConfig `mapstructure:",squash"` // provides Endpoint and Transport + ClientConfig configtls.ClientConfig `mapstructure:"tls,omitempty"` // provides SSL details + ConnectionPool ConnectionPool `mapstructure:"connection_pool,omitempty"` + MetricsBuilderConfig metadata.MetricsBuilderConfig `mapstructure:",squash"` + LogsBuilderConfig metadata.LogsBuilderConfig `mapstructure:",squash"` + QuerySampleCollection QuerySampleCollection `mapstructure:"query_sample_collection,omitempty"` + TopQueryCollection TopQueryCollection `mapstructure:"top_query_collection,omitempty"` // DBAuth optionally sources the connection credential from a db_auth provider // extension (e.g. AWS IAM) instead of a static password. When set, the provider // supplies the password at connection-open time. Mutually exclusive with the diff --git a/receiver/postgresqlreceiver/config.schema.yaml b/receiver/postgresqlreceiver/config.schema.yaml index c59ca22f084a4..80ec7dcb283de 100644 --- a/receiver/postgresqlreceiver/config.schema.yaml +++ b/receiver/postgresqlreceiver/config.schema.yaml @@ -44,6 +44,9 @@ $defs: x-customType: int64 type: object properties: + connect_database: + description: ConnectDatabase is the connection target for cluster-wide queries. Defaults to "postgres". Independent of Databases (the reporting scope). + type: string connection_pool: $ref: connection_pool databases: diff --git a/receiver/postgresqlreceiver/config_test.go b/receiver/postgresqlreceiver/config_test.go index ed9b2e58bfafc..1d671d7764b94 100644 --- a/receiver/postgresqlreceiver/config_test.go +++ b/receiver/postgresqlreceiver/config_test.go @@ -99,6 +99,16 @@ func TestValidate(t *testing.T) { }, expected: nil, }, + { + desc: "connect_database outside databases is not a config error", + defaultConfigModifier: func(cfg *Config) { + cfg.Username = "otel" + cfg.Password = "otel" + cfg.ConnectDatabase = "mon" + cfg.Databases = []string{"landonline"} + }, + expected: nil, + }, { desc: "no error", defaultConfigModifier: func(cfg *Config) { @@ -180,6 +190,7 @@ func TestLoadConfig(t *testing.T) { expected.Password = "${env:POSTGRESQL_PASSWORD}" expected.Databases = []string{"otel"} expected.ExcludeDatabases = []string{"template0"} + expected.ConnectDatabase = "mon" expected.ControllerConfig.CollectionInterval = 10 * time.Second expected.ClientConfig = configtls.ClientConfig{ Insecure: false, diff --git a/receiver/postgresqlreceiver/scraper.go b/receiver/postgresqlreceiver/scraper.go index 2d0b8722fcfb2..b69de43e63b05 100644 --- a/receiver/postgresqlreceiver/scraper.go +++ b/receiver/postgresqlreceiver/scraper.go @@ -207,10 +207,18 @@ type dbRetrieval struct { executionTimeMap map[databaseName]float64 } +// connectDatabase returns ConnectDatabase, or "postgres" when unset. +func (p *postgreSQLScraper) connectDatabase() string { + if p.config.ConnectDatabase != "" { + return p.config.ConnectDatabase + } + return defaultPostgreSQLDatabase +} + // scrape scrapes the metric stats, transforms them and attributes them into a metric slices. func (p *postgreSQLScraper) scrape(ctx context.Context) (pmetric.Metrics, error) { databases := p.config.Databases - listClient, err := p.clientFactory.getClient(ctx, defaultPostgreSQLDatabase) + listClient, err := p.clientFactory.getClient(ctx, p.connectDatabase()) if err != nil { p.logger.Error("Failed to initialize connection to postgres", zap.Error(err)) return pmetric.NewMetrics(), err @@ -280,7 +288,7 @@ func (p *postgreSQLScraper) scrape(ctx context.Context) (pmetric.Metrics, error) } func (p *postgreSQLScraper) scrapeQuerySamples(ctx context.Context, maxRowsPerQuery int64) (plog.Logs, error) { - dbClient, err := p.clientFactory.getClient(ctx, defaultPostgreSQLDatabase) + dbClient, err := p.clientFactory.getClient(ctx, p.connectDatabase()) if err != nil { p.logger.Error("Failed to initialize connection to postgres", zap.Error(err)) return plog.NewLogs(), err @@ -401,7 +409,7 @@ func (p *postgreSQLScraper) collectQuerySamples(ctx context.Context, dbClient cl func (p *postgreSQLScraper) collectTopQuery(ctx context.Context, clientFactory postgreSQLClientFactory, limit, topNQuery, maxExplainEachInterval int64, mux *errsMux, logger *zap.Logger, collectionTime time.Time) { timestamp := pcommon.NewTimestampFromTime(collectionTime) - defaultDbClient, err := clientFactory.getClient(ctx, defaultPostgreSQLDatabase) + defaultDbClient, err := clientFactory.getClient(ctx, p.connectDatabase()) if err != nil { logger.Error("failed to create db client for default postgresql database") mux.addPartial(err) diff --git a/receiver/postgresqlreceiver/scraper_test.go b/receiver/postgresqlreceiver/scraper_test.go index 0fd2c7df98886..d2af6014c4a4f 100644 --- a/receiver/postgresqlreceiver/scraper_test.go +++ b/receiver/postgresqlreceiver/scraper_test.go @@ -1296,6 +1296,83 @@ func TestScrapeTopQueriesHonorsExcludeDatabases(t *testing.T) { } } +func TestConnectDatabase(t *testing.T) { + t.Run("defaults to postgres when unset", func(t *testing.T) { + cfg := createDefaultConfig().(*Config) + scraper, err := newPostgreSQLScraper(receivertest.NewNopSettings(metadata.Type), cfg, mockSimpleClientFactory{}, newCache(1), newTTLCache[string](1, time.Second)) + require.NoError(t, err) + require.Equal(t, defaultPostgreSQLDatabase, scraper.connectDatabase()) + }) + + t.Run("uses the configured value when set", func(t *testing.T) { + cfg := createDefaultConfig().(*Config) + cfg.ConnectDatabase = "mon" + scraper, err := newPostgreSQLScraper(receivertest.NewNopSettings(metadata.Type), cfg, mockSimpleClientFactory{}, newCache(1), newTTLCache[string](1, time.Second)) + require.NoError(t, err) + require.Equal(t, "mon", scraper.connectDatabase()) + }) +} + +func TestScrapeQuerySamplesHonorsConnectDatabase(t *testing.T) { + cfg := createDefaultConfig().(*Config) + cfg.Databases = []string{"landonline"} + cfg.ConnectDatabase = "mon" + cfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled = true + + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + settings := receivertest.NewNopSettings(metadata.Type) + logger, err := zap.NewProduction() + require.NoError(t, err) + settings.TelemetrySettings = component.TelemetrySettings{Logger: logger} + + factory := &recordingClientFactory{mockSimpleClientFactory: mockSimpleClientFactory{db: db}} + scraper, err := newPostgreSQLScraper(settings, cfg, factory, newCache(30), newTTLCache[string](1, time.Second)) + require.NoError(t, err) + + mock.ExpectQuery(".*").WillReturnRows(sqlmock.NewRows(querySampleColumns)) + + _, err = scraper.scrapeQuerySamples(t.Context(), 30) + require.NoError(t, err) + require.NoError(t, mock.ExpectationsWereMet()) + + // The connection target (mon) is independent of the reporting scope + // (landonline) — connecting to mon must not add it to Databases. + require.Equal(t, []string{"mon"}, factory.requestedDatabases) + require.Equal(t, []string{"landonline"}, cfg.Databases) +} + +func TestScrapeTopQueryHonorsConnectDatabase(t *testing.T) { + cfg := createDefaultConfig().(*Config) + cfg.Databases = []string{"landonline"} + cfg.ConnectDatabase = "mon" + cfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled = true + + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) + require.NoError(t, err) + defer db.Close() + + settings := receivertest.NewNopSettings(metadata.Type) + logger, err := zap.NewProduction() + require.NoError(t, err) + settings.TelemetrySettings = component.TelemetrySettings{Logger: logger} + + factory := &recordingClientFactory{mockSimpleClientFactory: mockSimpleClientFactory{db: db}} + scraper, err := newPostgreSQLScraper(settings, cfg, factory, newCache(30), newTTLCache[string](1, time.Second)) + require.NoError(t, err) + + mock.ExpectQuery(".*").WillReturnRows(sqlmock.NewRows(topQueryColumns)) + + _, err = scraper.scrapeTopQuery(t.Context(), 30, 5, 5, time.Minute) + require.NoError(t, err) + require.NoError(t, mock.ExpectationsWereMet()) + + require.Equal(t, []string{"mon"}, factory.requestedDatabases) + require.Equal(t, []string{"landonline"}, cfg.Databases) +} + func TestScrapeQuerySamplesHonorsExcludeDatabases(t *testing.T) { cfg := createDefaultConfig().(*Config) cfg.Databases = []string{} diff --git a/receiver/postgresqlreceiver/testdata/config.yaml b/receiver/postgresqlreceiver/testdata/config.yaml index 10dc2613c6ba9..32a33e14345a6 100644 --- a/receiver/postgresqlreceiver/testdata/config.yaml +++ b/receiver/postgresqlreceiver/testdata/config.yaml @@ -22,6 +22,7 @@ postgresql/all: - otel exclude_databases: - template0 + connect_database: mon collection_interval: 10s tls: insecure: false