diff --git a/input/full.go b/input/full.go index e86369e96..24811f117 100644 --- a/input/full.go +++ b/input/full.go @@ -12,7 +12,6 @@ import ( "github.com/pganalyze/collector/input/postgres" "github.com/pganalyze/collector/input/system" "github.com/pganalyze/collector/logs" - "github.com/pganalyze/collector/scheduler" "github.com/pganalyze/collector/state" "github.com/pganalyze/collector/util" ) @@ -116,7 +115,8 @@ func CollectFull(ctx context.Context, server *state.Server, connection *sql.DB, logger.PrintError("Error setting query text timeout: %s", err) return } - ts.Statements, ts.StatementTexts, err = postgres.GetStatementTexts(ctx, c, connection) + var statementSize int + ts.Statements, ts.StatementTexts, statementSize, err = postgres.GetStatementTexts(ctx, c, connection) if err != nil { // Despite query performance data being an essential part of pganalyze, there are // situations where it may not be available (or it timed out), so treat it as a @@ -151,41 +151,38 @@ func CollectFull(ctx context.Context, server *state.Server, connection *sql.DB, return } - // Reset query stats and texts if needed (this must run after the query text collection) - ps.StatementResetCounter = server.PrevState.StatementResetCounter + 1 - config := server.Grant.Load().Config - if config.Features.StatementResetFrequency != 0 && ps.StatementResetCounter >= int(config.Features.StatementResetFrequency) { - // Block concurrent collection of query stats, as that may see the actual Postgres-side - // reset before we updated the struct that the collector diffs against. + if opts.CollectPostgresSettings { + ts.Settings, err = postgres.GetSettings(ctx, connection) + if err != nil { + logger.PrintError("Error collecting config settings: %s", err) + return + } + } + + shouldReset, err := postgres.ShouldResetStatements(server, &ps, &ts, statementSize) + if err != nil { + logger.PrintError("Failed to determine if reset of pg_stat_statements needed, skipping reset: %s", err) + err = nil + } else if shouldReset { server.HighFreqStateMutex.Lock() - ps.StatementResetCounter = 0 err = postgres.ResetStatements(ctx, c, connection) if err != nil { - logger.PrintError("Error calling pg_stat_statements_reset() as requested: %s", err) + logger.PrintError("Error calling pg_stat_statements_reset(): %s", err) err = nil } else { - logger.PrintInfo("Successfully called pg_stat_statements_reset() for all queries, next reset in %d hours", config.Features.StatementResetFrequency/scheduler.FullSnapshotsPerHour) - - // Make sure the next high frequency run has an empty reference point - newHighFreqState.LastStatementStatsAt = time.Now() - resetStatementStats, err := postgres.GetStatementStats(ctx, c, connection) - if err != nil { - logger.PrintError("Error collecting pg_stat_statements after reset: %s", err) - err = nil - newHighFreqState.StatementStats = make(state.PostgresStatementStatsMap) - } else { - newHighFreqState.StatementStats = resetStatementStats - } + logger.PrintInfo("Successfully called pg_stat_statements_reset") } - server.HighFreqStateMutex.Unlock() - } - - if opts.CollectPostgresSettings { - ts.Settings, err = postgres.GetSettings(ctx, connection) + // Make sure the next high frequency run has an empty reference point + newHighFreqState.LastStatementStatsAt = time.Now() + resetStatementStats, err := postgres.GetStatementStats(ctx, c, connection) if err != nil { - logger.PrintError("Error collecting config settings: %s", err) - return + logger.PrintError("Error collecting pg_stat_statements after reset: %s", err) + err = nil + newHighFreqState.StatementStats = make(state.PostgresStatementStatsMap) + } else { + newHighFreqState.StatementStats = resetStatementStats } + server.HighFreqStateMutex.Unlock() } // CollectAllSchemas relies on GetBufferCache to access the filenode OIDs before that data is discarded diff --git a/input/postgres/statements.go b/input/postgres/statements.go index 84cd9a9f2..17cd1894b 100644 --- a/input/postgres/statements.go +++ b/input/postgres/statements.go @@ -3,12 +3,16 @@ package postgres import ( "context" "database/sql" + "errors" "fmt" "io" "os" + "strconv" "strings" + "time" "github.com/guregu/null" + "github.com/pganalyze/collector/scheduler" "github.com/pganalyze/collector/selftest" "github.com/pganalyze/collector/state" "github.com/pganalyze/collector/util" @@ -59,7 +63,41 @@ func insufficientPrivilege(query string) bool { return query == "" } -func ResetStatements(ctx context.Context, c *Collection, db *sql.DB) error { +const resetThreshold = 0.9 + +func ShouldResetStatements(server *state.Server, ps *state.PersistedState, ts *state.TransientState, size int) (reset bool, err error) { + config := server.Grant.Load().Config + lastReset := ps.PgStatStatementsStats.Reset + resetFreq := config.Features.StatementResetFrequency * scheduler.FullSnapshotMinutes + maxSize := int(config.Features.StatementMaxSizeMb) + if !lastReset.Valid { + return // It's always set on PG14+ with the extension enabled. Older versions aren't supported + } + if maxSize == 0 { + maxSize = 250 + } + entryCount := len(ts.Statements) + entryMax := 0 + for _, setting := range ts.Settings { + if setting.Name == "pg_stat_statements.max" && setting.CurrentValue.Valid { + entryMax, err = strconv.Atoi(setting.CurrentValue.String) + if err != nil { + return + } + } + } + if entryMax == 0 { + err = errors.New("Could not find pg_stat_statements.max setting") + return + } + resetAllowed := resetFreq > 0 && time.Since(lastReset.Time).Minutes() >= float64(resetFreq) + tooMany := float64(entryCount) >= float64(entryMax)*resetThreshold + tooLarge := size > maxSize*1024*1024 + reset = resetAllowed && (tooMany || tooLarge) + return +} + +func ResetStatements(ctx context.Context, c *Collection, db *sql.DB) (err error) { var method string if c.HelperExists("reset_stat_statements", nil) { c.Logger.PrintVerbose("Found pganalyze.reset_stat_statements() stats helper") @@ -71,11 +109,8 @@ func ResetStatements(ctx context.Context, c *Collection, db *sql.DB) error { } method = "pg_stat_statements_reset()" } - _, err := db.ExecContext(ctx, QueryMarkerSQL+"SELECT "+method) - if err != nil { - return err - } - return nil + _, err = db.ExecContext(ctx, QueryMarkerSQL+"SELECT "+method) + return } func GetStatementStats(ctx context.Context, c *Collection, db *sql.DB) (state.PostgresStatementStatsMap, error) { @@ -152,10 +187,10 @@ func GetStatementStats(ctx context.Context, c *Collection, db *sql.DB) (state.Po return statementStats, nil } -func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.PostgresStatementMap, state.PostgresStatementTextMap, error) { +func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (statements state.PostgresStatementMap, statementTextsByFp state.PostgresStatementTextMap, querySize int, err error) { sourceTable, foundExtMinorVersion, err := getStatementSource(ctx, c, db, true) if err != nil { - return nil, nil, err + return } topLevelField := statementSQLTopLevelFieldDefault @@ -166,13 +201,13 @@ func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.Po querySql := QueryMarkerSQL + fmt.Sprintf(statementTextSQL, topLevelField, sourceTable) stmt, err := db.PrepareContext(ctx, querySql) if err != nil { - return nil, nil, err + return } defer stmt.Close() rows, err := stmt.QueryContext(ctx) if err != nil { - return nil, nil, err + return } defer rows.Close() @@ -180,14 +215,13 @@ func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.Po tmpFile, err = os.CreateTemp("", util.TempFilePrefix) if err != nil { - return nil, nil, err + return } defer tmpFile.Close() defer os.Remove(tmpFile.Name()) - statements := make(state.PostgresStatementMap) - statementTextsByFp := make(state.PostgresStatementTextMap) - + statements = make(state.PostgresStatementMap) + statementTextsByFp = make(state.PostgresStatementTextMap) queryKeys := make([]state.PostgresStatementKey, 0) queryLengths := make([]int, 0) @@ -198,8 +232,9 @@ func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.Po err = rows.Scan(&key.DatabaseOid, &key.UserOid, &queryID, &key.TopLevel, &receivedQuery) if err != nil { - return nil, nil, err + return } + querySize += len(receivedQuery.String) if queryID.Valid { key.QueryID = queryID.Int64 @@ -214,7 +249,7 @@ func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.Po } if err = rows.Err(); err != nil { - return nil, nil, err + return } tmpFile.Seek(0, io.SeekStart) @@ -222,7 +257,7 @@ func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.Po bytes := make([]byte, length) _, err = io.ReadFull(tmpFile, bytes) if err != nil { - return nil, nil, err + return } query := string(bytes) ignoreIoTiming := ignoreIOTiming(c.PostgresVersion, query) @@ -230,7 +265,8 @@ func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.Po select { // Since normalizing can take time, explicitly check for cancellations case <-ctx.Done(): - return nil, nil, ctx.Err() + err = ctx.Err() + return default: fingerprintAndNormalize(c, key, key.QueryID, query, statements, statementTextsByFp, ignoreIoTiming) } @@ -238,7 +274,7 @@ func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.Po c.SelfTest.MarkCollectionAspectOk(state.CollectionAspectPgStatStatements) - return statements, statementTextsByFp, nil + return } func getStatementSource(ctx context.Context, c *Collection, db *sql.DB, showtext bool) (string, int16, error) { diff --git a/output/full.go b/output/full.go index fee40e813..18a124dc6 100644 --- a/output/full.go +++ b/output/full.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" snapshot "github.com/pganalyze/collector/output/pganalyze_collector" "github.com/pganalyze/collector/output/transform" + "github.com/pganalyze/collector/scheduler" "github.com/pganalyze/collector/state" "github.com/pganalyze/collector/util" "google.golang.org/protobuf/encoding/protojson" @@ -18,6 +19,9 @@ import ( func SendFull(ctx context.Context, server *state.Server, collectionOpts state.CollectionOpts, logger *util.Logger, newState state.PersistedState, diffState state.DiffState, transientState state.TransientState, collectedIntervalSecs uint32) error { s := transform.StateToSnapshot(newState, diffState, transientState, server) + if s.ServerStatistic.PgStatStatementsDealloc > 0 { + logger.PrintWarning("Detected %d pg_stat_statements deallocations in the last %d minutes. Enable/adjust reset settings on the pganalyze server settings page to avoid ", s.ServerStatistic.PgStatStatementsDealloc, scheduler.FullSnapshotMinutes) + } s.CollectedIntervalSecs = collectedIntervalSecs err := verifyIntegrity(&s) if err != nil { diff --git a/output/pganalyze_collector/server_message.pb.go b/output/pganalyze_collector/server_message.pb.go index ca917bd30..1913bf734 100644 --- a/output/pganalyze_collector/server_message.pb.go +++ b/output/pganalyze_collector/server_message.pb.go @@ -219,12 +219,17 @@ type ServerMessage_Features struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // How often the collector should run pg_stat_statements_reset() + // The maximum frequency that the collector should run pg_stat_statements_reset() + // Defaults to zero, which disables the feature + // The unit is a number of full snapshots, so a value of 6 = 1 hour StatementResetFrequency int32 `protobuf:"varint,1,opt,name=statement_reset_frequency,json=statementResetFrequency,proto3" json:"statement_reset_frequency,omitempty"` // Statement timeout for all SQL statements sent to the database (defaults to 30s) StatementTimeoutMs int32 `protobuf:"varint,2,opt,name=statement_timeout_ms,json=statementTimeoutMs,proto3" json:"statement_timeout_ms,omitempty"` // Statement timeout for pg_stat_statements query text requests (defaults to 120s) StatementTimeoutMsQueryText int32 `protobuf:"varint,3,opt,name=statement_timeout_ms_query_text,json=statementTimeoutMsQueryText,proto3" json:"statement_timeout_ms_query_text,omitempty"` + // The maximum size (in MB) that pg_stat_statements query text can grow to before + // triggering a reset. If statement_reset_frequency = 0, the reset won't occur. + StatementMaxSizeMb int32 `protobuf:"varint,4,opt,name=statement_max_size_mb,json=statementMaxSizeMb,proto3" json:"statement_max_size_mb,omitempty"` } func (x *ServerMessage_Features) Reset() { @@ -280,6 +285,13 @@ func (x *ServerMessage_Features) GetStatementTimeoutMsQueryText() int32 { return 0 } +func (x *ServerMessage_Features) GetStatementMaxSizeMb() int32 { + if x != nil { + return x.StatementMaxSizeMb + } + return 0 +} + type ServerMessage_Pause struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -428,7 +440,7 @@ var file_server_message_proto_rawDesc = []byte{ 0x0a, 0x14, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x13, 0x70, 0x67, 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x1a, 0x0c, 0x73, 0x68, 0x61, - 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc1, 0x09, 0x0a, 0x0d, 0x53, 0x65, + 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xf4, 0x09, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x43, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x70, 0x67, 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, @@ -461,7 +473,7 @@ var file_server_message_proto_rawDesc = []byte{ 0x62, 0x6c, 0x65, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x54, 0x61, 0x62, 0x6c, 0x65, - 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x1a, 0xbe, 0x01, 0x0a, 0x08, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x1a, 0xf1, 0x01, 0x0a, 0x08, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x3a, 0x0a, 0x19, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x65, 0x74, 0x5f, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x17, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, @@ -473,43 +485,46 @@ var file_server_message_proto_rawDesc = []byte{ 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x6d, 0x73, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x1b, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x54, 0x65, 0x78, 0x74, 0x1a, 0x1d, 0x0a, 0x05, 0x50, 0x61, 0x75, 0x73, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x70, 0x61, 0x75, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, - 0x70, 0x61, 0x75, 0x73, 0x65, 0x1a, 0xca, 0x03, 0x0a, 0x08, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, - 0x75, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x35, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x21, 0x2e, 0x70, 0x67, 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x2e, 0x63, 0x6f, 0x6c, - 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x75, 0x6e, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x61, 0x74, - 0x61, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0c, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, - 0x0a, 0x0a, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x71, 0x75, 0x65, 0x72, 0x79, 0x54, 0x65, 0x78, 0x74, 0x12, 0x4a, 0x0a, - 0x10, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, - 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x70, 0x67, 0x61, 0x6e, 0x61, 0x6c, - 0x79, 0x7a, 0x65, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x4e, 0x75, - 0x6c, 0x6c, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x0f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x71, 0x75, 0x65, - 0x72, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x71, 0x75, 0x65, 0x72, 0x79, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x6e, 0x0a, - 0x11, 0x70, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x74, 0x74, 0x69, 0x6e, - 0x67, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x70, 0x67, 0x61, 0x6e, 0x61, - 0x6c, 0x79, 0x7a, 0x65, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x52, 0x75, 0x6e, 0x2e, 0x50, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x53, 0x65, - 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x70, 0x6f, 0x73, - 0x74, 0x67, 0x72, 0x65, 0x73, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x43, 0x0a, - 0x15, 0x50, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x3b, 0x5a, - 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x67, 0x61, 0x6e, - 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x2f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2f, - 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x2f, 0x70, 0x67, 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, - 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x72, 0x79, 0x54, 0x65, 0x78, 0x74, 0x12, 0x31, 0x0a, 0x15, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x6d, 0x62, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x4d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x4d, 0x62, 0x1a, 0x1d, 0x0a, 0x05, 0x50, 0x61, 0x75, + 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x75, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x05, 0x70, 0x61, 0x75, 0x73, 0x65, 0x1a, 0xca, 0x03, 0x0a, 0x08, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x52, 0x75, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x35, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x70, 0x67, 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x2e, + 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, + 0x75, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0d, + 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x71, 0x75, 0x65, 0x72, 0x79, 0x54, 0x65, 0x78, 0x74, + 0x12, 0x4a, 0x0a, 0x10, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, + 0x74, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x70, 0x67, 0x61, + 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x2e, 0x4e, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x0f, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x32, 0x0a, 0x15, + 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x73, + 0x12, 0x6e, 0x0a, 0x11, 0x70, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x74, + 0x74, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x70, 0x67, + 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x2e, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, + 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, + 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x75, 0x6e, 0x2e, 0x50, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, + 0x73, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, + 0x70, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x53, 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x73, + 0x1a, 0x43, 0x0a, 0x15, 0x50, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x53, 0x65, 0x74, 0x74, + 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, + 0x67, 0x61, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x2f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x2f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x2f, 0x70, 0x67, 0x61, 0x6e, 0x61, 0x6c, + 0x79, 0x7a, 0x65, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/protobuf/server_message.proto b/protobuf/server_message.proto index e7451e01f..8a8487a08 100644 --- a/protobuf/server_message.proto +++ b/protobuf/server_message.proto @@ -27,12 +27,17 @@ message ServerMessage { } message Features { - // How often the collector should run pg_stat_statements_reset() + // The maximum frequency that the collector should run pg_stat_statements_reset() + // Defaults to zero, which disables the feature + // The unit is a number of full snapshots, so a value of 6 = 1 hour int32 statement_reset_frequency = 1; // Statement timeout for all SQL statements sent to the database (defaults to 30s) int32 statement_timeout_ms = 2; // Statement timeout for pg_stat_statements query text requests (defaults to 120s) int32 statement_timeout_ms_query_text = 3; + // The maximum size (in MB) that pg_stat_statements query text can grow to before + // triggering a reset. If statement_reset_frequency = 0, the reset won't occur. + int32 statement_max_size_mb = 4; } message Pause { diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index fd9a90cd3..792180298 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -104,3 +104,4 @@ func (schedule Schedule) ScheduleSecondary(ctx context.Context, primarySchedule } const FullSnapshotsPerHour = 6 +const FullSnapshotMinutes = 10 diff --git a/state/state.go b/state/state.go index 40ac3aa48..537db4cae 100644 --- a/state/state.go +++ b/state/state.go @@ -34,11 +34,6 @@ type PersistedState struct { System SystemState CollectorStats CollectorStats PgStatStatementsStats PgStatStatementsStats - - // Incremented every full snapshot, indicates whether we should run pg_stat_statements_reset() - // on behalf of the user. Only activates once it reaches GrantFeatures.StatementReset, - // and is reset afterwards. - StatementResetCounter int } type PersistedHighFreqState struct {