From 218608fce7eadd3b37284ff26f4eff9276d174b3 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Thu, 29 Jan 2026 10:27:28 -0500 Subject: [PATCH 01/13] Add pg_query fingerprint cache --- go.mod | 1 + go.sum | 2 + input/postgres/backends.go | 16 +- input/postgres/collection.go | 5 +- input/postgres/schema.go | 2 +- input/postgres/statements.go | 6 +- output/transform/activity.go | 1 + output/transform/logs.go | 2 + output/transform/util.go | 6 +- runner/activity.go | 2 + runner/full.go | 3 + state/fingerprints.go | 67 ++++ state/postgres_backend.go | 3 +- state/state.go | 4 + vendor/github.com/brentp/intintmap/.gitignore | 1 + vendor/github.com/brentp/intintmap/LICENSE | 23 ++ vendor/github.com/brentp/intintmap/README.md | 109 ++++++ .../github.com/brentp/intintmap/intintmap.go | 322 ++++++++++++++++++ vendor/modules.txt | 3 + 19 files changed, 565 insertions(+), 13 deletions(-) create mode 100644 state/fingerprints.go create mode 100644 vendor/github.com/brentp/intintmap/.gitignore create mode 100644 vendor/github.com/brentp/intintmap/LICENSE create mode 100644 vendor/github.com/brentp/intintmap/README.md create mode 100644 vendor/github.com/brentp/intintmap/intintmap.go diff --git a/go.mod b/go.mod index 190718676..99d55f1d2 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/monitor/azquery v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cosmosforpostgresql/armcosmosforpostgresql v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers/v4 v4.0.0-beta.5 + github.com/brentp/intintmap v0.0.0-20251106190759-56907b1f8479 github.com/fatih/color v1.16.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 diff --git a/go.sum b/go.sum index 48aea17c7..69143e51c 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,8 @@ github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nB github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E= github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/brentp/intintmap v0.0.0-20251106190759-56907b1f8479 h1:UZbbt19ACBOFO+CiDQFjaEoPJkBhj7GNGtIq59WR6Os= +github.com/brentp/intintmap v0.0.0-20251106190759-56907b1f8479/go.mod h1:TOk10ahXejq9wkEaym3KPRNeuR/h5Jx+s8QRWIa2oTM= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/input/postgres/backends.go b/input/postgres/backends.go index 3e552ba92..18ffb6d60 100644 --- a/input/postgres/backends.go +++ b/input/postgres/backends.go @@ -19,12 +19,14 @@ END const activitySQL string = ` SELECT (extract(epoch from COALESCE(backend_start, pg_catalog.pg_postmaster_start_time()))::int::text || pg_catalog.to_char(pid, 'FM0000000'))::bigint, datid, datname, usesysid, usename, pid, application_name, client_addr::text, client_port, - backend_start, xact_start, query_start, state_change, COALESCE(wait_event_type, '') = 'Lock' as waiting, backend_xid, backend_xmin, wait_event_type, wait_event, backend_type, %s, state, query + backend_start, xact_start, query_start, state_change, COALESCE(wait_event_type, '') = 'Lock' as waiting, + backend_xid, backend_xmin, wait_event_type, wait_event, backend_type, %s, state, query, %s FROM %s WHERE pid IS NOT NULL` func GetBackends(ctx context.Context, c *Collection, db *sql.DB) ([]state.PostgresBackend, error) { var blockingPidsField string + var queryIdField string var sourceTable string if c.GlobalOpts.CollectPostgresLocks { @@ -33,13 +35,19 @@ func GetBackends(ctx context.Context, c *Collection, db *sql.DB) ([]state.Postgr blockingPidsField = "NULL" } + if c.PostgresVersion.Numeric >= state.PostgresVersion14 { + queryIdField = "coalesce(query_id, 0)" + } else { + queryIdField = "0" + } + if c.HelperExists("get_stat_activity", nil) { - sourceTable = "pganalyze.get_stat_activity()" + sourceTable = "pganalyze.get_stat_activity()" // TODO: where is this defined? } else { sourceTable = "pg_catalog.pg_stat_activity" } - stmt, err := db.PrepareContext(ctx, QueryMarkerSQL+fmt.Sprintf(activitySQL, blockingPidsField, sourceTable)) + stmt, err := db.PrepareContext(ctx, QueryMarkerSQL+fmt.Sprintf(activitySQL, blockingPidsField, queryIdField, sourceTable)) if err != nil { return nil, err } @@ -63,7 +71,7 @@ func GetBackends(ctx context.Context, c *Collection, db *sql.DB) ([]state.Postgr &row.ClientPort, &row.BackendStart, &row.XactStart, &row.QueryStart, &row.StateChange, &row.Waiting, &row.BackendXid, &row.BackendXmin, &row.WaitEventType, &row.WaitEvent, &row.BackendType, pq.Array(&row.BlockedByPids), - &row.State, &row.Query) + &row.State, &row.Query, &row.QueryId) if err != nil { return nil, err } diff --git a/input/postgres/collection.go b/input/postgres/collection.go index 7075e5050..78f915a43 100644 --- a/input/postgres/collection.go +++ b/input/postgres/collection.go @@ -23,6 +23,8 @@ type Collection struct { // Information that is specific to the current database we're connected to HelperFunctions map[string][]state.PostgresFunction + + Fingerprints *state.Fingerprints } func helpersFromFunctions(functions []state.PostgresFunction) map[string][]state.PostgresFunction { @@ -84,7 +86,7 @@ func NewCollection(ctx context.Context, logger *util.Logger, server *state.Serve }, nil } -func (c *Collection) ForCurrentDatabase(functions []state.PostgresFunction) *Collection { +func (c *Collection) ForCurrentDatabase(server *state.Server, functions []state.PostgresFunction) *Collection { return &Collection{ Config: c.Config, Logger: c.Logger, @@ -95,6 +97,7 @@ func (c *Collection) ForCurrentDatabase(functions []state.PostgresFunction) *Col ConnectedAsSuperUser: c.ConnectedAsSuperUser, ConnectedAsMonitoringRole: c.ConnectedAsMonitoringRole, HelperFunctions: helpersFromFunctions(functions), + Fingerprints: server.Fingerprints, } } diff --git a/input/postgres/schema.go b/input/postgres/schema.go index 5432634f2..a183d0c3f 100644 --- a/input/postgres/schema.go +++ b/input/postgres/schema.go @@ -146,7 +146,7 @@ func collectSchemaData(ctx context.Context, c *Collection, db *sql.DB, ps state. } ps.Functions = append(ps.Functions, newFunctions...) - c = c.ForCurrentDatabase(newFunctions) + c = c.ForCurrentDatabase(server, newFunctions) if c.GlobalOpts.CollectPostgresRelations { newRelations, err := GetRelations(ctx, c, db, databaseOid) diff --git a/input/postgres/statements.go b/input/postgres/statements.go index 282c60e71..ba432dc9b 100644 --- a/input/postgres/statements.go +++ b/input/postgres/statements.go @@ -232,7 +232,7 @@ func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.Po case <-ctx.Done(): return nil, nil, ctx.Err() default: - fingerprintAndNormalize(c, key, query, statements, statementTextsByFp, ignoreIoTiming) + fingerprintAndNormalize(c, key, key.QueryID, query, statements, statementTextsByFp, ignoreIoTiming) } } @@ -351,7 +351,7 @@ func ignoreIOTiming(postgresVersion state.PostgresVersion, receivedQuery string) var collectorQueryFingerprint = util.FingerprintText(util.QueryTextCollector) var insufficientPrivsQueryFingerprint = util.FingerprintText(util.QueryTextInsufficientPrivs) -func fingerprintAndNormalize(c *Collection, key state.PostgresStatementKey, text string, statements state.PostgresStatementMap, statementTextsByFp state.PostgresStatementTextMap, ignoreIoTiming bool) { +func fingerprintAndNormalize(c *Collection, key state.PostgresStatementKey, queryID int64, text string, statements state.PostgresStatementMap, statementTextsByFp state.PostgresStatementTextMap, ignoreIoTiming bool) { if insufficientPrivilege(text) { statements[key] = state.PostgresStatement{ InsufficientPrivilege: true, @@ -365,7 +365,7 @@ func fingerprintAndNormalize(c *Collection, key state.PostgresStatementKey, text IgnoreIoTiming: ignoreIoTiming, } } else { - fp := util.FingerprintQuery(text, c.Config.FilterQueryText, -1) + fp := uint64(c.Fingerprints.Add(queryID, text, c.Config.FilterQueryText, -1)) statements[key] = state.PostgresStatement{Fingerprint: fp, IgnoreIoTiming: ignoreIoTiming} _, ok := statementTextsByFp[fp] if !ok { diff --git a/output/transform/activity.go b/output/transform/activity.go index 614deac4b..f18298ba2 100644 --- a/output/transform/activity.go +++ b/output/transform/activity.go @@ -35,6 +35,7 @@ func ActivityStateToCompactActivitySnapshot(server *state.Server, activityState b.RoleIdx, b.DatabaseIdx, backend.Query.String, + backend.QueryId, activityState.TrackActivityQuerySize, ) b.HasQueryIdx = true diff --git a/output/transform/logs.go b/output/transform/logs.go index 05ce90cba..3693b1d4d 100644 --- a/output/transform/logs.go +++ b/output/transform/logs.go @@ -46,6 +46,7 @@ func transformPostgresQuerySamples(server *state.Server, s snapshot.CompactLogSn roleIdx, databaseIdx, sampleIn.Query, + 0, -1, ) @@ -184,6 +185,7 @@ func transformSystemLogLine(server *state.Server, r *snapshot.CompactSnapshot_Ba logLine.RoleIdx, logLine.DatabaseIdx, logLineIn.Query, + 0, -1, ) logLine.HasQueryIdx = true diff --git a/output/transform/util.go b/output/transform/util.go index 9f2804a89..035b35692 100644 --- a/output/transform/util.go +++ b/output/transform/util.go @@ -61,11 +61,11 @@ func upsertQueryReferenceAndInformation(s *snapshot.FullSnapshot, statementTexts return idx } -func upsertQueryReferenceAndInformationSimple(server *state.Server, refs []*snapshot.QueryReference, infos []*snapshot.QueryInformation, roleIdx int32, databaseIdx int32, originalQuery string, trackActivityQuerySize int) (int32, []*snapshot.QueryReference, []*snapshot.QueryInformation) { - fingerprint := util.FingerprintQuery(originalQuery, server.Config.FilterQueryText, trackActivityQuerySize) +func upsertQueryReferenceAndInformationSimple(server *state.Server, refs []*snapshot.QueryReference, infos []*snapshot.QueryInformation, roleIdx int32, databaseIdx int32, originalQuery string, queryID int64, trackActivityQuerySize int) (int32, []*snapshot.QueryReference, []*snapshot.QueryInformation) { + fingerprint := server.Fingerprints.Add(queryID, originalQuery, server.Config.FilterQueryText, trackActivityQuerySize) fpBuf := make([]byte, 8) - binary.BigEndian.PutUint64(fpBuf, fingerprint) + binary.BigEndian.PutUint64(fpBuf, uint64(fingerprint)) newRef := snapshot.QueryReference{ DatabaseIdx: databaseIdx, RoleIdx: roleIdx, diff --git a/runner/activity.go b/runner/activity.go index 029a20f1b..ba53c6895 100644 --- a/runner/activity.go +++ b/runner/activity.go @@ -99,6 +99,8 @@ func processActivityForServer(ctx context.Context, server *state.Server, opts st } newState.ActivitySnapshotAt = activity.CollectedAt + logger.PrintInfo("Fingerprints: %d", server.Fingerprints.Size()) + return newState, true, nil } diff --git a/runner/full.go b/runner/full.go index 026d77b1e..2940b8ea7 100644 --- a/runner/full.go +++ b/runner/full.go @@ -59,6 +59,9 @@ func collectDiffAndSubmit(ctx context.Context, server *state.Server, opts state. return newState, collectionStatus, err } + logger.PrintInfo("Fingerprints: %d", server.Fingerprints.Size()) + server.Fingerprints.Cleanup() + return newState, collectionStatus, nil } diff --git a/state/fingerprints.go b/state/fingerprints.go new file mode 100644 index 000000000..bb0d1ecdc --- /dev/null +++ b/state/fingerprints.go @@ -0,0 +1,67 @@ +package state + +import ( + "github.com/brentp/intintmap" + "github.com/pganalyze/collector/util" + "sync" +) + +// 1 million entries in intintmap's internal flat array takes ~16 MB +const MAX_SIZE = 1000000 +const FILL_FACTOR = 0.99 + +type Fingerprints struct { + cache *intintmap.Map + lock sync.RWMutex +} + +func NewFingerprints() *Fingerprints { + return &Fingerprints{ + cache: intintmap.New(MAX_SIZE, FILL_FACTOR), + lock: sync.RWMutex{}, + } +} + +func (c *Fingerprints) Get(queryID int64) (int64, bool) { + c.lock.RLock() + defer c.lock.RUnlock() + return c.cache.Get(queryID) +} + +func (c *Fingerprints) Add(queryID int64, text string, filterQueryText string, trackActivityQuerySize int) int64 { + if queryID == 0 { + return int64(util.FingerprintQuery(text, filterQueryText, trackActivityQuerySize)) + } + fingerprint, exists := c.Get(queryID) + if exists { + return fingerprint + } + c.lock.Lock() + fingerprint = int64(util.FingerprintQuery(text, filterQueryText, trackActivityQuerySize)) + c.cache.Put(queryID, fingerprint) + c.lock.Unlock() + return fingerprint +} + +func (c *Fingerprints) Size() int { + return c.cache.Size() +} + +// Retains 33% of entries, in an effort to avoid re-fingerprinting common queries. +// intintmap can't evict less used entries so isn't as CPU-efficient as an LRU cache, +// but since it's backed by a flat array it's much more memory-efficient. That +// allows us to have a larger cache that doesn't need to be emptied as often. +func (c *Fingerprints) Cleanup() { + if c.cache.Size() < MAX_SIZE { + return + } + cache := intintmap.New(MAX_SIZE, FILL_FACTOR) + index := 0 + c.cache.Each(func(key, value int64) { + if index%3 == 0 { + cache.Put(key, value) + } + index += 1 + }) + c.cache = cache +} diff --git a/state/postgres_backend.go b/state/postgres_backend.go index e7b3afe10..8f0c24cb3 100644 --- a/state/postgres_backend.go +++ b/state/postgres_backend.go @@ -29,7 +29,8 @@ type PostgresBackend struct { BackendType null.String // 10+ The process type of this backend - Query null.String // Text of this backend's most recent query + Query null.String // Text of this backend's most recent query + QueryId int64 // Current overall state of this backend. Possible values are: // - active: The backend is executing a query. diff --git a/state/state.go b/state/state.go index e218fc8f5..40ac3aa48 100644 --- a/state/state.go +++ b/state/state.go @@ -331,6 +331,9 @@ type Server struct { // differences (see https://groups.google.com/g/golang-nuts/c/eIqkhXh9PLg), // as we access this in high frequency log-related code paths. LogIgnoreFlags uint32 + + // Cache of Postgres query_id -> pg_query fingerprint mappings + Fingerprints *Fingerprints } func MakeServer(config config.ServerConfig, testRun bool) *Server { @@ -347,6 +350,7 @@ func MakeServer(config config.ServerConfig, testRun bool) *Server { QueryRuns: make(map[int64]*QueryRun), QueryRunsMutex: &sync.Mutex{}, LogParseMutex: &sync.RWMutex{}, + Fingerprints: NewFingerprints(), } server.Grant.Store(&Grant{Config: pganalyze_collector.ServerMessage_Config{Features: &pganalyze_collector.ServerMessage_Features{}}}) server.Pause.Store(false) diff --git a/vendor/github.com/brentp/intintmap/.gitignore b/vendor/github.com/brentp/intintmap/.gitignore new file mode 100644 index 000000000..1377554eb --- /dev/null +++ b/vendor/github.com/brentp/intintmap/.gitignore @@ -0,0 +1 @@ +*.swp diff --git a/vendor/github.com/brentp/intintmap/LICENSE b/vendor/github.com/brentp/intintmap/LICENSE new file mode 100644 index 000000000..1eac633b0 --- /dev/null +++ b/vendor/github.com/brentp/intintmap/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2016, Brent Pedersen - Bioinformatics +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/brentp/intintmap/README.md b/vendor/github.com/brentp/intintmap/README.md new file mode 100644 index 000000000..a541b1b35 --- /dev/null +++ b/vendor/github.com/brentp/intintmap/README.md @@ -0,0 +1,109 @@ +Fast int64 -> int64 hash in golang. + +[![GoDoc](https://godoc.org/github.com/brentp/intintmap?status.svg)](https://godoc.org/github.com/brentp/intintmap) +[![Go Report Card](https://goreportcard.com/badge/github.com/brentp/intintmap)](https://goreportcard.com/report/github.com/brentp/intintmap) + +# intintmap + + import "github.com/brentp/intintmap" + +Package intintmap is a fast int64 key -> int64 value map. + +It is copied nearly verbatim from +http://java-performance.info/implementing-world-fastest-java-int-to-int-hash-map/ . + +It interleaves keys and values in the same underlying array to improve locality. + +It is 2-5X faster than the builtin map: +``` +BenchmarkIntIntMapFill 10 158436598 ns/op +BenchmarkStdMapFill 5 312135474 ns/op +BenchmarkIntIntMapGet10PercentHitRate 5000 243108 ns/op +BenchmarkStdMapGet10PercentHitRate 5000 268927 ns/op +BenchmarkIntIntMapGet100PercentHitRate 500 2249349 ns/op +BenchmarkStdMapGet100PercentHitRate 100 10258929 ns/op +``` + +## Usage + +```go +m := intintmap.New(32768, 0.6) +m.Put(int64(1234), int64(-222)) +m.Put(int64(123), int64(33)) + +v, ok := m.Get(int64(222)) +v, ok := m.Get(int64(333)) + +m.Del(int64(222)) +m.Del(int64(333)) + +fmt.Println(m.Size()) + +for k := range m.Keys() { + fmt.Printf("key: %d\n", k) +} + +for kv := range m.Items() { + fmt.Printf("key: %d, value: %d\n", kv[0], kv[1]) +} +``` + +#### type Map + +```go +type Map struct { +} +``` + +Map is a map-like data-structure for int64s + +#### func New + +```go +func New(size int, fillFactor float64) *Map +``` +New returns a map initialized with n spaces and uses the stated fillFactor. The +map will grow as needed. + +#### func (*Map) Get + +```go +func (m *Map) Get(key int64) (int64, bool) +``` +Get returns the value if the key is found. + +#### func (*Map) Put + +```go +func (m *Map) Put(key int64, val int64) +``` +Put adds or updates key with value val. + +#### func (*Map) Del + +```go +func (m *Map) Del(key int64) +``` +Del deletes a key and its value. + +#### func (*Map) Keys + +```go +func (m *Map) Keys() chan int64 +``` +Keys returns a channel for iterating all keys. + +#### func (*Map) Items + +```go +func (m *Map) Items() chan [2]int64 +``` +Items returns a channel for iterating all key-value pairs. + + +#### func (*Map) Size + +```go +func (m *Map) Size() int +``` +Size returns size of the map. diff --git a/vendor/github.com/brentp/intintmap/intintmap.go b/vendor/github.com/brentp/intintmap/intintmap.go new file mode 100644 index 000000000..d98353d54 --- /dev/null +++ b/vendor/github.com/brentp/intintmap/intintmap.go @@ -0,0 +1,322 @@ +// Package intintmap is a fast int64 key -> int64 value map. +// +// It is copied nearly verbatim from http://java-performance.info/implementing-world-fastest-java-int-to-int-hash-map/ +package intintmap + +import ( + "math" +) + +// INT_PHI is for scrambling the keys +const INT_PHI = 0x9E3779B9 + +// FREE_KEY is the 'free' key +const FREE_KEY = 0 + +func phiMix(x int64) int64 { + h := x * INT_PHI + return h ^ (h >> 16) +} + +// Map is a map-like data-structure for int64s +type Map struct { + data []int64 // interleaved keys and values + fillFactor float64 + threshold int // we will resize a map once it reaches this size + size int + + mask int64 // mask to calculate the original position + mask2 int64 + + hasFreeKey bool // do we have 'free' key in the map? + freeVal int64 // value of 'free' key +} + +func nextPowerOf2(x uint32) uint32 { + if x == math.MaxUint32 { + return x + } + + if x == 0 { + return 1 + } + + x-- + x |= x >> 1 + x |= x >> 2 + x |= x >> 4 + x |= x >> 8 + x |= x >> 16 + + return x + 1 +} + +func arraySize(exp int, fill float64) int { + s := nextPowerOf2(uint32(math.Ceil(float64(exp) / fill))) + if s < 2 { + s = 2 + } + return int(s) +} + +// New returns a map initialized with n spaces and uses the stated fillFactor. +// The map will grow as needed. +func New(size int, fillFactor float64) *Map { + if fillFactor <= 0 || fillFactor >= 1 { + panic("FillFactor must be in (0, 1)") + } + if size <= 0 { + panic("Size must be positive") + } + + capacity := arraySize(size, fillFactor) + return &Map{ + data: make([]int64, 2*capacity), + fillFactor: fillFactor, + threshold: int(math.Floor(float64(capacity) * fillFactor)), + mask: int64(capacity - 1), + mask2: int64(2*capacity - 1), + } +} + +// Get returns the value if the key is found. +func (m *Map) Get(key int64) (int64, bool) { + if key == FREE_KEY { + if m.hasFreeKey { + return m.freeVal, true + } + return 0, false + } + + ptr := (phiMix(key) & m.mask) << 1 + if ptr < 0 || ptr >= int64(len(m.data)) { // Check to help to compiler to eliminate a bounds check below. + return 0, false + } + k := m.data[ptr] + + if k == FREE_KEY { // end of chain already + return 0, false + } + if k == key { // we check FREE prior to this call + return m.data[ptr+1], true + } + + for { + ptr = (ptr + 2) & m.mask2 + k = m.data[ptr] + if k == FREE_KEY { + return 0, false + } + if k == key { + return m.data[ptr+1], true + } + } +} + +// Put adds or updates key with value val. +func (m *Map) Put(key int64, val int64) { + if key == FREE_KEY { + if !m.hasFreeKey { + m.size++ + } + m.hasFreeKey = true + m.freeVal = val + return + } + + ptr := (phiMix(key) & m.mask) << 1 + k := m.data[ptr] + + if k == FREE_KEY { // end of chain already + m.data[ptr] = key + m.data[ptr+1] = val + if m.size >= m.threshold { + m.rehash() + } else { + m.size++ + } + return + } else if k == key { // overwrite existed value + m.data[ptr+1] = val + return + } + + for { + ptr = (ptr + 2) & m.mask2 + k = m.data[ptr] + + if k == FREE_KEY { + m.data[ptr] = key + m.data[ptr+1] = val + if m.size >= m.threshold { + m.rehash() + } else { + m.size++ + } + return + } else if k == key { + m.data[ptr+1] = val + return + } + } + +} + +// Del deletes a key and its value. +func (m *Map) Del(key int64) { + if key == FREE_KEY { + m.hasFreeKey = false + m.size-- + return + } + + ptr := (phiMix(key) & m.mask) << 1 + k := m.data[ptr] + + if k == key { + m.shiftKeys(ptr) + m.size-- + return + } else if k == FREE_KEY { // end of chain already + return + } + + for { + ptr = (ptr + 2) & m.mask2 + k = m.data[ptr] + + if k == key { + m.shiftKeys(ptr) + m.size-- + return + } else if k == FREE_KEY { + return + } + + } +} + +func (m *Map) shiftKeys(pos int64) int64 { + // Shift entries with the same hash. + var last, slot int64 + var k int64 + var data = m.data + for { + last = pos + pos = (last + 2) & m.mask2 + for { + k = data[pos] + if k == FREE_KEY { + data[last] = FREE_KEY + return last + } + + slot = (phiMix(k) & m.mask) << 1 + if last <= pos { + if last >= slot || slot > pos { + break + } + } else { + if last >= slot && slot > pos { + break + } + } + pos = (pos + 2) & m.mask2 + } + data[last] = k + data[last+1] = data[pos+1] + } +} + +func (m *Map) rehash() { + newCapacity := len(m.data) * 2 + m.threshold = int(math.Floor(float64(newCapacity/2) * m.fillFactor)) + m.mask = int64(newCapacity/2 - 1) + m.mask2 = int64(newCapacity - 1) + + data := make([]int64, len(m.data)) // copy of original data + copy(data, m.data) + + m.data = make([]int64, newCapacity) + if m.hasFreeKey { // reset size + m.size = 1 + } else { + m.size = 0 + } + + var o int64 + for i := 0; i < len(data); i += 2 { + o = data[i] + if o != FREE_KEY { + m.Put(o, data[i+1]) + } + } +} + +// Size returns size of the map. +func (m *Map) Size() int { + return m.size +} + +// Keys returns a channel for iterating all keys. +func (m *Map) Keys() chan int64 { + c := make(chan int64, 10) + go func() { + data := m.data + var k int64 + + if m.hasFreeKey { + c <- FREE_KEY // value is m.freeVal + } + + for i := 0; i < len(data); i += 2 { + k = data[i] + if k == FREE_KEY { + continue + } + c <- k // value is data[i+1] + } + close(c) + }() + return c +} + +// Items returns a channel for iterating all key-value pairs. +func (m *Map) Items() chan [2]int64 { + c := make(chan [2]int64, 10) + go func() { + data := m.data + var k int64 + + if m.hasFreeKey { + c <- [2]int64{FREE_KEY, m.freeVal} + } + + for i := 0; i < len(data); i += 2 { + k = data[i] + if k == FREE_KEY { + continue + } + c <- [2]int64{k, data[i+1]} + } + close(c) + }() + return c +} + +func (m *Map) Each(f func(k, v int64)) { + data := m.data + var k int64 + + if m.hasFreeKey { + f(FREE_KEY, m.freeVal) + } + + for i := 0; i < len(data); i += 2 { + k = data[i] + if k == FREE_KEY { + continue + } + f(k, data[i+1]) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 4d23195fb..c154ec0c2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -223,6 +223,9 @@ github.com/aws/aws-sdk-go/service/sso/ssoiface github.com/aws/aws-sdk-go/service/ssooidc github.com/aws/aws-sdk-go/service/sts github.com/aws/aws-sdk-go/service/sts/stsiface +# github.com/brentp/intintmap v0.0.0-20251106190759-56907b1f8479 +## explicit +github.com/brentp/intintmap # github.com/cenkalti/backoff/v4 v4.2.1 ## explicit; go 1.18 github.com/cenkalti/backoff/v4 From be28a72419f57e4715cdaaaae6e1b710c87e98e1 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Thu, 29 Jan 2026 11:52:45 -0500 Subject: [PATCH 02/13] Update lock usage --- state/fingerprints.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/state/fingerprints.go b/state/fingerprints.go index bb0d1ecdc..3cae01b0c 100644 --- a/state/fingerprints.go +++ b/state/fingerprints.go @@ -44,6 +44,8 @@ func (c *Fingerprints) Add(queryID int64, text string, filterQueryText string, t } func (c *Fingerprints) Size() int { + c.lock.RLock() + defer c.lock.RUnlock() return c.cache.Size() } @@ -52,7 +54,7 @@ func (c *Fingerprints) Size() int { // but since it's backed by a flat array it's much more memory-efficient. That // allows us to have a larger cache that doesn't need to be emptied as often. func (c *Fingerprints) Cleanup() { - if c.cache.Size() < MAX_SIZE { + if c.Size() < MAX_SIZE { return } cache := intintmap.New(MAX_SIZE, FILL_FACTOR) @@ -63,5 +65,7 @@ func (c *Fingerprints) Cleanup() { } index += 1 }) + c.lock.Lock() c.cache = cache + c.lock.Unlock() } From 122ab2753c95526792347a3ccd22f4044c07dab7 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Thu, 29 Jan 2026 15:01:45 -0500 Subject: [PATCH 03/13] Fix missing initialization of Collection struct --- input/postgres/collection.go | 1 + 1 file changed, 1 insertion(+) diff --git a/input/postgres/collection.go b/input/postgres/collection.go index 78f915a43..6d02c5460 100644 --- a/input/postgres/collection.go +++ b/input/postgres/collection.go @@ -83,6 +83,7 @@ func NewCollection(ctx context.Context, logger *util.Logger, server *state.Serve ConnectedAsSuperUser: connectedAsSuperUser, ConnectedAsMonitoringRole: connectedAsMonitoringRole, HelperFunctions: helpersFromFunctions(helperFunctions), + Fingerprints: server.Fingerprints, }, nil } From cdd4cd50ac75c6958bf5fd04582ebb54be37e525 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Fri, 30 Jan 2026 09:58:21 -0500 Subject: [PATCH 04/13] Take lock sooner in Cleanup() --- state/fingerprints.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/fingerprints.go b/state/fingerprints.go index 3cae01b0c..c0e460d18 100644 --- a/state/fingerprints.go +++ b/state/fingerprints.go @@ -57,6 +57,7 @@ func (c *Fingerprints) Cleanup() { if c.Size() < MAX_SIZE { return } + c.lock.Lock() cache := intintmap.New(MAX_SIZE, FILL_FACTOR) index := 0 c.cache.Each(func(key, value int64) { @@ -65,7 +66,6 @@ func (c *Fingerprints) Cleanup() { } index += 1 }) - c.lock.Lock() c.cache = cache c.lock.Unlock() } From 5a75db400fb11a1e841138f53636555d7dac5ab6 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Mon, 2 Feb 2026 16:17:44 -0500 Subject: [PATCH 05/13] Remove debug logging --- runner/activity.go | 2 -- runner/full.go | 1 - 2 files changed, 3 deletions(-) diff --git a/runner/activity.go b/runner/activity.go index ba53c6895..029a20f1b 100644 --- a/runner/activity.go +++ b/runner/activity.go @@ -99,8 +99,6 @@ func processActivityForServer(ctx context.Context, server *state.Server, opts st } newState.ActivitySnapshotAt = activity.CollectedAt - logger.PrintInfo("Fingerprints: %d", server.Fingerprints.Size()) - return newState, true, nil } diff --git a/runner/full.go b/runner/full.go index 2940b8ea7..c6a17ddfb 100644 --- a/runner/full.go +++ b/runner/full.go @@ -59,7 +59,6 @@ func collectDiffAndSubmit(ctx context.Context, server *state.Server, opts state. return newState, collectionStatus, err } - logger.PrintInfo("Fingerprints: %d", server.Fingerprints.Size()) server.Fingerprints.Cleanup() return newState, collectionStatus, nil From 0110cb2956349cc2f123ca76e1fb65203ac35cfb Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Mon, 2 Feb 2026 16:19:56 -0500 Subject: [PATCH 06/13] Remove TODO comment --- input/postgres/backends.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input/postgres/backends.go b/input/postgres/backends.go index 18ffb6d60..c9bd08c82 100644 --- a/input/postgres/backends.go +++ b/input/postgres/backends.go @@ -42,7 +42,7 @@ func GetBackends(ctx context.Context, c *Collection, db *sql.DB) ([]state.Postgr } if c.HelperExists("get_stat_activity", nil) { - sourceTable = "pganalyze.get_stat_activity()" // TODO: where is this defined? + sourceTable = "pganalyze.get_stat_activity()" } else { sourceTable = "pg_catalog.pg_stat_activity" } From d8ecab717e92584a1f95ecddadca0467ef3005bf Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Tue, 3 Feb 2026 20:19:48 -0500 Subject: [PATCH 07/13] Decrease cache size to 500k entries --- state/fingerprints.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/state/fingerprints.go b/state/fingerprints.go index c0e460d18..fc88d04aa 100644 --- a/state/fingerprints.go +++ b/state/fingerprints.go @@ -6,8 +6,8 @@ import ( "sync" ) -// 1 million entries in intintmap's internal flat array takes ~16 MB -const MAX_SIZE = 1000000 +// 500 thousand entries in intintmap's internal flat array takes ~8 MB +const MAX_SIZE = 500000 const FILL_FACTOR = 0.99 type Fingerprints struct { From 13432a495a385d4ccc63a7b487e36cea5d30d8e5 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Wed, 4 Feb 2026 18:07:10 -0500 Subject: [PATCH 08/13] Avoid adding virtual fingerprints to the cache for truncated query text --- state/fingerprints.go | 7 ++++++- util/fingerprint.go | 12 ++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/state/fingerprints.go b/state/fingerprints.go index fc88d04aa..14c4c6e1a 100644 --- a/state/fingerprints.go +++ b/state/fingerprints.go @@ -36,8 +36,13 @@ func (c *Fingerprints) Add(queryID int64, text string, filterQueryText string, t if exists { return fingerprint } + fp, virtual := util.TryFingerprintQuery(text, filterQueryText, trackActivityQuerySize) + fingerprint = int64(fp) + if virtual { + // Don't write virtual fingerprints to the cache so we can cache real fingerprints later + return fingerprint + } c.lock.Lock() - fingerprint = int64(util.FingerprintQuery(text, filterQueryText, trackActivityQuerySize)) c.cache.Put(queryID, fingerprint) c.lock.Unlock() return fingerprint diff --git a/util/fingerprint.go b/util/fingerprint.go index c96b566bd..11e580c83 100644 --- a/util/fingerprint.go +++ b/util/fingerprint.go @@ -4,10 +4,12 @@ import ( pg_query "github.com/pganalyze/pg_query_go/v6" ) -// FingerprintQuery - Generates a unique fingerprint for the given query -func FingerprintQuery(query string, filterQueryText string, trackActivityQuerySize int) (fp uint64) { +// TryFingerprintQuery - Generates a unique fingerprint for the given query, +// and whether the query text had to be massaged to generate a fingerprint +func TryFingerprintQuery(query string, filterQueryText string, trackActivityQuerySize int) (fp uint64, virtual bool) { fp, err := pg_query.FingerprintToUInt64(query) if err != nil { + virtual = true fixedQuery := fixTruncatedQuery(query) fp, err = pg_query.FingerprintToUInt64(fixedQuery) @@ -20,6 +22,12 @@ func FingerprintQuery(query string, filterQueryText string, trackActivityQuerySi return } +// FingerprintQuery - Generates a unique fingerprint for the given query +func FingerprintQuery(query string, filterQueryText string, trackActivityQuerySize int) (fp uint64) { + fp, _ = TryFingerprintQuery(query, filterQueryText, trackActivityQuerySize) + return +} + // FingerprintText - Generates a fingerprint for static texts (used for error scenarios) func FingerprintText(query string) (fp uint64) { return pg_query.HashXXH3_64([]byte(query), 0xee) From 5af80cd7a2af080e8d3796e4efafc01e5dd5125b Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Thu, 5 Feb 2026 16:31:03 -0500 Subject: [PATCH 09/13] No need to pass server to ForCurrentDatabase --- input/postgres/collection.go | 4 ++-- input/postgres/schema.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/input/postgres/collection.go b/input/postgres/collection.go index 6d02c5460..68bb41332 100644 --- a/input/postgres/collection.go +++ b/input/postgres/collection.go @@ -87,7 +87,7 @@ func NewCollection(ctx context.Context, logger *util.Logger, server *state.Serve }, nil } -func (c *Collection) ForCurrentDatabase(server *state.Server, functions []state.PostgresFunction) *Collection { +func (c *Collection) ForCurrentDatabase(functions []state.PostgresFunction) *Collection { return &Collection{ Config: c.Config, Logger: c.Logger, @@ -98,7 +98,7 @@ func (c *Collection) ForCurrentDatabase(server *state.Server, functions []state. ConnectedAsSuperUser: c.ConnectedAsSuperUser, ConnectedAsMonitoringRole: c.ConnectedAsMonitoringRole, HelperFunctions: helpersFromFunctions(functions), - Fingerprints: server.Fingerprints, + Fingerprints: c.Fingerprints, } } diff --git a/input/postgres/schema.go b/input/postgres/schema.go index a183d0c3f..5432634f2 100644 --- a/input/postgres/schema.go +++ b/input/postgres/schema.go @@ -146,7 +146,7 @@ func collectSchemaData(ctx context.Context, c *Collection, db *sql.DB, ps state. } ps.Functions = append(ps.Functions, newFunctions...) - c = c.ForCurrentDatabase(server, newFunctions) + c = c.ForCurrentDatabase(newFunctions) if c.GlobalOpts.CollectPostgresRelations { newRelations, err := GetRelations(ctx, c, db, databaseOid) From 2ab3d4fb5284ed6fe8053856e299562745123c9b Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Thu, 5 Feb 2026 17:30:08 -0500 Subject: [PATCH 10/13] Remove intintmap, use simple map instead --- go.mod | 1 - go.sum | 2 - runner/full.go | 2 - state/fingerprints.go | 39 +-- vendor/github.com/brentp/intintmap/.gitignore | 1 - vendor/github.com/brentp/intintmap/LICENSE | 23 -- vendor/github.com/brentp/intintmap/README.md | 109 ------ .../github.com/brentp/intintmap/intintmap.go | 322 ------------------ vendor/modules.txt | 3 - 9 files changed, 18 insertions(+), 484 deletions(-) delete mode 100644 vendor/github.com/brentp/intintmap/.gitignore delete mode 100644 vendor/github.com/brentp/intintmap/LICENSE delete mode 100644 vendor/github.com/brentp/intintmap/README.md delete mode 100644 vendor/github.com/brentp/intintmap/intintmap.go diff --git a/go.mod b/go.mod index 99d55f1d2..190718676 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/monitor/azquery v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cosmosforpostgresql/armcosmosforpostgresql v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/postgresql/armpostgresqlflexibleservers/v4 v4.0.0-beta.5 - github.com/brentp/intintmap v0.0.0-20251106190759-56907b1f8479 github.com/fatih/color v1.16.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 diff --git a/go.sum b/go.sum index 69143e51c..48aea17c7 100644 --- a/go.sum +++ b/go.sum @@ -66,8 +66,6 @@ github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nB github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E= github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/brentp/intintmap v0.0.0-20251106190759-56907b1f8479 h1:UZbbt19ACBOFO+CiDQFjaEoPJkBhj7GNGtIq59WR6Os= -github.com/brentp/intintmap v0.0.0-20251106190759-56907b1f8479/go.mod h1:TOk10ahXejq9wkEaym3KPRNeuR/h5Jx+s8QRWIa2oTM= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/runner/full.go b/runner/full.go index c6a17ddfb..026d77b1e 100644 --- a/runner/full.go +++ b/runner/full.go @@ -59,8 +59,6 @@ func collectDiffAndSubmit(ctx context.Context, server *state.Server, opts state. return newState, collectionStatus, err } - server.Fingerprints.Cleanup() - return newState, collectionStatus, nil } diff --git a/state/fingerprints.go b/state/fingerprints.go index 14c4c6e1a..227fd0db8 100644 --- a/state/fingerprints.go +++ b/state/fingerprints.go @@ -1,23 +1,21 @@ package state import ( - "github.com/brentp/intintmap" "github.com/pganalyze/collector/util" "sync" ) -// 500 thousand entries in intintmap's internal flat array takes ~8 MB -const MAX_SIZE = 500000 -const FILL_FACTOR = 0.99 +// 500,000 entries use around 12 MB +const MAX_SIZE = 500_000 type Fingerprints struct { - cache *intintmap.Map + cache map[int64]int64 lock sync.RWMutex } func NewFingerprints() *Fingerprints { return &Fingerprints{ - cache: intintmap.New(MAX_SIZE, FILL_FACTOR), + cache: make(map[int64]int64, MAX_SIZE), lock: sync.RWMutex{}, } } @@ -25,7 +23,8 @@ func NewFingerprints() *Fingerprints { func (c *Fingerprints) Get(queryID int64) (int64, bool) { c.lock.RLock() defer c.lock.RUnlock() - return c.cache.Get(queryID) + fingerprint, exists := c.cache[queryID] + return fingerprint, exists } func (c *Fingerprints) Add(queryID int64, text string, filterQueryText string, trackActivityQuerySize int) int64 { @@ -39,38 +38,36 @@ func (c *Fingerprints) Add(queryID int64, text string, filterQueryText string, t fp, virtual := util.TryFingerprintQuery(text, filterQueryText, trackActivityQuerySize) fingerprint = int64(fp) if virtual { - // Don't write virtual fingerprints to the cache so we can cache real fingerprints later + // Don't store virtual fingerprints so we can cache real fingerprints later return fingerprint } + c.cleanup() c.lock.Lock() - c.cache.Put(queryID, fingerprint) + c.cache[queryID] = fingerprint c.lock.Unlock() return fingerprint } -func (c *Fingerprints) Size() int { +func (c *Fingerprints) size() int { c.lock.RLock() defer c.lock.RUnlock() - return c.cache.Size() + return len(c.cache) } -// Retains 33% of entries, in an effort to avoid re-fingerprinting common queries. -// intintmap can't evict less used entries so isn't as CPU-efficient as an LRU cache, -// but since it's backed by a flat array it's much more memory-efficient. That -// allows us to have a larger cache that doesn't need to be emptied as often. -func (c *Fingerprints) Cleanup() { - if c.Size() < MAX_SIZE { +// Retains a random 33% sample of entries if the cache grows too large +func (c *Fingerprints) cleanup() { + if c.size() < MAX_SIZE { return } c.lock.Lock() - cache := intintmap.New(MAX_SIZE, FILL_FACTOR) + cache := make(map[int64]int64, MAX_SIZE) index := 0 - c.cache.Each(func(key, value int64) { + for key, value := range c.cache { if index%3 == 0 { - cache.Put(key, value) + cache[key] = value } index += 1 - }) + } c.cache = cache c.lock.Unlock() } diff --git a/vendor/github.com/brentp/intintmap/.gitignore b/vendor/github.com/brentp/intintmap/.gitignore deleted file mode 100644 index 1377554eb..000000000 --- a/vendor/github.com/brentp/intintmap/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.swp diff --git a/vendor/github.com/brentp/intintmap/LICENSE b/vendor/github.com/brentp/intintmap/LICENSE deleted file mode 100644 index 1eac633b0..000000000 --- a/vendor/github.com/brentp/intintmap/LICENSE +++ /dev/null @@ -1,23 +0,0 @@ -Copyright (c) 2016, Brent Pedersen - Bioinformatics -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/brentp/intintmap/README.md b/vendor/github.com/brentp/intintmap/README.md deleted file mode 100644 index a541b1b35..000000000 --- a/vendor/github.com/brentp/intintmap/README.md +++ /dev/null @@ -1,109 +0,0 @@ -Fast int64 -> int64 hash in golang. - -[![GoDoc](https://godoc.org/github.com/brentp/intintmap?status.svg)](https://godoc.org/github.com/brentp/intintmap) -[![Go Report Card](https://goreportcard.com/badge/github.com/brentp/intintmap)](https://goreportcard.com/report/github.com/brentp/intintmap) - -# intintmap - - import "github.com/brentp/intintmap" - -Package intintmap is a fast int64 key -> int64 value map. - -It is copied nearly verbatim from -http://java-performance.info/implementing-world-fastest-java-int-to-int-hash-map/ . - -It interleaves keys and values in the same underlying array to improve locality. - -It is 2-5X faster than the builtin map: -``` -BenchmarkIntIntMapFill 10 158436598 ns/op -BenchmarkStdMapFill 5 312135474 ns/op -BenchmarkIntIntMapGet10PercentHitRate 5000 243108 ns/op -BenchmarkStdMapGet10PercentHitRate 5000 268927 ns/op -BenchmarkIntIntMapGet100PercentHitRate 500 2249349 ns/op -BenchmarkStdMapGet100PercentHitRate 100 10258929 ns/op -``` - -## Usage - -```go -m := intintmap.New(32768, 0.6) -m.Put(int64(1234), int64(-222)) -m.Put(int64(123), int64(33)) - -v, ok := m.Get(int64(222)) -v, ok := m.Get(int64(333)) - -m.Del(int64(222)) -m.Del(int64(333)) - -fmt.Println(m.Size()) - -for k := range m.Keys() { - fmt.Printf("key: %d\n", k) -} - -for kv := range m.Items() { - fmt.Printf("key: %d, value: %d\n", kv[0], kv[1]) -} -``` - -#### type Map - -```go -type Map struct { -} -``` - -Map is a map-like data-structure for int64s - -#### func New - -```go -func New(size int, fillFactor float64) *Map -``` -New returns a map initialized with n spaces and uses the stated fillFactor. The -map will grow as needed. - -#### func (*Map) Get - -```go -func (m *Map) Get(key int64) (int64, bool) -``` -Get returns the value if the key is found. - -#### func (*Map) Put - -```go -func (m *Map) Put(key int64, val int64) -``` -Put adds or updates key with value val. - -#### func (*Map) Del - -```go -func (m *Map) Del(key int64) -``` -Del deletes a key and its value. - -#### func (*Map) Keys - -```go -func (m *Map) Keys() chan int64 -``` -Keys returns a channel for iterating all keys. - -#### func (*Map) Items - -```go -func (m *Map) Items() chan [2]int64 -``` -Items returns a channel for iterating all key-value pairs. - - -#### func (*Map) Size - -```go -func (m *Map) Size() int -``` -Size returns size of the map. diff --git a/vendor/github.com/brentp/intintmap/intintmap.go b/vendor/github.com/brentp/intintmap/intintmap.go deleted file mode 100644 index d98353d54..000000000 --- a/vendor/github.com/brentp/intintmap/intintmap.go +++ /dev/null @@ -1,322 +0,0 @@ -// Package intintmap is a fast int64 key -> int64 value map. -// -// It is copied nearly verbatim from http://java-performance.info/implementing-world-fastest-java-int-to-int-hash-map/ -package intintmap - -import ( - "math" -) - -// INT_PHI is for scrambling the keys -const INT_PHI = 0x9E3779B9 - -// FREE_KEY is the 'free' key -const FREE_KEY = 0 - -func phiMix(x int64) int64 { - h := x * INT_PHI - return h ^ (h >> 16) -} - -// Map is a map-like data-structure for int64s -type Map struct { - data []int64 // interleaved keys and values - fillFactor float64 - threshold int // we will resize a map once it reaches this size - size int - - mask int64 // mask to calculate the original position - mask2 int64 - - hasFreeKey bool // do we have 'free' key in the map? - freeVal int64 // value of 'free' key -} - -func nextPowerOf2(x uint32) uint32 { - if x == math.MaxUint32 { - return x - } - - if x == 0 { - return 1 - } - - x-- - x |= x >> 1 - x |= x >> 2 - x |= x >> 4 - x |= x >> 8 - x |= x >> 16 - - return x + 1 -} - -func arraySize(exp int, fill float64) int { - s := nextPowerOf2(uint32(math.Ceil(float64(exp) / fill))) - if s < 2 { - s = 2 - } - return int(s) -} - -// New returns a map initialized with n spaces and uses the stated fillFactor. -// The map will grow as needed. -func New(size int, fillFactor float64) *Map { - if fillFactor <= 0 || fillFactor >= 1 { - panic("FillFactor must be in (0, 1)") - } - if size <= 0 { - panic("Size must be positive") - } - - capacity := arraySize(size, fillFactor) - return &Map{ - data: make([]int64, 2*capacity), - fillFactor: fillFactor, - threshold: int(math.Floor(float64(capacity) * fillFactor)), - mask: int64(capacity - 1), - mask2: int64(2*capacity - 1), - } -} - -// Get returns the value if the key is found. -func (m *Map) Get(key int64) (int64, bool) { - if key == FREE_KEY { - if m.hasFreeKey { - return m.freeVal, true - } - return 0, false - } - - ptr := (phiMix(key) & m.mask) << 1 - if ptr < 0 || ptr >= int64(len(m.data)) { // Check to help to compiler to eliminate a bounds check below. - return 0, false - } - k := m.data[ptr] - - if k == FREE_KEY { // end of chain already - return 0, false - } - if k == key { // we check FREE prior to this call - return m.data[ptr+1], true - } - - for { - ptr = (ptr + 2) & m.mask2 - k = m.data[ptr] - if k == FREE_KEY { - return 0, false - } - if k == key { - return m.data[ptr+1], true - } - } -} - -// Put adds or updates key with value val. -func (m *Map) Put(key int64, val int64) { - if key == FREE_KEY { - if !m.hasFreeKey { - m.size++ - } - m.hasFreeKey = true - m.freeVal = val - return - } - - ptr := (phiMix(key) & m.mask) << 1 - k := m.data[ptr] - - if k == FREE_KEY { // end of chain already - m.data[ptr] = key - m.data[ptr+1] = val - if m.size >= m.threshold { - m.rehash() - } else { - m.size++ - } - return - } else if k == key { // overwrite existed value - m.data[ptr+1] = val - return - } - - for { - ptr = (ptr + 2) & m.mask2 - k = m.data[ptr] - - if k == FREE_KEY { - m.data[ptr] = key - m.data[ptr+1] = val - if m.size >= m.threshold { - m.rehash() - } else { - m.size++ - } - return - } else if k == key { - m.data[ptr+1] = val - return - } - } - -} - -// Del deletes a key and its value. -func (m *Map) Del(key int64) { - if key == FREE_KEY { - m.hasFreeKey = false - m.size-- - return - } - - ptr := (phiMix(key) & m.mask) << 1 - k := m.data[ptr] - - if k == key { - m.shiftKeys(ptr) - m.size-- - return - } else if k == FREE_KEY { // end of chain already - return - } - - for { - ptr = (ptr + 2) & m.mask2 - k = m.data[ptr] - - if k == key { - m.shiftKeys(ptr) - m.size-- - return - } else if k == FREE_KEY { - return - } - - } -} - -func (m *Map) shiftKeys(pos int64) int64 { - // Shift entries with the same hash. - var last, slot int64 - var k int64 - var data = m.data - for { - last = pos - pos = (last + 2) & m.mask2 - for { - k = data[pos] - if k == FREE_KEY { - data[last] = FREE_KEY - return last - } - - slot = (phiMix(k) & m.mask) << 1 - if last <= pos { - if last >= slot || slot > pos { - break - } - } else { - if last >= slot && slot > pos { - break - } - } - pos = (pos + 2) & m.mask2 - } - data[last] = k - data[last+1] = data[pos+1] - } -} - -func (m *Map) rehash() { - newCapacity := len(m.data) * 2 - m.threshold = int(math.Floor(float64(newCapacity/2) * m.fillFactor)) - m.mask = int64(newCapacity/2 - 1) - m.mask2 = int64(newCapacity - 1) - - data := make([]int64, len(m.data)) // copy of original data - copy(data, m.data) - - m.data = make([]int64, newCapacity) - if m.hasFreeKey { // reset size - m.size = 1 - } else { - m.size = 0 - } - - var o int64 - for i := 0; i < len(data); i += 2 { - o = data[i] - if o != FREE_KEY { - m.Put(o, data[i+1]) - } - } -} - -// Size returns size of the map. -func (m *Map) Size() int { - return m.size -} - -// Keys returns a channel for iterating all keys. -func (m *Map) Keys() chan int64 { - c := make(chan int64, 10) - go func() { - data := m.data - var k int64 - - if m.hasFreeKey { - c <- FREE_KEY // value is m.freeVal - } - - for i := 0; i < len(data); i += 2 { - k = data[i] - if k == FREE_KEY { - continue - } - c <- k // value is data[i+1] - } - close(c) - }() - return c -} - -// Items returns a channel for iterating all key-value pairs. -func (m *Map) Items() chan [2]int64 { - c := make(chan [2]int64, 10) - go func() { - data := m.data - var k int64 - - if m.hasFreeKey { - c <- [2]int64{FREE_KEY, m.freeVal} - } - - for i := 0; i < len(data); i += 2 { - k = data[i] - if k == FREE_KEY { - continue - } - c <- [2]int64{k, data[i+1]} - } - close(c) - }() - return c -} - -func (m *Map) Each(f func(k, v int64)) { - data := m.data - var k int64 - - if m.hasFreeKey { - f(FREE_KEY, m.freeVal) - } - - for i := 0; i < len(data); i += 2 { - k = data[i] - if k == FREE_KEY { - continue - } - f(k, data[i+1]) - } -} diff --git a/vendor/modules.txt b/vendor/modules.txt index c154ec0c2..4d23195fb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -223,9 +223,6 @@ github.com/aws/aws-sdk-go/service/sso/ssoiface github.com/aws/aws-sdk-go/service/ssooidc github.com/aws/aws-sdk-go/service/sts github.com/aws/aws-sdk-go/service/sts/stsiface -# github.com/brentp/intintmap v0.0.0-20251106190759-56907b1f8479 -## explicit -github.com/brentp/intintmap # github.com/cenkalti/backoff/v4 v4.2.1 ## explicit; go 1.18 github.com/cenkalti/backoff/v4 From 1abfc0cef5254ad451d853aaa99971fed88c09b4 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Fri, 6 Feb 2026 09:44:17 -0500 Subject: [PATCH 11/13] Retain 50% of entries --- state/fingerprints.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/state/fingerprints.go b/state/fingerprints.go index 227fd0db8..3528223ee 100644 --- a/state/fingerprints.go +++ b/state/fingerprints.go @@ -54,7 +54,7 @@ func (c *Fingerprints) size() int { return len(c.cache) } -// Retains a random 33% sample of entries if the cache grows too large +// Retains a random 50% sample of entries if the cache grows too large func (c *Fingerprints) cleanup() { if c.size() < MAX_SIZE { return @@ -63,7 +63,7 @@ func (c *Fingerprints) cleanup() { cache := make(map[int64]int64, MAX_SIZE) index := 0 for key, value := range c.cache { - if index%3 == 0 { + if index%2 == 0 { cache[key] = value } index += 1 From a0daecdf65207f8a00dde8de18624ab0de3baaea Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Fri, 6 Feb 2026 09:49:57 -0500 Subject: [PATCH 12/13] Switch to unsigned int fingerprint since that's now possible --- input/postgres/statements.go | 2 +- output/transform/util.go | 2 +- state/fingerprints.go | 15 +++++++-------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/input/postgres/statements.go b/input/postgres/statements.go index ba432dc9b..1dec730ff 100644 --- a/input/postgres/statements.go +++ b/input/postgres/statements.go @@ -365,7 +365,7 @@ func fingerprintAndNormalize(c *Collection, key state.PostgresStatementKey, quer IgnoreIoTiming: ignoreIoTiming, } } else { - fp := uint64(c.Fingerprints.Add(queryID, text, c.Config.FilterQueryText, -1)) + fp := c.Fingerprints.Add(queryID, text, c.Config.FilterQueryText, -1) statements[key] = state.PostgresStatement{Fingerprint: fp, IgnoreIoTiming: ignoreIoTiming} _, ok := statementTextsByFp[fp] if !ok { diff --git a/output/transform/util.go b/output/transform/util.go index 035b35692..e92990610 100644 --- a/output/transform/util.go +++ b/output/transform/util.go @@ -65,7 +65,7 @@ func upsertQueryReferenceAndInformationSimple(server *state.Server, refs []*snap fingerprint := server.Fingerprints.Add(queryID, originalQuery, server.Config.FilterQueryText, trackActivityQuerySize) fpBuf := make([]byte, 8) - binary.BigEndian.PutUint64(fpBuf, uint64(fingerprint)) + binary.BigEndian.PutUint64(fpBuf, fingerprint) newRef := snapshot.QueryReference{ DatabaseIdx: databaseIdx, RoleIdx: roleIdx, diff --git a/state/fingerprints.go b/state/fingerprints.go index 3528223ee..31b4b5a5e 100644 --- a/state/fingerprints.go +++ b/state/fingerprints.go @@ -9,34 +9,33 @@ import ( const MAX_SIZE = 500_000 type Fingerprints struct { - cache map[int64]int64 + cache map[int64]uint64 lock sync.RWMutex } func NewFingerprints() *Fingerprints { return &Fingerprints{ - cache: make(map[int64]int64, MAX_SIZE), + cache: make(map[int64]uint64, MAX_SIZE), lock: sync.RWMutex{}, } } -func (c *Fingerprints) Get(queryID int64) (int64, bool) { +func (c *Fingerprints) Get(queryID int64) (uint64, bool) { c.lock.RLock() defer c.lock.RUnlock() fingerprint, exists := c.cache[queryID] return fingerprint, exists } -func (c *Fingerprints) Add(queryID int64, text string, filterQueryText string, trackActivityQuerySize int) int64 { +func (c *Fingerprints) Add(queryID int64, text string, filterQueryText string, trackActivityQuerySize int) uint64 { if queryID == 0 { - return int64(util.FingerprintQuery(text, filterQueryText, trackActivityQuerySize)) + return util.FingerprintQuery(text, filterQueryText, trackActivityQuerySize) } fingerprint, exists := c.Get(queryID) if exists { return fingerprint } - fp, virtual := util.TryFingerprintQuery(text, filterQueryText, trackActivityQuerySize) - fingerprint = int64(fp) + fingerprint, virtual := util.TryFingerprintQuery(text, filterQueryText, trackActivityQuerySize) if virtual { // Don't store virtual fingerprints so we can cache real fingerprints later return fingerprint @@ -60,7 +59,7 @@ func (c *Fingerprints) cleanup() { return } c.lock.Lock() - cache := make(map[int64]int64, MAX_SIZE) + cache := make(map[int64]uint64, MAX_SIZE) index := 0 for key, value := range c.cache { if index%2 == 0 { From a7be41394f58ebce3bb37b1be9faa23984ddf2ca Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Mon, 9 Feb 2026 12:26:18 -0500 Subject: [PATCH 13/13] Skip known query texts from pg_stat_statements --- input/postgres/statements.go | 12 ++++++++++-- output/full.go | 2 +- output/transform/postgres.go | 4 ++-- output/transform/postgres_statements.go | 11 +++++++---- output/transform/transform.go | 4 ++-- output/transform/util.go | 24 ++++++++++++++---------- state/fingerprints.go | 20 ++++++++++++++++---- util/fingerprint.go | 6 ++++++ 8 files changed, 58 insertions(+), 25 deletions(-) diff --git a/input/postgres/statements.go b/input/postgres/statements.go index 1dec730ff..b662ea9c4 100644 --- a/input/postgres/statements.go +++ b/input/postgres/statements.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/guregu/null" + "github.com/lib/pq" "github.com/pganalyze/collector/selftest" "github.com/pganalyze/collector/state" "github.com/pganalyze/collector/util" @@ -41,7 +42,8 @@ SELECT dbid, userid, queryid, %s, calls, %s, rows, shared_blks_hit, shared_blks_ const statementTextSQL string = ` SELECT dbid, userid, queryid, %s, query - FROM %s` + FROM %s + WHERE queryid = ANY($1)` const statementExtensionVersionSQL string = ` SELECT nspname, @@ -135,6 +137,7 @@ func GetStatementStats(ctx context.Context, c *Collection, db *sql.DB) (state.Po if queryID.Valid { key.QueryID = queryID.Int64 + c.Fingerprints.Add(queryID.Int64, "", "", -1) } else { // We can't process this entry, most likely a permission problem with reading the query ID continue @@ -153,6 +156,11 @@ func GetStatementStats(ctx context.Context, c *Collection, db *sql.DB) (state.Po } func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.PostgresStatementMap, state.PostgresStatementTextMap, error) { + queryIDs := c.Fingerprints.TakeNewQueryIDs() + if len(queryIDs) == 0 { + return nil, nil, nil + } + sourceTable, foundExtMinorVersion, err := getStatementSource(ctx, c, db, true) if err != nil { return nil, nil, err @@ -170,7 +178,7 @@ func GetStatementTexts(ctx context.Context, c *Collection, db *sql.DB) (state.Po } defer stmt.Close() - rows, err := stmt.QueryContext(ctx) + rows, err := stmt.QueryContext(ctx, pq.Array(queryIDs)) if err != nil { return nil, nil, err } diff --git a/output/full.go b/output/full.go index ae5f1bcea..69d951337 100644 --- a/output/full.go +++ b/output/full.go @@ -17,7 +17,7 @@ import ( ) func SendFull(ctx context.Context, server *state.Server, collectionOpts state.CollectionOpts, logger *util.Logger, newState state.PersistedState, diffState state.DiffState, transientState state.TransientState, collectedIntervalSecs uint32) error { - s := transform.StateToSnapshot(newState, diffState, transientState) + s := transform.StateToSnapshot(server, newState, diffState, transientState) s.CollectedIntervalSecs = collectedIntervalSecs err := verifyIntegrity(&s) if err != nil { diff --git a/output/transform/postgres.go b/output/transform/postgres.go index f5e6f3a87..9f1b6dc1d 100644 --- a/output/transform/postgres.go +++ b/output/transform/postgres.go @@ -10,7 +10,7 @@ import ( type OidToIdx map[state.Oid]int32 -func transformPostgres(s snapshot.FullSnapshot, newState state.PersistedState, diffState state.DiffState, transientState state.TransientState) snapshot.FullSnapshot { +func transformPostgres(server *state.Server, s snapshot.FullSnapshot, newState state.PersistedState, diffState state.DiffState, transientState state.TransientState) snapshot.FullSnapshot { s, roleOidToIdx := transformPostgresRoles(s, transientState) s, databaseOidToIdx := transformPostgresDatabases(s, diffState, transientState, roleOidToIdx) s, typeOidToIdx := transformPostgresTypes(s, transientState, databaseOidToIdx) @@ -19,7 +19,7 @@ func transformPostgres(s snapshot.FullSnapshot, newState state.PersistedState, d s = transformPostgresConfig(s, transientState) s = transformPostgresServerStats(s, newState, diffState, transientState) s = transformPostgresReplication(s, transientState, roleOidToIdx) - s, queryIDKeyToIdx := transformPostgresStatements(s, newState, diffState, transientState, roleOidToIdx, databaseOidToIdx) + s, queryIDKeyToIdx := transformPostgresStatements(server, s, newState, diffState, transientState, roleOidToIdx, databaseOidToIdx) s = transformPostgresPlans(s, newState, diffState, transientState, queryIDKeyToIdx) s = transformPostgresRelations(s, newState, diffState, databaseOidToIdx, typeOidToIdx, s.ServerStatistic.CurrentXactId) s = transformPostgresFunctions(s, newState, diffState, roleOidToIdx, databaseOidToIdx) diff --git a/output/transform/postgres_statements.go b/output/transform/postgres_statements.go index 784e2053f..1f5dbd263 100644 --- a/output/transform/postgres_statements.go +++ b/output/transform/postgres_statements.go @@ -10,11 +10,14 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -func groupStatements(statements state.PostgresStatementMap, statsMap state.DiffedPostgresStatementStatsMap) map[statementKey]statementValue { +func groupStatements(server *state.Server, statsMap state.DiffedPostgresStatementStatsMap) map[statementKey]statementValue { groupedStatements := make(map[statementKey]statementValue) for sKey, stats := range statsMap { - statement, exist := statements[sKey] + // TODO: what do do about the extra logic in fingerprintAndNormalize? track IO timing, detection of collector queries, etc + // Maybe those should all be implemented on the server side? + fingerprint, exist := server.Fingerprints.Get(sKey.QueryID) + statement := state.PostgresStatement{Fingerprint: fingerprint} if !exist { statement = state.PostgresStatement{QueryTextUnavailable: true, Fingerprint: util.FingerprintText(util.QueryTextUnavailable)} } @@ -76,7 +79,7 @@ type queryIDKey struct { } type QueryIDKeyToIdx map[queryIDKey]int32 -func transformPostgresStatements(s snapshot.FullSnapshot, newState state.PersistedState, diffState state.DiffState, transientState state.TransientState, roleOidToIdx OidToIdx, databaseOidToIdx OidToIdx) (snapshot.FullSnapshot, QueryIDKeyToIdx) { +func transformPostgresStatements(server *state.Server, s snapshot.FullSnapshot, newState state.PersistedState, diffState state.DiffState, transientState state.TransientState, roleOidToIdx OidToIdx, databaseOidToIdx OidToIdx) (snapshot.FullSnapshot, QueryIDKeyToIdx) { var queryStats []*snapshot.HistoricQueryStatistics queryIDKeyToIDx := make(QueryIDKeyToIdx) @@ -91,7 +94,7 @@ func transformPostgresStatements(s snapshot.FullSnapshot, newState state.Persist h.CollectedAt = timestamppb.New(timeKey.CollectedAt) h.CollectedIntervalSecs = timeKey.CollectedIntervalSecs - groupedStatements := groupStatements(transientState.Statements, diffedStats) + groupedStatements := groupStatements(server, diffedStats) for key, value := range groupedStatements { idx := upsertQueryReferenceAndInformation(&s, transientState.StatementTexts, roleOidToIdx, databaseOidToIdx, key, value) // Store the map of QueryIdx (idx here) and databaseOid, userOid, queryID combinations diff --git a/output/transform/transform.go b/output/transform/transform.go index b2b7669d5..3fd701d9c 100644 --- a/output/transform/transform.go +++ b/output/transform/transform.go @@ -5,10 +5,10 @@ import ( "github.com/pganalyze/collector/state" ) -func StateToSnapshot(newState state.PersistedState, diffState state.DiffState, transientState state.TransientState) snapshot.FullSnapshot { +func StateToSnapshot(server *state.Server, newState state.PersistedState, diffState state.DiffState, transientState state.TransientState) snapshot.FullSnapshot { var s snapshot.FullSnapshot - s = transformPostgres(s, newState, diffState, transientState) + s = transformPostgres(server, s, newState, diffState, transientState) s = systemStateToFullSnapshot(s, newState, diffState) s = transformCollectorStats(s, newState, diffState) s = transformCollectorPlatform(s, transientState) diff --git a/output/transform/util.go b/output/transform/util.go index e92990610..4333cad52 100644 --- a/output/transform/util.go +++ b/output/transform/util.go @@ -22,6 +22,12 @@ type statementValue struct { } func upsertQueryReferenceAndInformation(s *snapshot.FullSnapshot, statementTexts state.PostgresStatementTextMap, roleOidToIdx OidToIdx, databaseOidToIdx OidToIdx, key statementKey, value statementValue) int32 { + normalizedQuery, exists := statementTexts[key.fingerprint] + if !exists { + return -1 // A query that already exists in the fingerprint cache, so won't be submitted + // TODO: this doesn't work, since the protobuf QueryStatistic uses query index instead of fingerprint + } + fpBuf := make([]byte, 8) binary.BigEndian.PutUint64(fpBuf, key.fingerprint) newRef := snapshot.QueryReference{ @@ -41,16 +47,14 @@ func upsertQueryReferenceAndInformation(s *snapshot.FullSnapshot, statementTexts s.QueryReferences = append(s.QueryReferences, &newRef) // Information - normalizedQuery := "" - if value.statement.QueryTextUnavailable { - normalizedQuery = "" - } else if value.statement.InsufficientPrivilege { - normalizedQuery = "" - } else if value.statement.Collector { - normalizedQuery = "" - } else { - normalizedQuery = statementTexts[key.fingerprint] - } + // TODO: does this work now? + // if value.statement.QueryTextUnavailable { + // normalizedQuery = "" + // } else if value.statement.InsufficientPrivilege { + // normalizedQuery = "" + // } else if value.statement.Collector { + // normalizedQuery = "" + // } queryInformation := snapshot.QueryInformation{ QueryIdx: idx, NormalizedQuery: normalizedQuery, diff --git a/state/fingerprints.go b/state/fingerprints.go index 31b4b5a5e..f9d1f1033 100644 --- a/state/fingerprints.go +++ b/state/fingerprints.go @@ -9,14 +9,15 @@ import ( const MAX_SIZE = 500_000 type Fingerprints struct { - cache map[int64]uint64 - lock sync.RWMutex + lock sync.RWMutex + cache map[int64]uint64 + newQueryIDs []int64 } func NewFingerprints() *Fingerprints { return &Fingerprints{ - cache: make(map[int64]uint64, MAX_SIZE), lock: sync.RWMutex{}, + cache: make(map[int64]uint64, MAX_SIZE), } } @@ -37,7 +38,9 @@ func (c *Fingerprints) Add(queryID int64, text string, filterQueryText string, t } fingerprint, virtual := util.TryFingerprintQuery(text, filterQueryText, trackActivityQuerySize) if virtual { - // Don't store virtual fingerprints so we can cache real fingerprints later + c.lock.Lock() + c.newQueryIDs = append(c.newQueryIDs, queryID) + c.lock.Unlock() return fingerprint } c.cleanup() @@ -47,6 +50,15 @@ func (c *Fingerprints) Add(queryID int64, text string, filterQueryText string, t return fingerprint } +// Called by GetStatementTexts to only look up query texts for new, unknown query IDs +func (c *Fingerprints) TakeNewQueryIDs() []int64 { + c.lock.Lock() + defer c.lock.Unlock() + newQueryIDs := c.newQueryIDs + c.newQueryIDs = nil + return newQueryIDs +} + func (c *Fingerprints) size() int { c.lock.RLock() defer c.lock.RUnlock() diff --git a/util/fingerprint.go b/util/fingerprint.go index 11e580c83..b5884f71c 100644 --- a/util/fingerprint.go +++ b/util/fingerprint.go @@ -7,6 +7,12 @@ import ( // TryFingerprintQuery - Generates a unique fingerprint for the given query, // and whether the query text had to be massaged to generate a fingerprint func TryFingerprintQuery(query string, filterQueryText string, trackActivityQuerySize int) (fp uint64, virtual bool) { + if query == "" { + fp = FingerprintText(QueryTextUnavailable) + virtual = true + return + } + fp, err := pg_query.FingerprintToUInt64(query) if err != nil { virtual = true