Skip to content

Commit 3cbab7e

Browse files
otroanclaude
andcommitted
adapter/statsclient: read each symlink target once when refreshing a dir
VPP exposes some counters only as one vector plus a set of symlinks naming its items. /node/errors is the case that hurts: it is a single counter vector, and every /err/<node>/<reason> is a symlink into one item of it. A capture from a live device has 4223 such symlinks over a 4223-item vector. UpdateDir resolved each of those individually, and resolving a symlink reads its whole backing vector - so refreshing a prepared dir over that fan re-read the same vector 4223 times. Group the symlinks in a prepared dir by the entry they alias, read each target once, and fan its items out into the prepared entries. The fan-out writes through the slices the entries already hold, so it does not allocate per symlink, and it produces exactly the shape resolving each symlink would: one value per worker thread. Anything the fan-out does not recognise falls back to resolving that symlink on its own, as does segment v1, which has no symlink target encoding. This stays entirely inside statsclient. An earlier version of this change exposed the (target, item) pair as a public ListSymlinks API so a caller could read the vector and label its items itself; doing the grouping here gives the same performance model with no API change at all, as suggested in review. Two allocation sources found on the way, both paid per symlink per refresh: storing the fanned-out slice back into the adapter.Stat interface boxed it, and getSymlinkIndexes serialised the union through a bytes.Buffer to split one uint64 into two uint32s. Refreshing 128 symlinks went from 654 allocations to 142 - the remainder being one name clone per prepared entry. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent e1718f5 commit 3cbab7e

8 files changed

Lines changed: 447 additions & 35 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,24 @@ This file lists changes for the GoVPP releases.
1111
-
1212
-->
1313

14+
## Unreleased
15+
16+
### Fixes
17+
18+
- adapter/statsclient: `UpdateDir` now refreshes symlink entries. Their resolved
19+
type never equals the directory type, so the type check skipped them and a
20+
prepared dir kept returning its `PrepareDir` values indefinitely.
21+
- adapter/statsclient: `UpdateDir` compares the prepared dir against the epoch
22+
`accessStart` settled on, closing a window where a directory re-layout between
23+
the two epoch reads went undetected.
24+
25+
### Other
26+
27+
- adapter/statsclient: `UpdateDir` resolves symlinks by target group, reading
28+
each aliased counter vector once instead of once per symlink naming it. No API
29+
change; a prepared dir over a large symlink fan simply refreshes far more
30+
cheaply.
31+
1432
## 0.13.0
1533

1634
> _13 November 2025_

‎adapter/statsclient/stat_segment_api.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// Copyright (c) 2020 Cisco and/or its affiliates.
2+
// Copyright (c) 2026 Meter, Inc.
23
//
34
// Licensed under the Apache License, Version 2.0 (the "License");
45
// you may not use this file except in compliance with the License.
@@ -93,6 +94,12 @@ type statSegment interface {
9394
// Use ^uint32(0) as an empty index (since 0 is a valid value).
9495
CopyEntryData(segment dirSegment, index uint32) adapter.Stat
9596

97+
// GetSymlinkIndexes returns, for a symlink directory segment, the directory
98+
// index of the entry it aliases and the item index within that entry.
99+
// ok is false if the segment is not a symlink, or if the segment version has
100+
// no notion of symlinks.
101+
GetSymlinkIndexes(segment dirSegment) (targetIndex, itemIndex uint32, ok bool)
102+
96103
// UpdateEntryData accepts pointer to a directory segment with data, and stat
97104
// segment to update
98105
UpdateEntryData(segment dirSegment, s *adapter.Stat) error

‎adapter/statsclient/statsclient.go‎

Lines changed: 127 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -299,10 +299,27 @@ func (sc *StatsClient) UpdateDir(dir *adapter.StatDir) (err error) {
299299
if dirVector == nil {
300300
return fmt.Errorf("failed to update dir: directory vector is nil")
301301
}
302+
// Symlinks are not refreshed one by one. VPP names every item of a counter
303+
// vector with its own symlink - /node/errors and its thousands of
304+
// /err/<node>/<reason> aliases being the case that hurts - so resolving them
305+
// individually re-reads the same backing vector once per item. Collect them
306+
// by target instead and read each target once, below.
307+
var symlinks map[uint32][]symlinkRef
302308
for i := 0; i < len(dir.Entries); i++ {
303-
if err := sc.updateStatOnIndex(&dir.Entries[i], dirVector); err != nil {
309+
ref, isSymlink, err := sc.updateStatOnIndex(&dir.Entries[i], dirVector)
310+
if err != nil {
304311
return err
305312
}
313+
if !isSymlink {
314+
continue
315+
}
316+
if symlinks == nil {
317+
symlinks = make(map[uint32][]symlinkRef)
318+
}
319+
symlinks[ref.target] = append(symlinks[ref.target], ref)
320+
}
321+
if err := sc.updateSymlinkGroups(dirVector, symlinks); err != nil {
322+
return err
306323
}
307324
if !sc.accessEnd(accessEpoch) {
308325
return adapter.ErrStatsDataBusy
@@ -627,32 +644,127 @@ func (sc *StatsClient) isConnected() bool {
627644
}
628645

629646
// updateStatOnIndex refreshes the entry data.
630-
func (sc *StatsClient) updateStatOnIndex(entry *adapter.StatEntry, vector dirVector) (err error) {
647+
// symlinkRef is a prepared symlink entry and the counter it aliases, held until
648+
// its target can be read once for the whole group.
649+
type symlinkRef struct {
650+
entry *adapter.StatEntry
651+
dirPtr dirSegment // the symlink's own directory segment, for the fallback
652+
target uint32 // directory index of the aliased entry
653+
item uint32 // item within that entry
654+
}
655+
656+
// updateStatOnIndex refreshes one entry. A symlink is not resolved here: it is
657+
// reported to the caller so that every symlink sharing a target can be served
658+
// from a single read of it.
659+
func (sc *StatsClient) updateStatOnIndex(entry *adapter.StatEntry, vector dirVector) (ref symlinkRef, isSymlink bool, err error) {
631660
dirLen := *(*uint32)(vectorLen(vector))
632661
if entry.Index >= dirLen {
633-
return fmt.Errorf("stat entry index %d out of dir vector length (%d)", entry.Index, dirLen)
662+
return ref, false, fmt.Errorf("stat entry index %d out of dir vector length (%d)", entry.Index, dirLen)
634663
}
635664
dirPtr, dirName, dirType := sc.GetStatDirOnIndex(vector, entry.Index)
636665
// Identity is the name; if it no longer matches, the directory changed under us
637666
// (the epoch check in UpdateDir normally catches this first).
638667
if len(dirName) == 0 || !bytes.Equal(dirName, entry.Name) || entry.Data == nil {
639-
return nil
668+
return ref, false, nil
640669
}
641670
if dirType == adapter.Symlink {
642-
// A symlink's directory entry holds (target, item) indexes rather than a data
643-
// pointer, so its resolved Type never equals dirType and the type check below
644-
// would skip it, leaving the entry frozen at its PrepareDir value forever.
645-
// Re-resolve through the symlink instead. This allocates, unlike the in-place
646-
// UpdateEntryData path, because the resolved item does not have a stable
647-
// backing slice to write into.
648-
entry.Data = sc.CopyEntryData(dirPtr, ^uint32(0))
649-
return nil
671+
// A symlink's directory entry holds (target, item) indexes rather than a
672+
// data pointer, so its resolved Type never equals dirType and the type
673+
// check below would skip it, leaving the entry frozen at its PrepareDir
674+
// value forever.
675+
target, item, ok := sc.GetSymlinkIndexes(dirPtr)
676+
if !ok {
677+
// No target encoding (segment v1): resolve it on its own.
678+
entry.Data = sc.CopyEntryData(dirPtr, ^uint32(0))
679+
return ref, false, nil
680+
}
681+
return symlinkRef{entry: entry, dirPtr: dirPtr, target: target, item: item}, true, nil
650682
}
651683
if dirType != entry.Type {
652-
return nil
684+
return ref, false, nil
653685
}
654686
if err := sc.UpdateEntryData(dirPtr, &entry.Data); err != nil {
655-
return fmt.Errorf("updating stat data for entry %s failed: %v", dirName, err)
687+
return ref, false, fmt.Errorf("updating stat data for entry %s failed: %v", dirName, err)
688+
}
689+
return ref, false, nil
690+
}
691+
692+
// updateSymlinkGroups reads each aliased entry once and fans its items out to
693+
// the symlinks that name them. Reading the target once is the whole point: a
694+
// vector of n items aliased by n symlinks costs one read here rather than n.
695+
func (sc *StatsClient) updateSymlinkGroups(vector dirVector, groups map[uint32][]symlinkRef) error {
696+
if len(groups) == 0 {
697+
return nil
698+
}
699+
dirLen := *(*uint32)(vectorLen(vector))
700+
for target, refs := range groups {
701+
if target >= dirLen {
702+
debugf("symlink target index %d out of dir vector length (%d)", target, dirLen)
703+
continue
704+
}
705+
targetPtr, targetName, _ := sc.GetStatDirOnIndex(vector, target)
706+
if len(targetName) == 0 {
707+
continue
708+
}
709+
full := sc.CopyEntryData(targetPtr, ^uint32(0))
710+
for _, ref := range refs {
711+
if !symlinkItem(full, ref.item, &ref.entry.Data) {
712+
// A shape this cannot fan out - resolve the symlink on its own
713+
// so behaviour matches the one-at-a-time path exactly.
714+
ref.entry.Data = sc.CopyEntryData(ref.dirPtr, ^uint32(0))
715+
}
716+
}
717+
}
718+
return nil
719+
}
720+
721+
// symlinkItem extracts one item from an already-read counter vector into dst, in
722+
// the same shape resolving the symlink directly would produce: one value per
723+
// worker thread.
724+
//
725+
// It writes THROUGH dst rather than returning a new value, and only stores back
726+
// when the outer slice had to be reallocated. Storing a slice into an
727+
// adapter.Stat boxes it, which allocates - once per symlink per tick, for a
728+
// value that is usually identical to the one already there.
729+
//
730+
// Reports false for anything it cannot fan out, so the caller can fall back.
731+
func symlinkItem(full adapter.Stat, item uint32, dst *adapter.Stat) bool {
732+
switch d := full.(type) {
733+
case adapter.SimpleCounterStat:
734+
out, ok := (*dst).(adapter.SimpleCounterStat)
735+
if !ok || len(out) != len(d) {
736+
out = make(adapter.SimpleCounterStat, len(d))
737+
*dst = out
738+
}
739+
for i, worker := range d {
740+
if len(out[i]) != 1 {
741+
out[i] = make([]adapter.Counter, 1)
742+
}
743+
if int(item) < len(worker) {
744+
out[i][0] = worker[item]
745+
} else {
746+
out[i][0] = 0
747+
}
748+
}
749+
return true
750+
751+
case adapter.CombinedCounterStat:
752+
out, ok := (*dst).(adapter.CombinedCounterStat)
753+
if !ok || len(out) != len(d) {
754+
out = make(adapter.CombinedCounterStat, len(d))
755+
*dst = out
756+
}
757+
for i, worker := range d {
758+
if len(out[i]) != 1 {
759+
out[i] = make([]adapter.CombinedCounter, 1)
760+
}
761+
if int(item) < len(worker) {
762+
out[i][0] = worker[item]
763+
} else {
764+
out[i][0] = adapter.CombinedCounter{}
765+
}
766+
}
767+
return true
656768
}
657-
return
769+
return false
658770
}
Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
// Copyright (c) 2026 Meter, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at:
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package statsclient
16+
17+
import (
18+
"sync/atomic"
19+
"testing"
20+
21+
"go.fd.io/govpp/adapter"
22+
)
23+
24+
// countingSegment counts data reads so a test can assert that a symlink fan
25+
// costs ONE read of its target rather than one per symlink.
26+
type countingSegment struct {
27+
statSegment
28+
copies int64
29+
}
30+
31+
func (c *countingSegment) CopyEntryData(segment dirSegment, index uint32) adapter.Stat {
32+
atomic.AddInt64(&c.copies, 1)
33+
return c.statSegment.CopyEntryData(segment, index)
34+
}
35+
36+
func countingClient(f *fakeSegment) (*StatsClient, *countingSegment) {
37+
sc := f.client()
38+
c := &countingSegment{statSegment: sc.statSegment}
39+
sc.statSegment = c
40+
return sc, c
41+
}
42+
43+
// TestUpdateDirReadsSymlinkTargetOnce is the property the whole grouping exists
44+
// for. VPP names every item of /node/errors with its own /err/<node>/<reason>
45+
// symlink, so resolving them one at a time re-reads the same backing vector once
46+
// per item - thousands of times per refresh on a real box.
47+
func TestUpdateDirReadsSymlinkTargetOnce(t *testing.T) {
48+
const n = 64
49+
f := newFakeSegment(t, fakeGroupValues(n))
50+
sc, counter := countingClient(f)
51+
52+
dir, err := sc.PrepareDir("^/err/")
53+
if err != nil {
54+
t.Fatal("PrepareDir failed:", err)
55+
}
56+
if len(dir.Entries) != n {
57+
t.Fatalf("prepared %d entries, want %d symlinks", len(dir.Entries), n)
58+
}
59+
60+
atomic.StoreInt64(&counter.copies, 0)
61+
if err := sc.UpdateDir(dir); err != nil {
62+
t.Fatal("UpdateDir failed:", err)
63+
}
64+
65+
// All n symlinks alias the same vector, so one read serves them all.
66+
if got := atomic.LoadInt64(&counter.copies); got != 1 {
67+
t.Errorf("refreshing %d symlinks over one target did %d data reads, want 1", n, got)
68+
}
69+
}
70+
71+
// TestUpdateDirGroupedMatchesIndividual pins the fan-out against the
72+
// one-at-a-time resolution it replaces: same values, same shape. An off-by-one
73+
// here would mislabel every error counter while looking entirely plausible.
74+
func TestUpdateDirGroupedMatchesIndividual(t *testing.T) {
75+
values := fakeGroupValues(24)
76+
f := newFakeSegment(t, values)
77+
sc := f.client()
78+
79+
dir, err := sc.PrepareDir("^/err/")
80+
if err != nil {
81+
t.Fatal("PrepareDir failed:", err)
82+
}
83+
// Move the counters so the refresh has to do real work.
84+
for i := range values {
85+
values[i] += 7777
86+
f.setCounter(i, values[i])
87+
}
88+
if err := sc.UpdateDir(dir); err != nil {
89+
t.Fatal("UpdateDir failed:", err)
90+
}
91+
92+
// Reference: resolve each symlink on its own, as DumpStats does.
93+
individual, err := sc.DumpStats("^/err/")
94+
if err != nil {
95+
t.Fatal("DumpStats failed:", err)
96+
}
97+
want := make(map[string]uint64, len(individual))
98+
for _, e := range individual {
99+
want[string(e.Name)] = symlinkValue(t, e)
100+
}
101+
102+
for _, e := range dir.Entries {
103+
got := symlinkValue(t, e)
104+
if w, ok := want[string(e.Name)]; !ok {
105+
t.Errorf("%s: not returned by DumpStats", e.Name)
106+
} else if got != w {
107+
t.Errorf("%s: grouped refresh read %d, individual resolution %d", e.Name, got, w)
108+
}
109+
if item := fakeErrItem(t, e.Name); got != values[item] {
110+
t.Errorf("%s: read %d, backing item %d holds %d", e.Name, got, item, values[item])
111+
}
112+
}
113+
}
114+
115+
// TestUpdateDirSymlinkRefreshDoesNotAllocate — the fan-out writes into the
116+
// slices the prepared entries already hold.
117+
func TestUpdateDirSymlinkRefreshDoesNotAllocate(t *testing.T) {
118+
f := newFakeSegment(t, fakeGroupValues(128))
119+
sc := f.client()
120+
121+
dir, err := sc.PrepareDir("^/err/")
122+
if err != nil {
123+
t.Fatal("PrepareDir failed:", err)
124+
}
125+
if err := sc.UpdateDir(dir); err != nil { // warm up
126+
t.Fatal(err)
127+
}
128+
allocs := testing.AllocsPerRun(20, func() {
129+
if err := sc.UpdateDir(dir); err != nil {
130+
t.Fatal(err)
131+
}
132+
})
133+
// What remains is one name clone per prepared entry, from GetStatDirOnIndex
134+
// in updateStatOnIndex, plus one read of the target vector. The fan-out
135+
// itself must add nothing per symlink: it writes into the slices the
136+
// prepared entries already hold.
137+
const n = 128
138+
if allocs > n+40 {
139+
t.Errorf("refreshing %d symlinks allocated %.0f times, want at most one per entry "+
140+
"(the name clone) plus the target read; the fan-out is not reusing the prepared slices",
141+
n, allocs)
142+
}
143+
t.Logf("allocations per refresh of %d symlinks: %.0f", n, allocs)
144+
}
145+
146+
func fakeGroupValues(n int) []uint64 {
147+
v := make([]uint64, n)
148+
for i := range v {
149+
v[i] = uint64(i)*13 + 500
150+
}
151+
return v
152+
}
153+
154+
// BenchmarkUpdateDirSymlinkFan measures the refresh at the fan size a real box
155+
// carries: a capture from a live device has 4223 /err/* symlinks over a
156+
// 4223-item /node/errors.
157+
func BenchmarkUpdateDirSymlinkFan(b *testing.B) {
158+
f := newFakeSegment(b, fakeGroupValues(4223))
159+
sc := f.client()
160+
dir, err := sc.PrepareDir("^/err/")
161+
if err != nil {
162+
b.Fatal(err)
163+
}
164+
b.ReportAllocs()
165+
b.ResetTimer()
166+
for i := 0; i < b.N; i++ {
167+
if err := sc.UpdateDir(dir); err != nil {
168+
b.Fatal(err)
169+
}
170+
}
171+
}

0 commit comments

Comments
 (0)