diff --git a/deploy/kube/configmap.yaml b/deploy/kube/configmap.yaml index a87deaea6..e9aaba692 100644 --- a/deploy/kube/configmap.yaml +++ b/deploy/kube/configmap.yaml @@ -769,8 +769,11 @@ data: # reload_drain_timeout: 30s # # reload_rate_limit specifies the rate limit timeout duration to apply to the HTTP reload interface. # # The reload interface is disabled for this duration of time whenever a config reload request is - # # made that fails because the underlying config file is unmodified. default is 3 + # # made that fails because the underlying config file is unmodified. default is 3s # reload_rate_limit: 3s + # # auto_reload_interval controls how often Trickster checks the configuration file for changes. + # # A zero value disables automatic reloads. default is 0s + # auto_reload_interval: 10s # # config_handler_path provides the HTTP path to view a read-only printout of the running configuration # # which can be reached at http://your-trickster-endpoint:port/$config_handler_path diff --git a/docs/configuring.md b/docs/configuring.md index 02c05df25..a2dc5c4ea 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -98,7 +98,18 @@ Trickster can validate a configuration file by running `trickster -validate-conf Trickster can gracefully reload the configuration file from disk without impacting the uptime and responsiveness of the application. -Trickster provides 2 ways to reload the Trickster configuration: by requesting an HTTP endpoint, or by sending a SIGHUP (e.g., `kill -1 $TRICKSTER_PID`) to the Trickster process. In both cases, the underlying running Configuration File must have been modified such that the last modified time of the file is different than from when it was previously loaded. +Trickster supports manual reloads by requesting an HTTP endpoint or sending a SIGHUP (e.g., `kill -1 $TRICKSTER_PID`) to the Trickster process. It can also poll the file automatically. In all cases, the running configuration file must have been modified since it was loaded. + +### Automatic Config Reload + +Trickster can also poll the configuration file and reload it after a change. This is disabled by default. Set `mgmt.auto_reload_interval` to a positive duration to enable it: + +```yaml +mgmt: + auto_reload_interval: 10s +``` + +Polling uses the same validation and graceful reload path as SIGHUP and the management endpoint. The interval itself is reloadable, so a successful configuration update can change or disable automatic reloads. Polling is suitable for Kubernetes ConfigMap projected volumes, whose atomic symlink updates are not reliably represented as writes to the mounted file by filesystem notification APIs. ### Config Reload via SIGHUP diff --git a/examples/conf/example.full.yaml b/examples/conf/example.full.yaml index af764c39e..8a4f16930 100644 --- a/examples/conf/example.full.yaml +++ b/examples/conf/example.full.yaml @@ -765,8 +765,11 @@ backends: # reload_drain_timeout: 30s # # reload_rate_limit specifies the rate limit timeout duration to apply to the HTTP reload interface. # # The reload interface is disabled for this duration of time whenever a config reload request is -# # made that fails because the underlying config file is unmodified. default is 3 +# # made that fails because the underlying config file is unmodified. default is 3s # reload_rate_limit: 3s +# # auto_reload_interval controls how often Trickster checks the configuration file for changes. +# # A zero value disables automatic reloads. default is 0s +# auto_reload_interval: 10s # # config_handler_path provides the HTTP path to view a read-only printout of the running configuration # # which can be reached at http://your-trickster-endpoint:port/$config_handler_path diff --git a/pkg/config/config.go b/pkg/config/config.go index df16ac0a6..3056116e1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -197,6 +197,25 @@ func (c *Config) CheckFileLastModified() time.Time { return file.ModTime() } +// HasConfigChanged reports whether the configuration file has changed since it was loaded. +// Unlike IsStale, it does not apply or update the reload rate limiter. +func (c *Config) HasConfigChanged() bool { + if c == nil || c.Main == nil { + return false + } + c.Main.stalenessCheckLock.Lock() + defer c.Main.stalenessCheckLock.Unlock() + return c.hasConfigChanged() +} + +func (c *Config) hasConfigChanged() bool { + if c.Main.configFilePath == "" { + return false + } + t := c.CheckFileLastModified() + return !t.IsZero() && !t.Equal(c.Main.configLastModified) +} + // Process converts various raw config options into internal data structures // as needed func (c *Config) Process() error { @@ -308,10 +327,13 @@ func (c *Config) Clone() *Config { // IsStale returns true if the running config is stale versus the config on disk func (c *Config) IsStale() bool { + if c == nil || c.Main == nil { + return false + } c.Main.stalenessCheckLock.Lock() defer c.Main.stalenessCheckLock.Unlock() - if c.Main == nil || c.Main.configFilePath == "" || + if c.Main.configFilePath == "" || time.Now().Before(c.Main.configRateLimitTime) { return false } @@ -321,20 +343,18 @@ func (c *Config) IsStale() bool { } c.Main.configRateLimitTime = time.Now().Add(time.Duration(c.MgmtConfig.ReloadRateLimit)) - t := c.CheckFileLastModified() - if t.IsZero() { - return false - } - return !t.Equal(c.Main.configLastModified) + return c.hasConfigChanged() } // CheckAndMarkReloadInProgress checks if the config is stale and // marks it as being reloaded to prevent duplicate reloads. func (c *Config) CheckAndMarkReloadInProgress() bool { + if c == nil || c.Main == nil || c.Main.configFilePath == "" { + return false + } c.Main.stalenessCheckLock.Lock() defer c.Main.stalenessCheckLock.Unlock() - if c.Main == nil || c.Main.configFilePath == "" || - time.Now().Before(c.Main.configRateLimitTime) { + if time.Now().Before(c.Main.configRateLimitTime) { return false } if c.MgmtConfig == nil { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index cf7054b06..2c381a90a 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -27,6 +27,7 @@ import ( rule "github.com/trickstercache/trickster/v2/pkg/backends/rule/options" ct "github.com/trickstercache/trickster/v2/pkg/config/types" tracing "github.com/trickstercache/trickster/v2/pkg/observability/tracing/options" + "github.com/trickstercache/trickster/v2/pkg/parsing/timeconv" auth "github.com/trickstercache/trickster/v2/pkg/proxy/authenticator/options" "github.com/trickstercache/trickster/v2/pkg/proxy/headers" po "github.com/trickstercache/trickster/v2/pkg/proxy/paths/options" @@ -395,6 +396,104 @@ func TestCheckAndMarkReloadInProgress(t *testing.T) { } } +func TestHasConfigChangedDoesNotApplyRateLimit(t *testing.T) { + var nilConfig *Config + if nilConfig.HasConfigChanged() { + t.Error("nil config reported changed") + } + if NewConfig().HasConfigChanged() { + t.Error("config without a file path reported changed") + } + missingConfig := NewConfig() + missingConfig.Main.configFilePath = filepath.Join(t.TempDir(), "missing.yaml") + if missingConfig.HasConfigChanged() { + t.Error("missing config file reported changed") + } + + testFile := filepath.Join(t.TempDir(), "trickster_test.conf") + _, yml := emptyTestConfig() + if err := os.WriteFile(testFile, []byte(yml), 0o600); err != nil { + t.Fatal(err) + } + + c, err := Load([]string{"-config", testFile}) + if err != nil { + t.Fatal(err) + } + c.MgmtConfig.ReloadRateLimit = timeconv.Duration(time.Hour) + if c.HasConfigChanged() { + t.Fatal("freshly loaded config reported changed") + } + + if err := os.WriteFile(testFile, []byte(yml+"\n"), 0o600); err != nil { + t.Fatal(err) + } + modified := c.Main.configLastModified.Add(time.Second) + if err := os.Chtimes(testFile, modified, modified); err != nil { + t.Fatal(err) + } + if !c.HasConfigChanged() || !c.HasConfigChanged() { + t.Error("read-only change check was unexpectedly rate limited") + } +} + +func TestHasConfigChangedAfterProjectedVolumeSwap(t *testing.T) { + root := t.TempDir() + firstRevision := filepath.Join(root, "..2026_01") + secondRevision := filepath.Join(root, "..2026_02") + for _, dir := range []string{firstRevision, secondRevision} { + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + } + + const backendYAML = "backends:\n test:\n provider: test\n origin_url: http://1\n" + const firstYAML = backendYAML + "frontend:\n listen_port: 8480\n" + const secondYAML = backendYAML + "frontend:\n listen_port: 8481\n" + firstConfig := filepath.Join(firstRevision, "trickster.yaml") + secondConfig := filepath.Join(secondRevision, "trickster.yaml") + if err := os.WriteFile(firstConfig, []byte(firstYAML), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(secondConfig, []byte(secondYAML), 0o600); err != nil { + t.Fatal(err) + } + firstModified := time.Now().Add(-2 * time.Hour) + secondModified := firstModified.Add(time.Hour) + if err := os.Chtimes(firstConfig, firstModified, firstModified); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(secondConfig, secondModified, secondModified); err != nil { + t.Fatal(err) + } + + dataLink := filepath.Join(root, "..data") + if err := os.Symlink(firstRevision, dataLink); err != nil { + t.Skipf("symlinks are unavailable: %v", err) + } + configPath := filepath.Join(root, "trickster.yaml") + if err := os.Symlink(filepath.Join("..data", "trickster.yaml"), configPath); err != nil { + t.Skipf("symlinks are unavailable: %v", err) + } + + c, err := Load([]string{"-config", configPath}) + if err != nil { + t.Fatal(err) + } + if c.HasConfigChanged() { + t.Fatal("fresh projected config reported changed") + } + if err := os.Remove(dataLink); err != nil { + t.Fatal(err) + } + if err := os.Symlink(secondRevision, dataLink); err != nil { + t.Fatal(err) + } + if !c.HasConfigChanged() { + t.Error("projected volume symlink swap was not detected") + } +} + func TestConfigFilePath(t *testing.T) { c, _ := emptyTestConfig() diff --git a/pkg/config/mgmt/mgmt.go b/pkg/config/mgmt/mgmt.go index 24e0e9d9b..3df216b90 100644 --- a/pkg/config/mgmt/mgmt.go +++ b/pkg/config/mgmt/mgmt.go @@ -59,6 +59,9 @@ type Options struct { // This prevents a bad actor from stating the config file with millions of concurrent requests // The rate limit does not apply to SIGHUP-based reload requests ReloadRateLimit timeconv.Duration `yaml:"reload_rate_limit,omitempty"` + // AutoReloadInterval controls how often Trickster checks the configuration file for + // changes. A zero value disables automatic reloads. + AutoReloadInterval timeconv.Duration `yaml:"auto_reload_interval,omitempty"` } // ErrInvalidPprofListenerName returns an error for invalid pprof listener name @@ -67,6 +70,9 @@ var ErrInvalidPprofListenerName = errors.New("invalid pprof listener name") // ErrInvalidConfigHandlerListenerName returns an error for an invalid config handler listener name var ErrInvalidConfigHandlerListenerName = errors.New("invalid config handler listener name") +// ErrInvalidAutoReloadInterval indicates that the configured interval is negative. +var ErrInvalidAutoReloadInterval = errors.New("auto reload interval cannot be negative") + // New returns a new Options references with Default Values set func New() *Options { return &Options{ @@ -86,6 +92,10 @@ func New() *Options { } func (o *Options) Validate() error { + if o.AutoReloadInterval < 0 { + return ErrInvalidAutoReloadInterval + } + switch o.ConfigHandlerListener { case ListenerNameMetrics, ListenerNameMgmt, ListenerNameOff, ListenerNameBoth: case "": diff --git a/pkg/config/mgmt/mgmt_test.go b/pkg/config/mgmt/mgmt_test.go index 060e6556d..2d326ea65 100644 --- a/pkg/config/mgmt/mgmt_test.go +++ b/pkg/config/mgmt/mgmt_test.go @@ -16,7 +16,15 @@ package mgmt -import "testing" +import ( + "errors" + "testing" + "time" + + "github.com/trickstercache/trickster/v2/pkg/parsing/timeconv" + + "gopkg.in/yaml.v2" +) func TestValidate(t *testing.T) { c := New() @@ -69,6 +77,39 @@ func TestValidatePprofListenerNames(t *testing.T) { t.Errorf("expected pprof listener name %q to be invalid, got %v", name, err) } } + + c := New() + c.AutoReloadInterval = timeconv.Duration(-time.Second) + if err := c.Validate(); !errors.Is(err, ErrInvalidAutoReloadInterval) { + t.Errorf("error = %v; want %v", err, ErrInvalidAutoReloadInterval) + } +} + +func TestReloadOptionsYAML(t *testing.T) { + o := New() + const yml = `reload_handler_path: /reload +reload_drain_timeout: 17s +reload_rate_limit: 2s +auto_reload_interval: 10s +` + if err := yaml.Unmarshal([]byte(yml), o); err != nil { + t.Fatal(err) + } + if o.ReloadHandlerPath != "/reload" { + t.Errorf("reload handler path = %q; want %q", o.ReloadHandlerPath, "/reload") + } + if o.ReloadDrainTimeout != timeconv.Duration(17*time.Second) { + t.Errorf("reload drain timeout = %v; want %v", o.ReloadDrainTimeout, 17*time.Second) + } + if o.ReloadRateLimit != timeconv.Duration(2*time.Second) { + t.Errorf("reload rate limit = %v; want %v", o.ReloadRateLimit, 2*time.Second) + } + if o.AutoReloadInterval != timeconv.Duration(10*time.Second) { + t.Errorf("auto reload interval = %v; want %v", o.AutoReloadInterval, 10*time.Second) + } + if got := o.Clone().AutoReloadInterval; got != o.AutoReloadInterval { + t.Errorf("cloned auto reload interval = %v; want %v", got, o.AutoReloadInterval) + } } func TestClone(t *testing.T) { diff --git a/pkg/daemon/auto_reload.go b/pkg/daemon/auto_reload.go new file mode 100644 index 000000000..58ac18698 --- /dev/null +++ b/pkg/daemon/auto_reload.go @@ -0,0 +1,146 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package daemon + +import ( + "context" + "sync" + "time" + + "github.com/trickstercache/trickster/v2/pkg/config" + "github.com/trickstercache/trickster/v2/pkg/config/reload" + "github.com/trickstercache/trickster/v2/pkg/daemon/instance" + "github.com/trickstercache/trickster/v2/pkg/util/safego" +) + +const autoReloadSource = "auto-reload" + +type autoReloadSettings struct { + interval time.Duration + hasChanged func() bool +} + +type autoReloader struct { + reloader reload.Reloader + + settingsLock sync.Mutex + settings autoReloadSettings + wake chan struct{} + cancel context.CancelFunc + done chan struct{} + closeOnce sync.Once +} + +func newAutoReloader(parent context.Context, reloader reload.Reloader) *autoReloader { + ctx, cancel := context.WithCancel(parent) // #nosec G118 -- Close calls the retained cancel function + r := &autoReloader{ + reloader: reloader, + wake: make(chan struct{}, 1), + cancel: cancel, + done: make(chan struct{}), + } + safego.Go(reloadGoroutinePanic("autoReloader", autoReloadSource), func() { + defer close(r.done) + r.run(ctx) + }) + return r +} + +func bindAutoReloader(ctx context.Context, si *instance.ServerInstance, + reloader reload.Reloader, +) *autoReloader { + r := newAutoReloader(ctx, reloader) + si.OnConfigReloaded = r.Update + return r +} + +func notifyAutoReloader(si *instance.ServerInstance) { + if si.OnConfigReloaded != nil { + si.OnConfigReloaded(si.Config) + } +} + +func (r *autoReloader) Update(c *config.Config) { + settings := autoReloadSettings{} + if c != nil && c.MgmtConfig != nil { + settings.interval = time.Duration(c.MgmtConfig.AutoReloadInterval) + settings.hasChanged = c.HasConfigChanged + } + r.updateSettings(settings) +} + +func (r *autoReloader) updateSettings(settings autoReloadSettings) { + r.settingsLock.Lock() + r.settings = settings + r.settingsLock.Unlock() + select { + case r.wake <- struct{}{}: + default: + } +} + +func (r *autoReloader) currentSettings() autoReloadSettings { + r.settingsLock.Lock() + defer r.settingsLock.Unlock() + return r.settings +} + +func (r *autoReloader) Close() { + r.closeOnce.Do(r.cancel) + <-r.done +} + +func (r *autoReloader) run(ctx context.Context) { + var settings autoReloadSettings + var timer *time.Timer + var timerC <-chan time.Time + defer func() { stopAutoReloadTimer(timer) }() + + for { + select { + case <-ctx.Done(): + return + case <-r.wake: + settings = r.currentSettings() + timer, timerC = resetAutoReloadTimer(timer, settings.interval) + case <-timerC: + if settings.hasChanged != nil && settings.hasChanged() { + _, _ = r.reloader(autoReloadSource) + } + timer, timerC = resetAutoReloadTimer(timer, settings.interval) + } + } +} + +func resetAutoReloadTimer(timer *time.Timer, interval time.Duration) (*time.Timer, <-chan time.Time) { + stopAutoReloadTimer(timer) + if interval <= 0 { + return nil, nil + } + timer = time.NewTimer(interval) + return timer, timer.C +} + +func stopAutoReloadTimer(timer *time.Timer) { + if timer == nil || timer.Stop() { + return + } + select { + case <-timer.C: + default: + } +} diff --git a/pkg/daemon/auto_reload_test.go b/pkg/daemon/auto_reload_test.go new file mode 100644 index 000000000..3d95390ba --- /dev/null +++ b/pkg/daemon/auto_reload_test.go @@ -0,0 +1,128 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package daemon + +import ( + "context" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/trickstercache/trickster/v2/pkg/config" + "github.com/trickstercache/trickster/v2/pkg/daemon/instance" + "github.com/trickstercache/trickster/v2/pkg/parsing/timeconv" +) + +func TestAutoReloader(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + var calls atomic.Int32 + var source atomic.Value + r := newAutoReloader(ctx, func(gotSource string) (bool, error) { + source.Store(gotSource) + calls.Add(1) + return true, nil + }) + + var changed atomic.Bool + changed.Store(true) + r.updateSettings(autoReloadSettings{ + interval: time.Hour, + hasChanged: changed.Load, + }) + synctest.Wait() + if got := calls.Load(); got != 0 { + t.Fatalf("reload calls before interval = %d; want 0", got) + } + + time.Sleep(time.Hour) + synctest.Wait() + if got := calls.Load(); got != 1 { + t.Fatalf("reload calls after interval = %d; want 1", got) + } + if got, _ := source.Load().(string); got != autoReloadSource { + t.Errorf("reload source = %q; want %q", got, autoReloadSource) + } + + changed.Store(false) + time.Sleep(time.Hour) + synctest.Wait() + if got := calls.Load(); got != 1 { + t.Errorf("reload calls for unchanged config = %d; want 1", got) + } + + r.updateSettings(autoReloadSettings{ + interval: 2 * time.Hour, + hasChanged: changed.Load, + }) + synctest.Wait() + changed.Store(true) + time.Sleep(time.Hour) + synctest.Wait() + if got := calls.Load(); got != 1 { + t.Errorf("reload calls before updated interval = %d; want 1", got) + } + time.Sleep(time.Hour) + synctest.Wait() + if got := calls.Load(); got != 2 { + t.Errorf("reload calls after updated interval = %d; want 2", got) + } + + r.updateSettings(autoReloadSettings{}) + synctest.Wait() + time.Sleep(24 * time.Hour) + synctest.Wait() + if got := calls.Load(); got != 2 { + t.Errorf("reload calls while disabled = %d; want 2", got) + } + + cancel() + r.Close() + r.Close() + }) +} + +func TestAutoReloaderInstanceHooks(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + si := &instance.ServerInstance{} + r := bindAutoReloader(ctx, si, func(string) (bool, error) { return false, nil }) + defer r.Close() + + c := config.NewConfig() + c.MgmtConfig.AutoReloadInterval = timeconv.Duration(10 * time.Second) + si.Config = c + notifyAutoReloader(si) + settings := r.currentSettings() + if settings.interval != 10*time.Second || settings.hasChanged == nil { + t.Fatalf("settings = %#v; want configured interval and change callback", settings) + } + if settings.hasChanged() { + t.Error("config without a file path reported changed") + } + + si.Config = &config.Config{} + notifyAutoReloader(si) + settings = r.currentSettings() + if settings.interval != 0 || settings.hasChanged != nil { + t.Errorf("settings = %#v; want disabled settings", settings) + } + + si.OnConfigReloaded = nil + notifyAutoReloader(si) +} diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 8659f3bf8..f7bd682e8 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -98,6 +98,8 @@ func Start(ctx context.Context, args ...string) error { Listeners: listener.NewGroup(), } hupFunc := newHupFunc(si, args) + autoReloader := bindAutoReloader(ctx, si, hupFunc) + defer autoReloader.Close() // Serve with Config err = setup.ApplyConfig(si, conf, clients, hupFunc, func() { os.Exit(1) }, si.Listeners) if err != nil { @@ -116,10 +118,12 @@ func Start(ctx context.Context, args ...string) error { logger.Info("all listeners ready", nil) } } + autoReloader.Update(conf) skipUnlock = true mtx.Unlock() signaling.Wait(ctx, hupFunc) + autoReloader.Close() if si.Listeners != nil { si.Listeners.Shutdown(0) } @@ -222,6 +226,7 @@ func Hup(si *instance.ServerInstance, source string, args ...string) (bool, erro metrics.ReloadDurationSeconds.Observe(time.Since(startTime).Seconds()) logger.Info(reload.ConfigReloadedText, logging.Pairs{"source": source}) + notifyAutoReloader(si) return true, nil } diff --git a/pkg/daemon/instance/instance.go b/pkg/daemon/instance/instance.go index 820a95c75..d4d856675 100644 --- a/pkg/daemon/instance/instance.go +++ b/pkg/daemon/instance/instance.go @@ -25,9 +25,10 @@ import ( ) type ServerInstance struct { - Config *config.Config - Caches cache.Lookup - HealthChecker healthcheck.HealthChecker - Backends backends.Backends - Listeners *listener.Group + Config *config.Config + Caches cache.Lookup + HealthChecker healthcheck.HealthChecker + Backends backends.Backends + Listeners *listener.Group + OnConfigReloaded func(*config.Config) }