caddytls: synchronize storage cleaner with TLS.Stop via context and WaitGroup (#7954)
* caddytls: synchronize storage cleaner with TLS.Stop via context and WaitGroup * Unify the storage cleanup synchronization to both tls and ech. * Cannot embed the sync.WaitGroup directly as the TLS struct is copied. * caddytls: propagate cancellable context to ECH rotation and add sync tests Pass cancellable context to ECH key rotation so in-flight storage locks and operations unblock when TLS.Stop is invoked. Add comprehensive tests verifying TLS.Stop cleanly unblocks and waits for storage cleaner and ECH workers. --------- Co-authored-by: Zen Dodd <mail@steadytao.com>pull/7998/head
Jens-Uwe Mager
GitHub
parent
f7d58438b0
commit
8626fa3703
|
|
@ -101,8 +101,10 @@ func (ech *ECH) Provision(ctx caddy.Context) ([]string, error) {
|
|||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if err := storage.Unlock(ctx, echStorageLockName); err != nil {
|
||||
logger.Error("unable to unlock ECH provisioning in storage", zap.Error(err))
|
||||
if err := storage.Unlock(context.WithoutCancel(ctx), echStorageLockName); err != nil {
|
||||
if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
|
||||
logger.Error("unable to unlock ECH provisioning in storage", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
|
|
@ -188,7 +190,16 @@ func (ech *ECH) setConfigsFromStorage(ctx caddy.Context, logger *zap.Logger) ([]
|
|||
//
|
||||
// This function sets/updates the stdlib-ready key list only if a rotation occurs.
|
||||
func (ech *ECH) rotateECHKeys(ctx caddy.Context, logger *zap.Logger, storageSynced bool) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx.Context == nil {
|
||||
return nil
|
||||
}
|
||||
storage := ctx.Storage()
|
||||
if storage == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// all existing configs are now loaded; rotate keys "regularly" as recommended by the spec
|
||||
// (also: "Rotating too frequently limits the client anonymity set." - but the more server
|
||||
|
|
@ -208,8 +219,10 @@ func (ech *ECH) rotateECHKeys(ctx caddy.Context, logger *zap.Logger, storageSync
|
|||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := storage.Unlock(ctx, echStorageLockName); err != nil {
|
||||
logger.Error("unable to unlock ECH rotation in storage", zap.Error(err))
|
||||
if err := storage.Unlock(context.WithoutCancel(ctx), echStorageLockName); err != nil {
|
||||
if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
|
||||
logger.Error("unable to unlock ECH rotation in storage", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
|
@ -221,7 +234,13 @@ func (ech *ECH) rotateECHKeys(ctx caddy.Context, logger *zap.Logger, storageSync
|
|||
|
||||
// iterate the updated list and do any updates as needed
|
||||
for publicName := range ech.configs {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
for i := 0; i < len(ech.configs[publicName]); i++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
cfg := ech.configs[publicName][i]
|
||||
if time.Since(cfg.meta.Created) >= rotationInterval && cfg.meta.Replaced.IsZero() {
|
||||
// key is due for rotation and it hasn't been replaced yet; do that now
|
||||
|
|
@ -311,16 +330,27 @@ func (ech *ECH) updateKeyList() {
|
|||
}
|
||||
|
||||
// publishECHConfigs publishes any configs that are configured for publication and which haven't been published already.
|
||||
func (t *TLS) publishECHConfigs(logger *zap.Logger) error {
|
||||
func (t *TLS) publishECHConfigs(ctx context.Context, logger *zap.Logger) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if t.ctx.Context == nil {
|
||||
return nil
|
||||
}
|
||||
// make publication exclusive, since we don't need to repeat this unnecessarily
|
||||
storage := t.ctx.Storage()
|
||||
if storage == nil {
|
||||
return nil
|
||||
}
|
||||
const echLockName = "ech_publish"
|
||||
if err := storage.Lock(t.ctx, echLockName); err != nil {
|
||||
if err := storage.Lock(ctx, echLockName); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := storage.Unlock(t.ctx, echLockName); err != nil {
|
||||
logger.Error("unable to unlock ECH provisioning in storage", zap.Error(err))
|
||||
if err := storage.Unlock(context.WithoutCancel(ctx), echLockName); err != nil {
|
||||
if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
|
||||
logger.Error("unable to unlock ECH provisioning in storage", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
|
|
@ -446,7 +476,7 @@ func (t *TLS) publishECHConfigs(logger *zap.Logger) error {
|
|||
|
||||
// publish this ECH config list with this publisher
|
||||
pubTime := time.Now()
|
||||
err := publisher.PublishECHConfigList(t.ctx, dnsNamesToPublish, echCfgListBin)
|
||||
err := publisher.PublishECHConfigList(ctx, dnsNamesToPublish, echCfgListBin)
|
||||
|
||||
var publishErrs PublishECHConfigListErrors
|
||||
if errors.As(err, &publishErrs) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,471 @@
|
|||
// Copyright 2015 Matthew Holt and The Caddy Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package caddytls
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/caddyserver/caddy/v2"
|
||||
"github.com/caddyserver/certmagic"
|
||||
"github.com/cloudflare/circl/hpke"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var (
|
||||
testStorageMu sync.Mutex
|
||||
activeTestStorage certmagic.Storage
|
||||
)
|
||||
|
||||
type testStorageModule struct{}
|
||||
|
||||
func (testStorageModule) CaddyModule() caddy.ModuleInfo {
|
||||
return caddy.ModuleInfo{
|
||||
ID: "caddy.storage.test_blocking",
|
||||
New: func() caddy.Module { return new(testStorageModule) },
|
||||
}
|
||||
}
|
||||
|
||||
func (testStorageModule) CertMagicStorage() (certmagic.Storage, error) {
|
||||
testStorageMu.Lock()
|
||||
defer testStorageMu.Unlock()
|
||||
return activeTestStorage, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
caddy.RegisterModule(testStorageModule{})
|
||||
}
|
||||
|
||||
func newTestContextWithStorage(t *testing.T, storage certmagic.Storage) (caddy.Context, context.CancelFunc) {
|
||||
testStorageMu.Lock()
|
||||
activeTestStorage = storage
|
||||
testStorageMu.Unlock()
|
||||
|
||||
cfg := &caddy.Config{
|
||||
StorageRaw: []byte(`{"module": "test_blocking"}`),
|
||||
}
|
||||
ctx, err := caddy.ProvisionContext(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("caddy.ProvisionContext failed: %v", err)
|
||||
}
|
||||
return ctx, func() {
|
||||
testStorageMu.Lock()
|
||||
activeTestStorage = nil
|
||||
testStorageMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
type blockingStorage struct {
|
||||
lockFunc func(ctx context.Context, key string) error
|
||||
unlockFunc func(ctx context.Context, key string) error
|
||||
storeFunc func(ctx context.Context, key string, value []byte) error
|
||||
loadFunc func(ctx context.Context, key string) ([]byte, error)
|
||||
deleteFunc func(ctx context.Context, key string) error
|
||||
existsFunc func(ctx context.Context, key string) bool
|
||||
listFunc func(ctx context.Context, path string, recursive bool) ([]string, error)
|
||||
statFunc func(ctx context.Context, key string) (certmagic.KeyInfo, error)
|
||||
}
|
||||
|
||||
func (s *blockingStorage) Lock(ctx context.Context, key string) error {
|
||||
if s.lockFunc != nil {
|
||||
return s.lockFunc(ctx, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *blockingStorage) Unlock(ctx context.Context, key string) error {
|
||||
if s.unlockFunc != nil {
|
||||
return s.unlockFunc(ctx, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *blockingStorage) Store(ctx context.Context, key string, value []byte) error {
|
||||
if s.storeFunc != nil {
|
||||
return s.storeFunc(ctx, key, value)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *blockingStorage) Load(ctx context.Context, key string) ([]byte, error) {
|
||||
if s.loadFunc != nil {
|
||||
return s.loadFunc(ctx, key)
|
||||
}
|
||||
return nil, fs.ErrNotExist
|
||||
}
|
||||
|
||||
func (s *blockingStorage) Delete(ctx context.Context, key string) error {
|
||||
if s.deleteFunc != nil {
|
||||
return s.deleteFunc(ctx, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *blockingStorage) Exists(ctx context.Context, key string) bool {
|
||||
if s.existsFunc != nil {
|
||||
return s.existsFunc(ctx, key)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *blockingStorage) List(ctx context.Context, path string, recursive bool) ([]string, error) {
|
||||
if s.listFunc != nil {
|
||||
return s.listFunc(ctx, path, recursive)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *blockingStorage) Stat(ctx context.Context, key string) (certmagic.KeyInfo, error) {
|
||||
if s.statFunc != nil {
|
||||
return s.statFunc(ctx, key)
|
||||
}
|
||||
return certmagic.KeyInfo{}, fs.ErrNotExist
|
||||
}
|
||||
|
||||
type blockingPublisher struct {
|
||||
publishFunc func(ctx context.Context, innerNames []string, echConfigList []byte) error
|
||||
}
|
||||
|
||||
func (p *blockingPublisher) PublisherKey() string {
|
||||
return "blocking_test_publisher"
|
||||
}
|
||||
|
||||
func (p *blockingPublisher) PublishECHConfigList(ctx context.Context, innerNames []string, echConfigList []byte) error {
|
||||
if p.publishFunc != nil {
|
||||
return p.publishFunc(ctx, innerNames, echConfigList)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTLSStorageCleanStopSynchronization(t *testing.T) {
|
||||
ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
|
||||
defer cancel()
|
||||
|
||||
tlsApp := &TLS{
|
||||
ctx: ctx,
|
||||
logger: zap.NewNop(),
|
||||
Automation: &AutomationConfig{
|
||||
StorageCleanInterval: caddy.Duration(1 * time.Hour),
|
||||
},
|
||||
}
|
||||
|
||||
// Start storage cleaner
|
||||
tlsApp.keepStorageClean()
|
||||
|
||||
// Stop must cancel cleaner context and wait for completion cleanly
|
||||
err := tlsApp.Stop()
|
||||
if err != nil {
|
||||
t.Fatalf("TLS.Stop failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTLSStorageCleanUnitsCanceledContext(t *testing.T) {
|
||||
ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
|
||||
cancel() // canceled immediately
|
||||
|
||||
tlsApp := &TLS{
|
||||
ctx: ctx,
|
||||
logger: zap.NewNop(),
|
||||
}
|
||||
|
||||
// cleanStorageUnits with canceled context should return immediately
|
||||
tlsApp.cleanStorageUnits(ctx.Context)
|
||||
}
|
||||
|
||||
func TestTLSECHStopSynchronization(t *testing.T) {
|
||||
ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
|
||||
defer cancel()
|
||||
|
||||
tlsApp := &TLS{
|
||||
ctx: ctx,
|
||||
logger: zap.NewNop(),
|
||||
Automation: &AutomationConfig{},
|
||||
EncryptedClientHello: &ECH{},
|
||||
DisableStorageClean: true,
|
||||
}
|
||||
|
||||
err := tlsApp.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("TLS.Start failed: %v", err)
|
||||
}
|
||||
|
||||
// Stop must cancel ECH context and wait for background ECH worker to exit
|
||||
err = tlsApp.Stop()
|
||||
if err != nil {
|
||||
t.Fatalf("TLS.Stop failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTLSStorageCleanBlockingStorageStopSynchronization(t *testing.T) {
|
||||
startedLock := make(chan struct{})
|
||||
storage := &blockingStorage{
|
||||
lockFunc: func(ctx context.Context, key string) error {
|
||||
if key == "storage_clean" {
|
||||
close(startedLock)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cleanup := newTestContextWithStorage(t, storage)
|
||||
defer cleanup()
|
||||
|
||||
tlsApp := &TLS{
|
||||
ctx: ctx,
|
||||
logger: zap.NewNop(),
|
||||
Automation: &AutomationConfig{
|
||||
StorageCleanInterval: caddy.Duration(1 * time.Hour),
|
||||
},
|
||||
}
|
||||
|
||||
// Reset storageClean timestamp so cleaning runs
|
||||
storageCleanMu.Lock()
|
||||
storageClean = time.Time{}
|
||||
storageCleanMu.Unlock()
|
||||
|
||||
tlsApp.keepStorageClean()
|
||||
|
||||
select {
|
||||
case <-startedLock:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for storage lock to be attempted")
|
||||
}
|
||||
|
||||
stopped := make(chan error, 1)
|
||||
go func() {
|
||||
stopped <- tlsApp.Stop()
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-stopped:
|
||||
if err != nil {
|
||||
t.Fatalf("TLS.Stop failed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("TLS.Stop hung while storage cleaner was blocked on storage lock")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTLSECHPublishBlockingStorageStopSynchronization(t *testing.T) {
|
||||
startedLock := make(chan struct{})
|
||||
storage := &blockingStorage{
|
||||
lockFunc: func(ctx context.Context, key string) error {
|
||||
if key == "ech_publish" {
|
||||
close(startedLock)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cleanup := newTestContextWithStorage(t, storage)
|
||||
defer cleanup()
|
||||
|
||||
ech := &ECH{
|
||||
configsMu: new(sync.RWMutex),
|
||||
Configs: []ECHConfiguration{{PublicName: "example.com"}},
|
||||
}
|
||||
tlsApp := &TLS{
|
||||
ctx: ctx,
|
||||
logger: zap.NewNop(),
|
||||
Automation: &AutomationConfig{},
|
||||
EncryptedClientHello: ech,
|
||||
DisableStorageClean: true,
|
||||
}
|
||||
|
||||
err := tlsApp.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("TLS.Start failed: %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-startedLock:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for ech_publish storage lock to be attempted")
|
||||
}
|
||||
|
||||
stopped := make(chan error, 1)
|
||||
go func() {
|
||||
stopped <- tlsApp.Stop()
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-stopped:
|
||||
if err != nil {
|
||||
t.Fatalf("TLS.Stop failed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("TLS.Stop hung while ECH publisher was blocked on ech_publish storage lock")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTLSECHPublishBlockingPublisherStopSynchronization(t *testing.T) {
|
||||
startedPublish := make(chan struct{})
|
||||
publisher := &blockingPublisher{
|
||||
publishFunc: func(ctx context.Context, innerNames []string, echConfigList []byte) error {
|
||||
close(startedPublish)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
},
|
||||
}
|
||||
|
||||
storage := &blockingStorage{}
|
||||
ctx, cleanup := newTestContextWithStorage(t, storage)
|
||||
defer cleanup()
|
||||
|
||||
publicKey, _, err := hpke.KEM_X25519_HKDF_SHA256.Scheme().GenerateKeyPair()
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateKeyPair failed: %v", err)
|
||||
}
|
||||
|
||||
ech := &ECH{
|
||||
configsMu: new(sync.RWMutex),
|
||||
Configs: []ECHConfiguration{{PublicName: "example.com"}},
|
||||
Publication: []*ECHPublication{
|
||||
{
|
||||
Domains: []string{"example.com"},
|
||||
publishers: []ECHPublisher{publisher},
|
||||
},
|
||||
},
|
||||
configs: map[string][]echConfig{
|
||||
"example.com": {
|
||||
{
|
||||
PublicKey: publicKey,
|
||||
Version: draftTLSESNI25,
|
||||
ConfigID: 1,
|
||||
RawPublicName: "example.com",
|
||||
KEMID: hpke.KEM_X25519_HKDF_SHA256,
|
||||
CipherSuites: []hpkeSymmetricCipherSuite{
|
||||
{
|
||||
KDFID: hpke.KDF_HKDF_SHA256,
|
||||
AEADID: hpke.AEAD_AES128GCM,
|
||||
},
|
||||
},
|
||||
meta: echConfigMeta{
|
||||
Publications: make(publicationHistory),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
tlsApp := &TLS{
|
||||
ctx: ctx,
|
||||
logger: zap.NewNop(),
|
||||
Automation: &AutomationConfig{},
|
||||
EncryptedClientHello: ech,
|
||||
DisableStorageClean: true,
|
||||
serverNames: map[string]serverNameRegistration{
|
||||
"example.com": {},
|
||||
},
|
||||
}
|
||||
|
||||
err = tlsApp.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("TLS.Start failed: %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-startedPublish:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for ECH publication to be attempted")
|
||||
}
|
||||
|
||||
stopped := make(chan error, 1)
|
||||
go func() {
|
||||
stopped <- tlsApp.Stop()
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-stopped:
|
||||
if err != nil {
|
||||
t.Fatalf("TLS.Stop failed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("TLS.Stop hung while ECH publisher was blocked inside PublishECHConfigList")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTLSECHRotateBlockingStorageStopSynchronization(t *testing.T) {
|
||||
startedLock := make(chan struct{})
|
||||
storage := &blockingStorage{
|
||||
lockFunc: func(ctx context.Context, key string) error {
|
||||
if key == echStorageLockName {
|
||||
close(startedLock)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cleanup := newTestContextWithStorage(t, storage)
|
||||
defer cleanup()
|
||||
|
||||
ech := &ECH{
|
||||
configsMu: new(sync.RWMutex),
|
||||
Configs: []ECHConfiguration{{PublicName: "example.com"}},
|
||||
configs: map[string][]echConfig{
|
||||
"example.com": {
|
||||
{
|
||||
ConfigID: 1,
|
||||
RawPublicName: "example.com",
|
||||
meta: echConfigMeta{
|
||||
Created: time.Now().Add(-24 * time.Hour * 31), // > 30 days old so rotationNeeded returns true
|
||||
Publications: make(publicationHistory),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
tlsApp := &TLS{
|
||||
ctx: ctx,
|
||||
logger: zap.NewNop(),
|
||||
Automation: &AutomationConfig{},
|
||||
EncryptedClientHello: ech,
|
||||
DisableStorageClean: true,
|
||||
echRotateInterval: 5 * time.Millisecond,
|
||||
}
|
||||
|
||||
err := tlsApp.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("TLS.Start failed: %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-startedLock:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for echStorageLockName storage lock to be attempted during rotation")
|
||||
}
|
||||
|
||||
stopped := make(chan error, 1)
|
||||
go func() {
|
||||
stopped <- tlsApp.Stop()
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-stopped:
|
||||
if err != nil {
|
||||
t.Fatalf("TLS.Stop failed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("TLS.Stop hung while ECH key rotation was blocked on storage lock (context was likely ignored)")
|
||||
}
|
||||
}
|
||||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
|
|
@ -136,8 +137,11 @@ type TLS struct {
|
|||
certificateLoaders []CertificateLoader
|
||||
automateNames map[string]struct{}
|
||||
ctx caddy.Context
|
||||
bgCtx context.Context
|
||||
bgCancel context.CancelFunc
|
||||
bgWg *sync.WaitGroup
|
||||
storageCleanTicker *time.Ticker
|
||||
storageCleanStop chan struct{}
|
||||
echRotateInterval time.Duration
|
||||
logger *zap.Logger
|
||||
events *caddyevents.App
|
||||
|
||||
|
|
@ -411,6 +415,17 @@ func (t *TLS) Start() error {
|
|||
}
|
||||
}
|
||||
|
||||
if t.bgCtx == nil {
|
||||
parentCtx := t.ctx.Context
|
||||
if parentCtx == nil {
|
||||
parentCtx = context.Background()
|
||||
}
|
||||
t.bgCtx, t.bgCancel = context.WithCancel(parentCtx)
|
||||
}
|
||||
if t.bgWg == nil {
|
||||
t.bgWg = new(sync.WaitGroup)
|
||||
}
|
||||
|
||||
// now that we are running, and all manual certificates have
|
||||
// been loaded, time to load the automated/managed certificates
|
||||
err := t.Manage(t.automateNames)
|
||||
|
|
@ -423,34 +438,57 @@ func (t *TLS) Start() error {
|
|||
|
||||
// publish ECH configs in the background; does not need to block
|
||||
// server startup, as it could take a while; then keep keys rotated
|
||||
go func() {
|
||||
t.bgWg.Add(1)
|
||||
go func(ctx context.Context) {
|
||||
defer t.bgWg.Done()
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
log.Printf("[PANIC] tls ech publisher: %v\n%s", err, debug.Stack())
|
||||
}
|
||||
}()
|
||||
|
||||
// publish immediately first
|
||||
if err := t.publishECHConfigs(echLogger); err != nil {
|
||||
echLogger.Error("publication(s) failed", zap.Error(err))
|
||||
if err := t.publishECHConfigs(ctx, echLogger); err != nil {
|
||||
if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
|
||||
echLogger.Error("publication(s) failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(t.echRotationInterval())
|
||||
defer ticker.Stop()
|
||||
|
||||
// then every so often, rotate and publish if needed
|
||||
// (both of these functions only do something if needed)
|
||||
for {
|
||||
select {
|
||||
case <-time.After(1 * time.Hour):
|
||||
case <-ticker.C:
|
||||
// ensure old keys are rotated out
|
||||
t.EncryptedClientHello.configsMu.Lock()
|
||||
err = t.EncryptedClientHello.rotateECHKeys(t.ctx, echLogger, false)
|
||||
err := t.EncryptedClientHello.rotateECHKeys(t.caddyContext(ctx), echLogger, false)
|
||||
t.EncryptedClientHello.configsMu.Unlock()
|
||||
if err != nil {
|
||||
echLogger.Error("rotating ECH configs failed", zap.Error(err))
|
||||
if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
|
||||
echLogger.Error("rotating ECH configs failed", zap.Error(err))
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
err := t.publishECHConfigs(echLogger)
|
||||
if err != nil {
|
||||
echLogger.Error("publication(s) failed", zap.Error(err))
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
case <-t.ctx.Done():
|
||||
err = t.publishECHConfigs(ctx, echLogger)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
|
||||
echLogger.Error("publication(s) failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}(t.bgCtx)
|
||||
}
|
||||
|
||||
if !t.DisableStorageClean {
|
||||
|
|
@ -464,13 +502,18 @@ func (t *TLS) Start() error {
|
|||
|
||||
// Stop stops the TLS module and cleans up any allocations.
|
||||
func (t *TLS) Stop() error {
|
||||
// stop the storage cleaner goroutine and ticker
|
||||
if t.storageCleanStop != nil {
|
||||
close(t.storageCleanStop)
|
||||
// cancel all background goroutines (storage cleaner, ECH rotation/publication, etc.)
|
||||
if t.bgCancel != nil {
|
||||
t.bgCancel()
|
||||
}
|
||||
if t.storageCleanTicker != nil {
|
||||
t.storageCleanTicker.Stop()
|
||||
}
|
||||
// wait for all background goroutines to finish before returning,
|
||||
// ensuring no background storage users are active when module Cleanup() runs
|
||||
if t.bgWg != nil {
|
||||
t.bgWg.Wait()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -907,30 +950,45 @@ func (t *TLS) HasCertificateForSubject(subject string) bool {
|
|||
// known storage units if it was not recently done, and then runs the
|
||||
// operation at every tick from t.storageCleanTicker.
|
||||
func (t *TLS) keepStorageClean() {
|
||||
if t.bgCtx == nil {
|
||||
parentCtx := t.ctx.Context
|
||||
if parentCtx == nil {
|
||||
parentCtx = context.Background()
|
||||
}
|
||||
t.bgCtx, t.bgCancel = context.WithCancel(parentCtx)
|
||||
}
|
||||
if t.bgWg == nil {
|
||||
t.bgWg = new(sync.WaitGroup)
|
||||
}
|
||||
t.storageCleanTicker = time.NewTicker(t.storageCleanInterval())
|
||||
t.storageCleanStop = make(chan struct{})
|
||||
go func() {
|
||||
t.bgWg.Add(1)
|
||||
go func(ctx context.Context) {
|
||||
defer t.bgWg.Done()
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
log.Printf("[PANIC] storage cleaner: %v\n%s", err, debug.Stack())
|
||||
}
|
||||
}()
|
||||
t.cleanStorageUnits()
|
||||
t.cleanStorageUnits(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-t.storageCleanStop:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.storageCleanTicker.C:
|
||||
t.cleanStorageUnits()
|
||||
t.cleanStorageUnits(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}(t.bgCtx)
|
||||
}
|
||||
|
||||
func (t *TLS) cleanStorageUnits() {
|
||||
func (t *TLS) cleanStorageUnits(ctx context.Context) {
|
||||
storageCleanMu.Lock()
|
||||
defer storageCleanMu.Unlock()
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// TODO: This check might not be needed anymore now that CertMagic syncs
|
||||
// and throttles storage cleaning globally across the cluster.
|
||||
// The original comment below might be outdated:
|
||||
|
|
@ -963,12 +1021,17 @@ func (t *TLS) cleanStorageUnits() {
|
|||
}
|
||||
|
||||
// start with the default/global storage
|
||||
err = certmagic.CleanStorage(t.ctx, t.ctx.Storage(), options)
|
||||
if err != nil {
|
||||
// probably don't want to return early, since we should still
|
||||
// see if any other storages can get cleaned up
|
||||
if c := t.logger.Check(zapcore.ErrorLevel, "could not clean default/global storage"); c != nil {
|
||||
c.Write(zap.Error(err))
|
||||
if storage := t.ctx.Storage(); storage != nil {
|
||||
err = certmagic.CleanStorage(ctx, storage, options)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
// probably don't want to return early, since we should still
|
||||
// see if any other storages can get cleaned up
|
||||
if c := t.logger.Check(zapcore.ErrorLevel, "could not clean default/global storage"); c != nil {
|
||||
c.Write(zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -978,7 +1041,10 @@ func (t *TLS) cleanStorageUnits() {
|
|||
if ap.storage == nil {
|
||||
continue
|
||||
}
|
||||
if err := certmagic.CleanStorage(t.ctx, ap.storage, options); err != nil {
|
||||
if err := certmagic.CleanStorage(ctx, ap.storage, options); err != nil {
|
||||
if errors.Is(err, context.Canceled) || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if c := t.logger.Check(zapcore.ErrorLevel, "could not clean storage configured in automation policy"); c != nil {
|
||||
c.Write(zap.Error(err))
|
||||
}
|
||||
|
|
@ -986,6 +1052,10 @@ func (t *TLS) cleanStorageUnits() {
|
|||
}
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// remember last time storage was finished cleaning
|
||||
storageClean = time.Now()
|
||||
|
||||
|
|
@ -999,6 +1069,19 @@ func (t *TLS) storageCleanInterval() time.Duration {
|
|||
return defaultStorageCleanInterval
|
||||
}
|
||||
|
||||
func (t *TLS) echRotationInterval() time.Duration {
|
||||
if t.echRotateInterval > 0 {
|
||||
return t.echRotateInterval
|
||||
}
|
||||
return 1 * time.Hour
|
||||
}
|
||||
|
||||
func (t *TLS) caddyContext(ctx context.Context) caddy.Context {
|
||||
caddyCtx := t.ctx
|
||||
caddyCtx.Context = ctx
|
||||
return caddyCtx
|
||||
}
|
||||
|
||||
// onEvent translates CertMagic events into Caddy events then dispatches them.
|
||||
func (t *TLS) onEvent(ctx context.Context, eventName string, data map[string]any) error {
|
||||
evt := t.events.Emit(t.ctx, eventName, data)
|
||||
|
|
|
|||
Loading…
Reference in New Issue