acmeserver: say when the CA database is locked by another process (#8007)

Opening the bbolt database only reported timeout, which sounds like a
broken config. Catch bolt.ErrTimeout with errors.Is and say another
process holds the lock, and that caddy reload is the right command when
Caddy is already running. The error stays fatal.

go.mod

@@ -165,7 +165,7 @@ require (
 	github.com/slackhq/nebula v1.10.3 // indirect
 	github.com/spf13/cast v1.7.0 // indirect
 	github.com/urfave/cli v1.22.17 // indirect
-	go.etcd.io/bbolt v1.4.3 // indirect
+	go.etcd.io/bbolt v1.4.3
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
 	go.opentelemetry.io/otel/metric v1.44.0 // indirect
 	go.opentelemetry.io/otel/trace v1.44.0

modules/caddypki/acmeserver/acmeserver.go

@@ -16,6 +16,7 @@ package acmeserver
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	weakrand "math/rand/v2"
 	"net"
@@ -34,6 +35,7 @@ import (
 	"github.com/smallstep/certificates/authority/provisioner"
 	"github.com/smallstep/certificates/db"
 	"github.com/smallstep/nosql"
+	bolterrors "go.etcd.io/bbolt/errors"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
@@ -290,6 +292,10 @@ func (ash Handler) openDatabase() (*db.AuthDB, error) {
 			DataSource: dbPath,
 		}
 		database, err := db.New(dbConfig)
+		if errors.Is(err, bolterrors.ErrTimeout) {
+			// bbolt holds an exclusive file lock, so something else has this database open
+			err = fmt.Errorf("%w: CA database %s is already locked; this is usually another running Caddy instance, in which case use 'caddy reload' to apply config changes to it", err, dbPath)
+		}
 		return databaseCloser{&database}, err
 	})
 

modules/caddypki/acmeserver/acmeserver_test.go

@@ -1,9 +1,15 @@
 package acmeserver
 
 import (
+	"errors"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
+	bolt "go.etcd.io/bbolt"
+	bolterrors "go.etcd.io/bbolt/errors"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zaptest/observer"
 )
@@ -92,3 +98,42 @@ func TestHandler_warnIfPolicyAllowsAll(t *testing.T) {
 		})
 	}
 }
+
+func TestHandler_openDatabaseLocked(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping: bbolt waits 5s for the lock")
+	}
+
+	dataDir := t.TempDir()
+	t.Setenv("XDG_DATA_HOME", dataDir)
+
+	ash := Handler{CA: "locked-db-test", logger: zap.NewNop()}
+	key := ash.getDatabaseKey()
+
+	dbFolder := filepath.Join(dataDir, "caddy", "acme_server", key)
+	if err := os.MkdirAll(dbFolder, 0o755); err != nil {
+		t.Fatalf("making database folder: %v", err)
+	}
+	dbPath := filepath.Join(dbFolder, "db")
+
+	// hold the lock like a running Caddy instance would
+	locked, err := bolt.Open(dbPath, 0o600, &bolt.Options{Timeout: 5 * time.Second})
+	if err != nil {
+		t.Fatalf("opening database: %v", err)
+	}
+	defer locked.Close()
+
+	_, err = ash.openDatabase()
+	if err == nil {
+		t.Fatal("expected an error opening a locked database, got none")
+	}
+	if !errors.Is(err, bolterrors.ErrTimeout) {
+		t.Errorf("expected error to wrap bolterrors.ErrTimeout, got: %v", err)
+	}
+	if !strings.Contains(err.Error(), "already locked") {
+		t.Errorf("expected error to explain the lock, got: %v", err)
+	}
+	if _, err := databasePool.Delete(key); err != nil {
+		t.Errorf("cleaning up database pool: %v", err)
+	}
+}