From c6fdfe6740e35ad49d3470568ca188e0b163ff2e Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 26 Sep 2024 21:20:51 +0000 Subject: [PATCH 01/15] feat: add keychain abstraction for fetching rotated keys --- coderd/cryptokeys/dbkeycache.go | 175 +++++++++ coderd/cryptokeys/dbkeycache_internal_test.go | 338 ++++++++++++++++++ coderd/cryptokeys/dbkeycache_test.go | 216 +++++++++++ coderd/cryptokeys/keycache.go | 19 + coderd/{keyrotate => cryptokeys}/rotate.go | 10 +- .../rotate_internal_test.go | 2 +- .../{keyrotate => cryptokeys}/rotate_test.go | 14 +- coderd/database/modelmethods.go | 18 + 8 files changed, 779 insertions(+), 13 deletions(-) create mode 100644 coderd/cryptokeys/dbkeycache.go create mode 100644 coderd/cryptokeys/dbkeycache_internal_test.go create mode 100644 coderd/cryptokeys/dbkeycache_test.go create mode 100644 coderd/cryptokeys/keycache.go rename coderd/{keyrotate => cryptokeys}/rotate.go (97%) rename coderd/{keyrotate => cryptokeys}/rotate_internal_test.go (99%) rename coderd/{keyrotate => cryptokeys}/rotate_test.go (85%) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go new file mode 100644 index 0000000000000..5350987345d0b --- /dev/null +++ b/coderd/cryptokeys/dbkeycache.go @@ -0,0 +1,175 @@ +package cryptokeys + +import ( + "context" + "database/sql" + "sync" + "time" + + "golang.org/x/xerrors" + + "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/quartz" +) + +// DBKeyCache implements KeyCache for callers with access to the database. +type DBKeyCache struct { + Clock quartz.Clock + db database.Store + feature database.CryptoKeyFeature + logger slog.Logger + + // The following are initialized by NewDBKeyCache. + cacheMu sync.RWMutex + cache map[int32]database.CryptoKey + latestKey database.CryptoKey +} + +// NewDBKeyCache creates a new DBKeyCache. It starts a background +// process that periodically refreshes the cache. The context should +// be canceled to stop the background process. +func NewDBKeyCache(ctx context.Context, logger slog.Logger, db database.Store, feature database.CryptoKeyFeature, opts ...func(*DBKeyCache)) (*DBKeyCache, error) { + d := &DBKeyCache{ + db: db, + feature: feature, + Clock: quartz.NewReal(), + logger: logger, + } + for _, opt := range opts { + opt(d) + } + + cache, latest, err := d.newCache(ctx) + if err != nil { + return nil, xerrors.Errorf("new cache: %w", err) + } + d.cache, d.latestKey = cache, latest + + go d.refresh(ctx) + return d, nil +} + +// Version returns the CryptoKey with the given sequence number, provided that +// it is not deleted or has breached its deletion date. +func (d *DBKeyCache) Version(ctx context.Context, sequence int32) (database.CryptoKey, error) { + now := d.Clock.Now().UTC() + d.cacheMu.RLock() + key, ok := d.cache[sequence] + d.cacheMu.RUnlock() + if ok { + if key.IsInvalid(now) { + return database.CryptoKey{}, ErrKeyNotFound + } + return key, nil + } + + d.cacheMu.Lock() + defer d.cacheMu.Unlock() + + key, ok = d.cache[sequence] + if ok { + return key, nil + } + + key, err := d.db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ + Feature: d.feature, + Sequence: sequence, + }) + if xerrors.Is(err, sql.ErrNoRows) { + return database.CryptoKey{}, ErrKeyNotFound + } + if err != nil { + return database.CryptoKey{}, err + } + + if key.IsInvalid(now) { + return database.CryptoKey{}, ErrKeyInvalid + } + + if key.IsActive(now) && key.Sequence > d.latestKey.Sequence { + d.latestKey = key + } + + d.cache[sequence] = key + + return key, nil +} + +func (d *DBKeyCache) Latest(ctx context.Context) (database.CryptoKey, error) { + d.cacheMu.RLock() + latest := d.latestKey + d.cacheMu.RUnlock() + + now := d.Clock.Now().UTC() + if latest.IsActive(now) { + return latest, nil + } + + d.cacheMu.Lock() + defer d.cacheMu.Unlock() + + if latest.IsActive(now) { + return latest, nil + } + + cache, latest, err := d.newCache(ctx) + if err != nil { + return database.CryptoKey{}, xerrors.Errorf("new cache: %w", err) + } + + if len(cache) == 0 { + return database.CryptoKey{}, ErrKeyNotFound + } + + if !latest.IsActive(now) { + return database.CryptoKey{}, ErrKeyInvalid + } + + d.cache, d.latestKey = cache, latest + + return d.latestKey, nil +} + +func (d *DBKeyCache) refresh(ctx context.Context) { + d.Clock.TickerFunc(ctx, time.Minute*10, func() error { + cache, latest, err := d.newCache(ctx) + if err != nil { + d.logger.Error(ctx, "failed to refresh cache", slog.Error(err)) + return nil + } + d.cacheMu.Lock() + defer d.cacheMu.Unlock() + + d.cache, d.latestKey = cache, latest + return nil + }) +} + +func (d *DBKeyCache) newCache(ctx context.Context) (map[int32]database.CryptoKey, database.CryptoKey, error) { + now := d.Clock.Now().UTC() + keys, err := d.db.GetCryptoKeysByFeature(ctx, d.feature) + if err != nil { + return nil, database.CryptoKey{}, xerrors.Errorf("get crypto keys by feature: %w", err) + } + cache := toMap(keys) + var latest database.CryptoKey + // Keys are returned in order from highest sequence to lowest. + for _, key := range keys { + if !key.IsActive(now) { + continue + } + latest = key + break + } + + return cache, latest, nil +} + +func toMap(keys []database.CryptoKey) map[int32]database.CryptoKey { + m := make(map[int32]database.CryptoKey) + for _, key := range keys { + m[key.Sequence] = key + } + return m +} diff --git a/coderd/cryptokeys/dbkeycache_internal_test.go b/coderd/cryptokeys/dbkeycache_internal_test.go new file mode 100644 index 0000000000000..b957775b0c2c8 --- /dev/null +++ b/coderd/cryptokeys/dbkeycache_internal_test.go @@ -0,0 +1,338 @@ +package cryptokeys + +import ( + "database/sql" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbmock" + "github.com/coder/coder/v2/testutil" + "github.com/coder/quartz" +) + +func Test_Version(t *testing.T) { + t.Parallel() + + t.Run("HitsCache", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + expectedKey := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + } + + cache := map[int32]database.CryptoKey{ + 32: expectedKey, + } + + k := &DBKeyCache{ + db: mockDB, + feature: database.CryptoKeyFeatureWorkspaceApps, + cache: cache, + Clock: clock, + } + + got, err := k.Version(ctx, 32) + require.NoError(t, err) + require.Equal(t, expectedKey, got) + }) + + t.Run("MissesCache", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + expectedKey := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 33, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC(), + } + + mockDB.EXPECT().GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 33, + }).Return(expectedKey, nil) + + k := &DBKeyCache{ + db: mockDB, + feature: database.CryptoKeyFeatureWorkspaceApps, + cache: map[int32]database.CryptoKey{}, + Clock: clock, + } + + got, err := k.Version(ctx, 33) + require.NoError(t, err) + require.Equal(t, expectedKey, got) + require.Equal(t, expectedKey, k.latestKey) + }) + + t.Run("InvalidCachedKey", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + cache := map[int32]database.CryptoKey{ + 32: { + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + DeletesAt: sql.NullTime{ + Time: clock.Now().UTC(), + Valid: true, + }, + }, + } + + k := &DBKeyCache{ + db: mockDB, + feature: database.CryptoKeyFeatureWorkspaceApps, + cache: cache, + Clock: clock, + } + + _, err := k.Version(ctx, 32) + require.ErrorIs(t, err, ErrKeyNotFound) + }) + + t.Run("InvalidDBKey", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + invalidKey := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + DeletesAt: sql.NullTime{ + Time: clock.Now().UTC(), + Valid: true, + }, + } + mockDB.EXPECT().GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + }).Return(invalidKey, nil) + + k := &DBKeyCache{ + db: mockDB, + feature: database.CryptoKeyFeatureWorkspaceApps, + cache: map[int32]database.CryptoKey{}, + Clock: clock, + } + + _, err := k.Version(ctx, 32) + require.ErrorIs(t, err, ErrKeyInvalid) + }) +} + +func Test_Latest(t *testing.T) { + t.Parallel() + + t.Run("HitsCache", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + latestKey := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC(), + } + k := &DBKeyCache{ + db: mockDB, + feature: database.CryptoKeyFeatureWorkspaceApps, + Clock: clock, + latestKey: latestKey, + } + + got, err := k.Latest(ctx) + require.NoError(t, err) + require.Equal(t, latestKey, got) + }) + + t.Run("InvalidCachedKey", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + latestKey := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 33, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC(), + } + + mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{latestKey}, nil) + + k := &DBKeyCache{ + db: mockDB, + feature: database.CryptoKeyFeatureWorkspaceApps, + Clock: clock, + latestKey: database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC().Add(-time.Hour), + DeletesAt: sql.NullTime{ + Time: clock.Now().UTC(), + Valid: true, + }, + }, + } + + got, err := k.Latest(ctx) + require.NoError(t, err) + require.Equal(t, latestKey, got) + }) + + t.Run("UsesActiveKey", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + inactiveKey := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC().Add(time.Hour), + } + + activeKey := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 33, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC(), + } + + mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{inactiveKey, activeKey}, nil) + + k := &DBKeyCache{ + db: mockDB, + feature: database.CryptoKeyFeatureWorkspaceApps, + Clock: clock, + cache: map[int32]database.CryptoKey{}, + } + + got, err := k.Latest(ctx) + require.NoError(t, err) + require.Equal(t, activeKey, got) + }) + + t.Run("NoValidKeys", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + inactiveKey := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC().Add(time.Hour), + } + + invalidKey := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 33, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC().Add(-time.Hour), + DeletesAt: sql.NullTime{ + Time: clock.Now().UTC(), + Valid: true, + }, + } + + mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{inactiveKey, invalidKey}, nil) + + k := &DBKeyCache{ + db: mockDB, + feature: database.CryptoKeyFeatureWorkspaceApps, + Clock: clock, + cache: map[int32]database.CryptoKey{}, + } + + _, err := k.Latest(ctx) + require.ErrorIs(t, err, ErrKeyInvalid) + }) +} diff --git a/coderd/cryptokeys/dbkeycache_test.go b/coderd/cryptokeys/dbkeycache_test.go new file mode 100644 index 0000000000000..a410aefe9ea1b --- /dev/null +++ b/coderd/cryptokeys/dbkeycache_test.go @@ -0,0 +1,216 @@ +package cryptokeys_test + +import ( + "database/sql" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "cdr.dev/slog/sloggers/slogtest" + + "github.com/coder/coder/v2/coderd/cryptokeys" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/testutil" + "github.com/coder/quartz" +) + +func TestDBKeyCache(t *testing.T) { + t.Parallel() + + t.Run("NoKeys", func(t *testing.T) { + t.Parallel() + + var ( + db, _ = dbtestutil.NewDB(t) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) + ) + + _, err := cryptokeys.NewDBKeyCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, withClock(clock)) + require.NoError(t, err) + }) + + t.Run("Version", func(t *testing.T) { + t.Parallel() + + t.Run("HitsCache", func(t *testing.T) { + t.Parallel() + + var ( + db, _ = dbtestutil.NewDB(t) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) + ) + + key := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 1, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC(), + }) + + k, err := cryptokeys.NewDBKeyCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, withClock(clock)) + require.NoError(t, err) + + got, err := k.Version(ctx, key.Sequence) + require.NoError(t, err) + require.Equal(t, key, got) + }) + + t.Run("MissesCache", func(t *testing.T) { + t.Parallel() + + var ( + db, _ = dbtestutil.NewDB(t) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) + ) + + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 1, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC(), + }) + + k, err := cryptokeys.NewDBKeyCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, withClock(clock)) + require.NoError(t, err) + + key := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 3, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC(), + }) + + got, err := k.Version(ctx, key.Sequence) + require.NoError(t, err) + require.Equal(t, key, got) + }) + }) + + t.Run("Latest", func(t *testing.T) { + t.Parallel() + + var ( + db, _ = dbtestutil.NewDB(t) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) + ) + + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 10, + StartsAt: clock.Now().UTC(), + }) + + expectedKey := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 12, + StartsAt: clock.Now().UTC(), + }) + + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 2, + StartsAt: clock.Now().UTC(), + }) + + k, err := cryptokeys.NewDBKeyCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, withClock(clock)) + require.NoError(t, err) + + got, err := k.Latest(ctx) + require.NoError(t, err) + require.Equal(t, expectedKey, got) + }) + + t.Run("CacheRefreshes", func(t *testing.T) { + t.Parallel() + + var ( + db, _ = dbtestutil.NewDB(t) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) + ) + + expiringKey := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 12, + StartsAt: clock.Now().UTC(), + DeletesAt: sql.NullTime{ + Time: clock.Now().UTC().Add(time.Minute * 10), + Valid: true, + }, + }) + latest := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 24, + StartsAt: clock.Now().UTC(), + DeletesAt: sql.NullTime{ + Time: clock.Now().UTC().Add(2 * 2 * time.Hour), + Valid: true, + }, + }) + trap := clock.Trap().TickerFunc() + k, err := cryptokeys.NewDBKeyCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, withClock(clock)) + require.NoError(t, err) + + // Should be able to fetch the expiring key since it's still valid. + got, err := k.Version(ctx, expiringKey.Sequence) + require.NoError(t, err) + require.Equal(t, expiringKey, got) + + newLatest := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 25, + StartsAt: clock.Now().UTC(), + DeletesAt: sql.NullTime{ + Time: clock.Now().UTC().Add(2 * 2 * time.Hour), + Valid: true, + }, + }) + + // The latest key should not be the one we just generated. + got, err = k.Latest(ctx) + require.NoError(t, err) + require.Equal(t, latest, got) + + // Wait for the ticker to fire and the cache to refresh. + trap.MustWait(ctx).Release() + _, wait := clock.AdvanceNext() + wait.MustWait(ctx) + + // The latest key should be the one we just generated. + got, err = k.Latest(ctx) + require.NoError(t, err) + require.Equal(t, newLatest, got) + + // The expiring key should be gone. + + _, err = k.Version(ctx, expiringKey.Sequence) + require.ErrorIs(t, err, cryptokeys.ErrKeyNotFound) + }) +} + +func withClock(clock quartz.Clock) func(*cryptokeys.DBKeyCache) { + return func(d *cryptokeys.DBKeyCache) { + d.Clock = clock + } +} diff --git a/coderd/cryptokeys/keycache.go b/coderd/cryptokeys/keycache.go new file mode 100644 index 0000000000000..75516069460f6 --- /dev/null +++ b/coderd/cryptokeys/keycache.go @@ -0,0 +1,19 @@ +package cryptokeys + +import ( + "context" + + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/database" +) + +var ErrKeyNotFound = xerrors.New("key not found") + +var ErrKeyInvalid = xerrors.New("key is invalid for use") + +// Keycache provides an abstraction for fetching signing keys. +type Keycache interface { + Latest(ctx context.Context) (database.CryptoKey, error) + Version(ctx context.Context, sequence int32) (database.CryptoKey, error) +} diff --git a/coderd/keyrotate/rotate.go b/coderd/cryptokeys/rotate.go similarity index 97% rename from coderd/keyrotate/rotate.go rename to coderd/cryptokeys/rotate.go index b3046161aa930..224b9100d5bf8 100644 --- a/coderd/keyrotate/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -1,4 +1,4 @@ -package keyrotate +package cryptokeys import ( "context" @@ -36,15 +36,15 @@ type rotator struct { features []database.CryptoKeyFeature } -type Option func(*rotator) +type RotatorOption func(*rotator) -func WithClock(clock quartz.Clock) Option { +func WithClock(clock quartz.Clock) RotatorOption { return func(r *rotator) { r.clock = clock } } -func WithKeyDuration(keyDuration time.Duration) Option { +func WithKeyDuration(keyDuration time.Duration) RotatorOption { return func(r *rotator) { r.keyDuration = keyDuration } @@ -53,7 +53,7 @@ func WithKeyDuration(keyDuration time.Duration) Option { // StartRotator starts a background process that rotates keys in the database. // It ensures there's at least one valid key per feature prior to returning. // Canceling the provided context will stop the background process. -func StartRotator(ctx context.Context, logger slog.Logger, db database.Store, opts ...Option) error { +func StartRotator(ctx context.Context, logger slog.Logger, db database.Store, opts ...RotatorOption) error { kr := &rotator{ db: db, logger: logger, diff --git a/coderd/keyrotate/rotate_internal_test.go b/coderd/cryptokeys/rotate_internal_test.go similarity index 99% rename from coderd/keyrotate/rotate_internal_test.go rename to coderd/cryptokeys/rotate_internal_test.go index 94160a947bf11..36ecf4fa9d76d 100644 --- a/coderd/keyrotate/rotate_internal_test.go +++ b/coderd/cryptokeys/rotate_internal_test.go @@ -1,4 +1,4 @@ -package keyrotate +package cryptokeys import ( "database/sql" diff --git a/coderd/keyrotate/rotate_test.go b/coderd/cryptokeys/rotate_test.go similarity index 85% rename from coderd/keyrotate/rotate_test.go rename to coderd/cryptokeys/rotate_test.go index 43a62ac451b62..190ad213b1153 100644 --- a/coderd/keyrotate/rotate_test.go +++ b/coderd/cryptokeys/rotate_test.go @@ -1,4 +1,4 @@ -package keyrotate_test +package cryptokeys_test import ( "testing" @@ -9,10 +9,10 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" - "github.com/coder/coder/v2/coderd/keyrotate" "github.com/coder/coder/v2/testutil" "github.com/coder/quartz" ) @@ -34,7 +34,7 @@ func TestRotator(t *testing.T) { require.NoError(t, err) require.Len(t, dbkeys, 0) - err = keyrotate.StartRotator(ctx, logger, db, keyrotate.WithClock(clock)) + err = cryptokeys.StartRotator(ctx, logger, db, cryptokeys.WithClock(clock)) require.NoError(t, err) // Fetch the keys from the database and ensure they @@ -59,14 +59,14 @@ func TestRotator(t *testing.T) { rotatingKey := dbgen.CryptoKey(t, db, database.CryptoKey{ Feature: database.CryptoKeyFeatureWorkspaceApps, - StartsAt: now.Add(-keyrotate.DefaultKeyDuration + time.Hour + time.Minute), + StartsAt: now.Add(-cryptokeys.DefaultKeyDuration + time.Hour + time.Minute), Sequence: 12345, }) trap := clock.Trap().TickerFunc() t.Cleanup(trap.Close) - err := keyrotate.StartRotator(ctx, logger, db, keyrotate.WithClock(clock)) + err := cryptokeys.StartRotator(ctx, logger, db, cryptokeys.WithClock(clock)) require.NoError(t, err) initialKeyLen := len(database.AllCryptoKeyFeatureValues()) @@ -88,14 +88,14 @@ func TestRotator(t *testing.T) { newKey, err := db.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps) require.NoError(t, err) require.Equal(t, rotatingKey.Sequence+1, newKey.Sequence) - require.Equal(t, rotatingKey.ExpiresAt(keyrotate.DefaultKeyDuration), newKey.StartsAt.UTC()) + require.Equal(t, rotatingKey.ExpiresAt(cryptokeys.DefaultKeyDuration), newKey.StartsAt.UTC()) require.False(t, newKey.DeletesAt.Valid) oldKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ Feature: rotatingKey.Feature, Sequence: rotatingKey.Sequence, }) - expectedDeletesAt := rotatingKey.StartsAt.Add(keyrotate.DefaultKeyDuration + time.Hour + keyrotate.WorkspaceAppsTokenDuration) + expectedDeletesAt := rotatingKey.StartsAt.Add(cryptokeys.DefaultKeyDuration + time.Hour + cryptokeys.WorkspaceAppsTokenDuration) require.NoError(t, err) require.Equal(t, rotatingKey.StartsAt, oldKey.StartsAt) require.True(t, oldKey.DeletesAt.Valid) diff --git a/coderd/database/modelmethods.go b/coderd/database/modelmethods.go index 82be5e710c058..75759c4cd0728 100644 --- a/coderd/database/modelmethods.go +++ b/coderd/database/modelmethods.go @@ -1,6 +1,7 @@ package database import ( + "encoding/hex" "sort" "strconv" "time" @@ -451,3 +452,20 @@ func (r GetAuthorizationUserRolesRow) RoleNames() ([]rbac.RoleIdentifier, error) func (k CryptoKey) ExpiresAt(keyDuration time.Duration) time.Time { return k.StartsAt.Add(keyDuration).UTC() } + +func (k CryptoKey) DecodeString() ([]byte, error) { + return hex.DecodeString(k.Secret.String) +} + +func (k CryptoKey) IsActive(now time.Time) bool { + now = now.UTC() + isAfterStart := !k.StartsAt.IsZero() && !now.Before(k.StartsAt.UTC()) + return isAfterStart && !k.IsInvalid(now) +} + +func (k CryptoKey) IsInvalid(now time.Time) bool { + now = now.UTC() + isDeleted := !k.Secret.Valid + isPastDeletion := k.DeletesAt.Valid && !now.Before(k.DeletesAt.Time.UTC()) + return isDeleted || isPastDeletion +} From f9bff686fe18a4597411c5087e5a45d9bba03792 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 26 Sep 2024 21:33:31 +0000 Subject: [PATCH 02/15] Refactor DBKeyCache to DBCache with clock support - Simplify naming by renaming `DBKeyCache` to `DBCache`. - Introduce `DBCacheOption` for flexible clock configuration. - Ensure consistent clock handling across the methods. --- coderd/cryptokeys/dbkeycache.go | 40 +++++++++++-------- coderd/cryptokeys/dbkeycache_internal_test.go | 32 +++++++-------- coderd/cryptokeys/dbkeycache_test.go | 16 +++----- coderd/cryptokeys/keycache.go | 6 +-- 4 files changed, 48 insertions(+), 46 deletions(-) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index 5350987345d0b..c3efbde5481a8 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -13,27 +13,35 @@ import ( "github.com/coder/quartz" ) -// DBKeyCache implements KeyCache for callers with access to the database. -type DBKeyCache struct { - Clock quartz.Clock +// DBCache implements Keycache for callers with access to the database. +type DBCache struct { db database.Store feature database.CryptoKeyFeature logger slog.Logger + clock quartz.Clock - // The following are initialized by NewDBKeyCache. + // The following are initialized by NewDBCache. cacheMu sync.RWMutex cache map[int32]database.CryptoKey latestKey database.CryptoKey } -// NewDBKeyCache creates a new DBKeyCache. It starts a background +type DBCacheOption func(*DBCache) + +func WithDBCacheClock(clock quartz.Clock) DBCacheOption { + return func(d *DBCache) { + d.clock = clock + } +} + +// NewDBCache creates a new DBCache. It starts a background // process that periodically refreshes the cache. The context should // be canceled to stop the background process. -func NewDBKeyCache(ctx context.Context, logger slog.Logger, db database.Store, feature database.CryptoKeyFeature, opts ...func(*DBKeyCache)) (*DBKeyCache, error) { - d := &DBKeyCache{ +func NewDBCache(ctx context.Context, logger slog.Logger, db database.Store, feature database.CryptoKeyFeature, opts ...func(*DBCache)) (*DBCache, error) { + d := &DBCache{ db: db, feature: feature, - Clock: quartz.NewReal(), + clock: quartz.NewReal(), logger: logger, } for _, opt := range opts { @@ -52,8 +60,8 @@ func NewDBKeyCache(ctx context.Context, logger slog.Logger, db database.Store, f // Version returns the CryptoKey with the given sequence number, provided that // it is not deleted or has breached its deletion date. -func (d *DBKeyCache) Version(ctx context.Context, sequence int32) (database.CryptoKey, error) { - now := d.Clock.Now().UTC() +func (d *DBCache) Version(ctx context.Context, sequence int32) (database.CryptoKey, error) { + now := d.clock.Now().UTC() d.cacheMu.RLock() key, ok := d.cache[sequence] d.cacheMu.RUnlock() @@ -96,12 +104,12 @@ func (d *DBKeyCache) Version(ctx context.Context, sequence int32) (database.Cryp return key, nil } -func (d *DBKeyCache) Latest(ctx context.Context) (database.CryptoKey, error) { +func (d *DBCache) Latest(ctx context.Context) (database.CryptoKey, error) { d.cacheMu.RLock() latest := d.latestKey d.cacheMu.RUnlock() - now := d.Clock.Now().UTC() + now := d.clock.Now().UTC() if latest.IsActive(now) { return latest, nil } @@ -131,8 +139,8 @@ func (d *DBKeyCache) Latest(ctx context.Context) (database.CryptoKey, error) { return d.latestKey, nil } -func (d *DBKeyCache) refresh(ctx context.Context) { - d.Clock.TickerFunc(ctx, time.Minute*10, func() error { +func (d *DBCache) refresh(ctx context.Context) { + d.clock.TickerFunc(ctx, time.Minute*10, func() error { cache, latest, err := d.newCache(ctx) if err != nil { d.logger.Error(ctx, "failed to refresh cache", slog.Error(err)) @@ -146,8 +154,8 @@ func (d *DBKeyCache) refresh(ctx context.Context) { }) } -func (d *DBKeyCache) newCache(ctx context.Context) (map[int32]database.CryptoKey, database.CryptoKey, error) { - now := d.Clock.Now().UTC() +func (d *DBCache) newCache(ctx context.Context) (map[int32]database.CryptoKey, database.CryptoKey, error) { + now := d.clock.Now().UTC() keys, err := d.db.GetCryptoKeysByFeature(ctx, d.feature) if err != nil { return nil, database.CryptoKey{}, xerrors.Errorf("get crypto keys by feature: %w", err) diff --git a/coderd/cryptokeys/dbkeycache_internal_test.go b/coderd/cryptokeys/dbkeycache_internal_test.go index b957775b0c2c8..4962cd424b966 100644 --- a/coderd/cryptokeys/dbkeycache_internal_test.go +++ b/coderd/cryptokeys/dbkeycache_internal_test.go @@ -40,11 +40,11 @@ func Test_Version(t *testing.T) { 32: expectedKey, } - k := &DBKeyCache{ + k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, cache: cache, - Clock: clock, + clock: clock, } got, err := k.Version(ctx, 32) @@ -77,11 +77,11 @@ func Test_Version(t *testing.T) { Sequence: 33, }).Return(expectedKey, nil) - k := &DBKeyCache{ + k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, cache: map[int32]database.CryptoKey{}, - Clock: clock, + clock: clock, } got, err := k.Version(ctx, 33) @@ -115,11 +115,11 @@ func Test_Version(t *testing.T) { }, } - k := &DBKeyCache{ + k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, cache: cache, - Clock: clock, + clock: clock, } _, err := k.Version(ctx, 32) @@ -153,11 +153,11 @@ func Test_Version(t *testing.T) { Sequence: 32, }).Return(invalidKey, nil) - k := &DBKeyCache{ + k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, cache: map[int32]database.CryptoKey{}, - Clock: clock, + clock: clock, } _, err := k.Version(ctx, 32) @@ -187,10 +187,10 @@ func Test_Latest(t *testing.T) { }, StartsAt: clock.Now().UTC(), } - k := &DBKeyCache{ + k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, - Clock: clock, + clock: clock, latestKey: latestKey, } @@ -221,10 +221,10 @@ func Test_Latest(t *testing.T) { mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{latestKey}, nil) - k := &DBKeyCache{ + k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, - Clock: clock, + clock: clock, latestKey: database.CryptoKey{ Feature: database.CryptoKeyFeatureWorkspaceApps, Sequence: 32, @@ -277,10 +277,10 @@ func Test_Latest(t *testing.T) { mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{inactiveKey, activeKey}, nil) - k := &DBKeyCache{ + k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, - Clock: clock, + clock: clock, cache: map[int32]database.CryptoKey{}, } @@ -325,10 +325,10 @@ func Test_Latest(t *testing.T) { mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{inactiveKey, invalidKey}, nil) - k := &DBKeyCache{ + k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, - Clock: clock, + clock: clock, cache: map[int32]database.CryptoKey{}, } diff --git a/coderd/cryptokeys/dbkeycache_test.go b/coderd/cryptokeys/dbkeycache_test.go index a410aefe9ea1b..5057fa5e3bf2d 100644 --- a/coderd/cryptokeys/dbkeycache_test.go +++ b/coderd/cryptokeys/dbkeycache_test.go @@ -30,7 +30,7 @@ func TestDBKeyCache(t *testing.T) { logger = slogtest.Make(t, nil) ) - _, err := cryptokeys.NewDBKeyCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, withClock(clock)) + _, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) require.NoError(t, err) }) @@ -57,7 +57,7 @@ func TestDBKeyCache(t *testing.T) { StartsAt: clock.Now().UTC(), }) - k, err := cryptokeys.NewDBKeyCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, withClock(clock)) + k, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) require.NoError(t, err) got, err := k.Version(ctx, key.Sequence) @@ -85,7 +85,7 @@ func TestDBKeyCache(t *testing.T) { StartsAt: clock.Now().UTC(), }) - k, err := cryptokeys.NewDBKeyCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, withClock(clock)) + k, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) require.NoError(t, err) key := dbgen.CryptoKey(t, db, database.CryptoKey{ @@ -132,7 +132,7 @@ func TestDBKeyCache(t *testing.T) { StartsAt: clock.Now().UTC(), }) - k, err := cryptokeys.NewDBKeyCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, withClock(clock)) + k, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) require.NoError(t, err) got, err := k.Latest(ctx) @@ -169,7 +169,7 @@ func TestDBKeyCache(t *testing.T) { }, }) trap := clock.Trap().TickerFunc() - k, err := cryptokeys.NewDBKeyCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, withClock(clock)) + k, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) require.NoError(t, err) // Should be able to fetch the expiring key since it's still valid. @@ -208,9 +208,3 @@ func TestDBKeyCache(t *testing.T) { require.ErrorIs(t, err, cryptokeys.ErrKeyNotFound) }) } - -func withClock(clock quartz.Clock) func(*cryptokeys.DBKeyCache) { - return func(d *cryptokeys.DBKeyCache) { - d.Clock = clock - } -} diff --git a/coderd/cryptokeys/keycache.go b/coderd/cryptokeys/keycache.go index 75516069460f6..0fbe9c7c50630 100644 --- a/coderd/cryptokeys/keycache.go +++ b/coderd/cryptokeys/keycache.go @@ -5,7 +5,7 @@ import ( "golang.org/x/xerrors" - "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/enterprise/wsproxy/wsproxysdk" ) var ErrKeyNotFound = xerrors.New("key not found") @@ -14,6 +14,6 @@ var ErrKeyInvalid = xerrors.New("key is invalid for use") // Keycache provides an abstraction for fetching signing keys. type Keycache interface { - Latest(ctx context.Context) (database.CryptoKey, error) - Version(ctx context.Context, sequence int32) (database.CryptoKey, error) + Latest(ctx context.Context) (wsproxysdk.CryptoKey, error) + Version(ctx context.Context, sequence int32) (wsproxysdk.CryptoKey, error) } From 46503b66b3e1a5e27bf1958cd393cfdb25363712 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 26 Sep 2024 22:08:39 +0000 Subject: [PATCH 03/15] Refactor CryptoKey handling to use codersdk package Refactored the CryptoKey handling logic to leverage the codersdk package. This enhances consistency and modularity by centralizing CryptoKey management and related operations. The refactoring also includes renaming functions to better reflect their functionality and using the db2sdk package for database to SDK conversions. --- coderd/cryptokeys/dbkeycache.go | 71 ++++++++++--------- coderd/cryptokeys/dbkeycache_internal_test.go | 15 ++-- coderd/cryptokeys/dbkeycache_test.go | 30 +++++--- coderd/cryptokeys/keycache.go | 6 +- coderd/database/db2sdk/db2sdk.go | 14 ++++ coderd/database/modelmethods.go | 12 ++-- codersdk/deployment.go | 29 ++++++++ enterprise/coderd/workspaceproxy.go | 8 +-- enterprise/coderd/workspaceproxy_test.go | 6 +- enterprise/wsproxy/wsproxysdk/wsproxysdk.go | 31 +------- 10 files changed, 128 insertions(+), 94 deletions(-) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index c3efbde5481a8..56c00fbbf97c4 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -10,6 +10,8 @@ import ( "cdr.dev/slog" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/db2sdk" + "github.com/coder/coder/v2/codersdk" "github.com/coder/quartz" ) @@ -59,17 +61,17 @@ func NewDBCache(ctx context.Context, logger slog.Logger, db database.Store, feat } // Version returns the CryptoKey with the given sequence number, provided that -// it is not deleted or has breached its deletion date. -func (d *DBCache) Version(ctx context.Context, sequence int32) (database.CryptoKey, error) { +// it is neither deleted nor has breached its deletion date. +func (d *DBCache) Version(ctx context.Context, sequence int32) (codersdk.CryptoKey, error) { now := d.clock.Now().UTC() d.cacheMu.RLock() key, ok := d.cache[sequence] d.cacheMu.RUnlock() if ok { - if key.IsInvalid(now) { - return database.CryptoKey{}, ErrKeyNotFound + if !key.CanVerify(now) { + return codersdk.CryptoKey{}, ErrKeyInvalid } - return key, nil + return db2sdk.CryptoKey(key), nil } d.cacheMu.Lock() @@ -77,7 +79,7 @@ func (d *DBCache) Version(ctx context.Context, sequence int32) (database.CryptoK key, ok = d.cache[sequence] if ok { - return key, nil + return db2sdk.CryptoKey(key), nil } key, err := d.db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ @@ -85,58 +87,62 @@ func (d *DBCache) Version(ctx context.Context, sequence int32) (database.CryptoK Sequence: sequence, }) if xerrors.Is(err, sql.ErrNoRows) { - return database.CryptoKey{}, ErrKeyNotFound + return codersdk.CryptoKey{}, ErrKeyNotFound } if err != nil { - return database.CryptoKey{}, err + return codersdk.CryptoKey{}, err } - if key.IsInvalid(now) { - return database.CryptoKey{}, ErrKeyInvalid + if !key.CanVerify(now) { + return codersdk.CryptoKey{}, ErrKeyInvalid } - if key.IsActive(now) && key.Sequence > d.latestKey.Sequence { + // If this key is valid for signing then mark it as the latest key. + if key.CanSign(now) && key.Sequence > d.latestKey.Sequence { d.latestKey = key } d.cache[sequence] = key - return key, nil + return db2sdk.CryptoKey(key), nil } -func (d *DBCache) Latest(ctx context.Context) (database.CryptoKey, error) { +// Latest returns the latest valid key for signing. A valid key is one that is +// both past its start time and before its deletion time. +func (d *DBCache) Latest(ctx context.Context) (codersdk.CryptoKey, error) { d.cacheMu.RLock() latest := d.latestKey d.cacheMu.RUnlock() now := d.clock.Now().UTC() - if latest.IsActive(now) { - return latest, nil + if latest.CanSign(now) { + return checkKey(latest, now) } d.cacheMu.Lock() defer d.cacheMu.Unlock() - if latest.IsActive(now) { - return latest, nil + if latest.CanSign(now) { + return checkKey(latest, now) } + // Refetch all keys for this feature so we can find the latest valid key. cache, latest, err := d.newCache(ctx) if err != nil { - return database.CryptoKey{}, xerrors.Errorf("new cache: %w", err) + return codersdk.CryptoKey{}, xerrors.Errorf("new cache: %w", err) } if len(cache) == 0 { - return database.CryptoKey{}, ErrKeyNotFound + return codersdk.CryptoKey{}, ErrKeyNotFound } - if !latest.IsActive(now) { - return database.CryptoKey{}, ErrKeyInvalid + if !latest.CanSign(now) { + return codersdk.CryptoKey{}, ErrKeyInvalid } d.cache, d.latestKey = cache, latest - return d.latestKey, nil + return checkKey(latest, now) } func (d *DBCache) refresh(ctx context.Context) { @@ -154,30 +160,29 @@ func (d *DBCache) refresh(ctx context.Context) { }) } +// newCache fetches all keys for the given feature and determines the latest key. func (d *DBCache) newCache(ctx context.Context) (map[int32]database.CryptoKey, database.CryptoKey, error) { now := d.clock.Now().UTC() keys, err := d.db.GetCryptoKeysByFeature(ctx, d.feature) if err != nil { return nil, database.CryptoKey{}, xerrors.Errorf("get crypto keys by feature: %w", err) } - cache := toMap(keys) + cache := make(map[int32]database.CryptoKey) var latest database.CryptoKey - // Keys are returned in order from highest sequence to lowest. for _, key := range keys { - if !key.IsActive(now) { - continue + cache[key.Sequence] = key + if key.CanSign(now) && key.Sequence > latest.Sequence { + latest = key } - latest = key - break } return cache, latest, nil } -func toMap(keys []database.CryptoKey) map[int32]database.CryptoKey { - m := make(map[int32]database.CryptoKey) - for _, key := range keys { - m[key.Sequence] = key +func checkKey(key database.CryptoKey, now time.Time) (codersdk.CryptoKey, error) { + if !key.CanVerify(now) { + return codersdk.CryptoKey{}, ErrKeyInvalid } - return m + + return db2sdk.CryptoKey(key), nil } diff --git a/coderd/cryptokeys/dbkeycache_internal_test.go b/coderd/cryptokeys/dbkeycache_internal_test.go index 4962cd424b966..f7b4c08e29ac0 100644 --- a/coderd/cryptokeys/dbkeycache_internal_test.go +++ b/coderd/cryptokeys/dbkeycache_internal_test.go @@ -9,6 +9,7 @@ import ( "go.uber.org/mock/gomock" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbmock" "github.com/coder/coder/v2/testutil" "github.com/coder/quartz" @@ -49,7 +50,7 @@ func Test_Version(t *testing.T) { got, err := k.Version(ctx, 32) require.NoError(t, err) - require.Equal(t, expectedKey, got) + require.Equal(t, db2sdk.CryptoKey(expectedKey), got) }) t.Run("MissesCache", func(t *testing.T) { @@ -86,8 +87,8 @@ func Test_Version(t *testing.T) { got, err := k.Version(ctx, 33) require.NoError(t, err) - require.Equal(t, expectedKey, got) - require.Equal(t, expectedKey, k.latestKey) + require.Equal(t, db2sdk.CryptoKey(expectedKey), got) + require.Equal(t, db2sdk.CryptoKey(expectedKey), db2sdk.CryptoKey(k.latestKey)) }) t.Run("InvalidCachedKey", func(t *testing.T) { @@ -123,7 +124,7 @@ func Test_Version(t *testing.T) { } _, err := k.Version(ctx, 32) - require.ErrorIs(t, err, ErrKeyNotFound) + require.ErrorIs(t, err, ErrKeyInvalid) }) t.Run("InvalidDBKey", func(t *testing.T) { @@ -196,7 +197,7 @@ func Test_Latest(t *testing.T) { got, err := k.Latest(ctx) require.NoError(t, err) - require.Equal(t, latestKey, got) + require.Equal(t, db2sdk.CryptoKey(latestKey), got) }) t.Run("InvalidCachedKey", func(t *testing.T) { @@ -242,7 +243,7 @@ func Test_Latest(t *testing.T) { got, err := k.Latest(ctx) require.NoError(t, err) - require.Equal(t, latestKey, got) + require.Equal(t, db2sdk.CryptoKey(latestKey), got) }) t.Run("UsesActiveKey", func(t *testing.T) { @@ -286,7 +287,7 @@ func Test_Latest(t *testing.T) { got, err := k.Latest(ctx) require.NoError(t, err) - require.Equal(t, activeKey, got) + require.Equal(t, db2sdk.CryptoKey(activeKey), got) }) t.Run("NoValidKeys", func(t *testing.T) { diff --git a/coderd/cryptokeys/dbkeycache_test.go b/coderd/cryptokeys/dbkeycache_test.go index 5057fa5e3bf2d..2848309009367 100644 --- a/coderd/cryptokeys/dbkeycache_test.go +++ b/coderd/cryptokeys/dbkeycache_test.go @@ -11,6 +11,7 @@ import ( "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/testutil" @@ -62,7 +63,7 @@ func TestDBKeyCache(t *testing.T) { got, err := k.Version(ctx, key.Sequence) require.NoError(t, err) - require.Equal(t, key, got) + require.Equal(t, db2sdk.CryptoKey(key), got) }) t.Run("MissesCache", func(t *testing.T) { @@ -100,7 +101,7 @@ func TestDBKeyCache(t *testing.T) { got, err := k.Version(ctx, key.Sequence) require.NoError(t, err) - require.Equal(t, key, got) + require.Equal(t, db2sdk.CryptoKey(key), got) }) }) @@ -137,7 +138,7 @@ func TestDBKeyCache(t *testing.T) { got, err := k.Latest(ctx) require.NoError(t, err) - require.Equal(t, expectedKey, got) + require.Equal(t, db2sdk.CryptoKey(expectedKey), got) }) t.Run("CacheRefreshes", func(t *testing.T) { @@ -168,6 +169,13 @@ func TestDBKeyCache(t *testing.T) { Valid: true, }, }) + + wrongFeature := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureOidcConvert, + Sequence: 30, + StartsAt: clock.Now().UTC(), + }) + trap := clock.Trap().TickerFunc() k, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) require.NoError(t, err) @@ -175,7 +183,10 @@ func TestDBKeyCache(t *testing.T) { // Should be able to fetch the expiring key since it's still valid. got, err := k.Version(ctx, expiringKey.Sequence) require.NoError(t, err) - require.Equal(t, expiringKey, got) + require.Equal(t, db2sdk.CryptoKey(expiringKey), got) + + _, err = k.Version(ctx, wrongFeature.Sequence) + require.ErrorIs(t, err, cryptokeys.ErrKeyNotFound) newLatest := dbgen.CryptoKey(t, db, database.CryptoKey{ Feature: database.CryptoKeyFeatureWorkspaceApps, @@ -190,7 +201,7 @@ func TestDBKeyCache(t *testing.T) { // The latest key should not be the one we just generated. got, err = k.Latest(ctx) require.NoError(t, err) - require.Equal(t, latest, got) + require.Equal(t, db2sdk.CryptoKey(latest), got) // Wait for the ticker to fire and the cache to refresh. trap.MustWait(ctx).Release() @@ -200,11 +211,14 @@ func TestDBKeyCache(t *testing.T) { // The latest key should be the one we just generated. got, err = k.Latest(ctx) require.NoError(t, err) - require.Equal(t, newLatest, got) - - // The expiring key should be gone. + require.Equal(t, db2sdk.CryptoKey(newLatest), got) + // The expiring key should be invalid. _, err = k.Version(ctx, expiringKey.Sequence) + require.ErrorIs(t, err, cryptokeys.ErrKeyInvalid) + + // Sanity check that the wrong feature is still not found. + _, err = k.Version(ctx, wrongFeature.Sequence) require.ErrorIs(t, err, cryptokeys.ErrKeyNotFound) }) } diff --git a/coderd/cryptokeys/keycache.go b/coderd/cryptokeys/keycache.go index 0fbe9c7c50630..503edec2ca63c 100644 --- a/coderd/cryptokeys/keycache.go +++ b/coderd/cryptokeys/keycache.go @@ -5,7 +5,7 @@ import ( "golang.org/x/xerrors" - "github.com/coder/coder/v2/enterprise/wsproxy/wsproxysdk" + "github.com/coder/coder/v2/codersdk" ) var ErrKeyNotFound = xerrors.New("key not found") @@ -14,6 +14,6 @@ var ErrKeyInvalid = xerrors.New("key is invalid for use") // Keycache provides an abstraction for fetching signing keys. type Keycache interface { - Latest(ctx context.Context) (wsproxysdk.CryptoKey, error) - Version(ctx context.Context, sequence int32) (wsproxysdk.CryptoKey, error) + Latest(ctx context.Context) (codersdk.CryptoKey, error) + Version(ctx context.Context, sequence int32) (codersdk.CryptoKey, error) } diff --git a/coderd/database/db2sdk/db2sdk.go b/coderd/database/db2sdk/db2sdk.go index a8e2c6cb93fad..59f683e6ce0e6 100644 --- a/coderd/database/db2sdk/db2sdk.go +++ b/coderd/database/db2sdk/db2sdk.go @@ -659,3 +659,17 @@ func Organization(organization database.Organization) codersdk.Organization { IsDefault: organization.IsDefault, } } + +func CryptoKeys(keys []database.CryptoKey) []codersdk.CryptoKey { + return List(keys, CryptoKey) +} + +func CryptoKey(key database.CryptoKey) codersdk.CryptoKey { + return codersdk.CryptoKey{ + Feature: codersdk.CryptoKeyFeature(key.Feature), + Sequence: key.Sequence, + StartsAt: key.StartsAt.UTC(), + DeletesAt: key.DeletesAt.Time.UTC(), + Secret: key.Secret.String, + } +} diff --git a/coderd/database/modelmethods.go b/coderd/database/modelmethods.go index 75759c4cd0728..d93a458dccee9 100644 --- a/coderd/database/modelmethods.go +++ b/coderd/database/modelmethods.go @@ -457,15 +457,15 @@ func (k CryptoKey) DecodeString() ([]byte, error) { return hex.DecodeString(k.Secret.String) } -func (k CryptoKey) IsActive(now time.Time) bool { +func (k CryptoKey) CanSign(now time.Time) bool { now = now.UTC() isAfterStart := !k.StartsAt.IsZero() && !now.Before(k.StartsAt.UTC()) - return isAfterStart && !k.IsInvalid(now) + return isAfterStart && k.CanVerify(now) } -func (k CryptoKey) IsInvalid(now time.Time) bool { +func (k CryptoKey) CanVerify(now time.Time) bool { now = now.UTC() - isDeleted := !k.Secret.Valid - isPastDeletion := k.DeletesAt.Valid && !now.Before(k.DeletesAt.Time.UTC()) - return isDeleted || isPastDeletion + hasSecret := k.Secret.Valid + isBeforeDeletion := !k.DeletesAt.Valid || now.Before(k.DeletesAt.Time.UTC()) + return hasSecret && isBeforeDeletion } diff --git a/codersdk/deployment.go b/codersdk/deployment.go index e8b90a07af98f..7189657029dba 100644 --- a/codersdk/deployment.go +++ b/codersdk/deployment.go @@ -3103,3 +3103,32 @@ func (c *Client) SSHConfiguration(ctx context.Context) (SSHConfigResponse, error var sshConfig SSHConfigResponse return sshConfig, json.NewDecoder(res.Body).Decode(&sshConfig) } + +type CryptoKeyFeature string + +const ( + CryptoKeyFeatureWorkspaceApp CryptoKeyFeature = "workspace_apps" + CryptoKeyFeatureOIDCConvert CryptoKeyFeature = "oidc_convert" + CryptoKeyFeatureTailnetResume CryptoKeyFeature = "tailnet_resume" +) + +type CryptoKey struct { + Feature CryptoKeyFeature `json:"feature"` + Secret string `json:"secret"` + DeletesAt time.Time `json:"deletes_at"` + Sequence int32 `json:"sequence"` + StartsAt time.Time `json:"starts_at"` +} + +func (c CryptoKey) CanSign(now time.Time) bool { + now = now.UTC() + isAfterStartsAt := !c.StartsAt.IsZero() && !now.Before(c.StartsAt) + return isAfterStartsAt && c.CanVerify(now) +} + +func (c CryptoKey) CanVerify(now time.Time) bool { + now = now.UTC() + hasSecret := c.Secret != "" + beforeDelete := c.DeletesAt.IsZero() || now.Before(c.DeletesAt) + return hasSecret && beforeDelete +} diff --git a/enterprise/coderd/workspaceproxy.go b/enterprise/coderd/workspaceproxy.go index eef12b1d1b13a..b085aedb09e5b 100644 --- a/enterprise/coderd/workspaceproxy.go +++ b/enterprise/coderd/workspaceproxy.go @@ -995,11 +995,11 @@ func (w *workspaceProxiesFetchUpdater) Update(ctx context.Context) error { return w.updateFunc(ctx) } -func fromDBCryptoKeys(keys []database.CryptoKey) []wsproxysdk.CryptoKey { - wskeys := make([]wsproxysdk.CryptoKey, 0, len(keys)) +func fromDBCryptoKeys(keys []database.CryptoKey) []codersdk.CryptoKey { + wskeys := make([]codersdk.CryptoKey, 0, len(keys)) for _, key := range keys { - wskeys = append(wskeys, wsproxysdk.CryptoKey{ - Feature: wsproxysdk.CryptoKeyFeature(key.Feature), + wskeys = append(wskeys, codersdk.CryptoKey{ + Feature: codersdk.CryptoKeyFeature(key.Feature), Sequence: key.Sequence, StartsAt: key.StartsAt.UTC(), DeletesAt: key.DeletesAt.Time.UTC(), diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index e2a687517473a..e6f75a7a50ef8 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -995,9 +995,9 @@ func TestGetCryptoKeys(t *testing.T) { }) } -func fromDBCryptoKeys(key database.CryptoKey) wsproxysdk.CryptoKey { - return wsproxysdk.CryptoKey{ - Feature: wsproxysdk.CryptoKeyFeature(key.Feature), +func fromDBCryptoKeys(key database.CryptoKey) codersdk.CryptoKey { + return codersdk.CryptoKey{ + Feature: codersdk.CryptoKeyFeature(key.Feature), Sequence: key.Sequence, StartsAt: key.StartsAt.UTC(), DeletesAt: key.DeletesAt.Time.UTC(), diff --git a/enterprise/wsproxy/wsproxysdk/wsproxysdk.go b/enterprise/wsproxy/wsproxysdk/wsproxysdk.go index 891224216003a..77d36561c6de8 100644 --- a/enterprise/wsproxy/wsproxysdk/wsproxysdk.go +++ b/enterprise/wsproxy/wsproxysdk/wsproxysdk.go @@ -204,35 +204,6 @@ type RegisterWorkspaceProxyRequest struct { Version string `json:"version"` } -type CryptoKeyFeature string - -const ( - CryptoKeyFeatureWorkspaceApp CryptoKeyFeature = "workspace_apps" - CryptoKeyFeatureOIDCConvert CryptoKeyFeature = "oidc_convert" - CryptoKeyFeatureTailnetResume CryptoKeyFeature = "tailnet_resume" -) - -type CryptoKey struct { - Feature CryptoKeyFeature `json:"feature"` - Secret string `json:"secret"` - DeletesAt time.Time `json:"deletes_at"` - Sequence int32 `json:"sequence"` - StartsAt time.Time `json:"starts_at"` -} - -func (c CryptoKey) CanSign(now time.Time) bool { - now = now.UTC() - isAfterStartsAt := !c.StartsAt.IsZero() && !now.Before(c.StartsAt) - return isAfterStartsAt && c.CanVerify(now) -} - -func (c CryptoKey) CanVerify(now time.Time) bool { - now = now.UTC() - hasSecret := c.Secret != "" - beforeDelete := c.DeletesAt.IsZero() || now.Before(c.DeletesAt) - return hasSecret && beforeDelete -} - type RegisterWorkspaceProxyResponse struct { AppSecurityKey string `json:"app_security_key"` DERPMeshKey string `json:"derp_mesh_key"` @@ -612,7 +583,7 @@ func (c *Client) DialCoordinator(ctx context.Context) (agpl.MultiAgentConn, erro } type CryptoKeysResponse struct { - CryptoKeys []CryptoKey `json:"crypto_keys"` + CryptoKeys []codersdk.CryptoKey `json:"crypto_keys"` } func (c *Client) CryptoKeys(ctx context.Context) (CryptoKeysResponse, error) { From c63020ad781115a5100de0648bbe59a98c21e7f4 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 26 Sep 2024 22:16:29 +0000 Subject: [PATCH 04/15] Refactor CryptoKey conversion to use db2sdk package - Simplifies conversion and improves consistency by using a common package for translating database keys to SDK keys. - Improves test reliability and code maintainability by centralizing the conversion process. --- coderd/cryptokeys/dbkeycache.go | 7 ++----- enterprise/coderd/workspaceproxy.go | 17 ++--------------- enterprise/coderd/workspaceproxy_test.go | 15 +++------------ 3 files changed, 7 insertions(+), 32 deletions(-) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index 56c00fbbf97c4..de6b7e2f0c1a6 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -68,10 +68,7 @@ func (d *DBCache) Version(ctx context.Context, sequence int32) (codersdk.CryptoK key, ok := d.cache[sequence] d.cacheMu.RUnlock() if ok { - if !key.CanVerify(now) { - return codersdk.CryptoKey{}, ErrKeyInvalid - } - return db2sdk.CryptoKey(key), nil + return checkKey(key, now) } d.cacheMu.Lock() @@ -79,7 +76,7 @@ func (d *DBCache) Version(ctx context.Context, sequence int32) (codersdk.CryptoK key, ok = d.cache[sequence] if ok { - return db2sdk.CryptoKey(key), nil + return checkKey(key, now) } key, err := d.db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ diff --git a/enterprise/coderd/workspaceproxy.go b/enterprise/coderd/workspaceproxy.go index b085aedb09e5b..47bdf53493489 100644 --- a/enterprise/coderd/workspaceproxy.go +++ b/enterprise/coderd/workspaceproxy.go @@ -17,6 +17,7 @@ import ( agpl "github.com/coder/coder/v2/coderd" "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/httpapi" @@ -733,7 +734,7 @@ func (api *API) workspaceProxyCryptoKeys(rw http.ResponseWriter, r *http.Request } httpapi.Write(ctx, rw, http.StatusOK, wsproxysdk.CryptoKeysResponse{ - CryptoKeys: fromDBCryptoKeys(keys), + CryptoKeys: db2sdk.CryptoKeys(keys), }) } @@ -994,17 +995,3 @@ func (w *workspaceProxiesFetchUpdater) Fetch(ctx context.Context) (codersdk.Regi func (w *workspaceProxiesFetchUpdater) Update(ctx context.Context) error { return w.updateFunc(ctx) } - -func fromDBCryptoKeys(keys []database.CryptoKey) []codersdk.CryptoKey { - wskeys := make([]codersdk.CryptoKey, 0, len(keys)) - for _, key := range keys { - wskeys = append(wskeys, codersdk.CryptoKey{ - Feature: codersdk.CryptoKeyFeature(key.Feature), - Sequence: key.Sequence, - StartsAt: key.StartsAt.UTC(), - DeletesAt: key.DeletesAt.Time.UTC(), - Secret: key.Secret.String, - }) - } - return wskeys -} diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index e6f75a7a50ef8..4003cd55e503b 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -18,6 +18,7 @@ import ( "github.com/coder/coder/v2/buildinfo" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/database/dbtime" @@ -918,14 +919,14 @@ func TestGetCryptoKeys(t *testing.T) { StartsAt: now.Add(-time.Hour), Sequence: 2, }) - key1 := fromDBCryptoKeys(expectedKey1) + key1 := db2sdk.CryptoKey(expectedKey1) expectedKey2 := dbgen.CryptoKey(t, db, database.CryptoKey{ Feature: database.CryptoKeyFeatureWorkspaceApps, StartsAt: now, Sequence: 3, }) - key2 := fromDBCryptoKeys(expectedKey2) + key2 := db2sdk.CryptoKey(expectedKey2) // Create a deleted key. _ = dbgen.CryptoKey(t, db, database.CryptoKey{ @@ -994,13 +995,3 @@ func TestGetCryptoKeys(t *testing.T) { require.Equal(t, http.StatusUnauthorized, sdkErr.StatusCode()) }) } - -func fromDBCryptoKeys(key database.CryptoKey) codersdk.CryptoKey { - return codersdk.CryptoKey{ - Feature: codersdk.CryptoKeyFeature(key.Feature), - Sequence: key.Sequence, - StartsAt: key.StartsAt.UTC(), - DeletesAt: key.DeletesAt.Time.UTC(), - Secret: key.Secret.String, - } -} From 3e723ce85f80652d8935305184dcdc8701acc0b4 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 26 Sep 2024 22:29:07 +0000 Subject: [PATCH 05/15] make gen --- coderd/apidoc/docs.go | 70 +++++++++++++-------------- coderd/apidoc/swagger.json | 62 ++++++++++++------------ codersdk/deployment.go | 4 +- docs/reference/api/schemas.md | 86 +++++++++++++++++----------------- site/src/api/typesGenerated.ts | 13 +++++ 5 files changed, 126 insertions(+), 109 deletions(-) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index d6cc74cafbcaa..cb0736659ce4c 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -9848,6 +9848,41 @@ const docTemplate = `{ } } }, + "codersdk.CryptoKey": { + "type": "object", + "properties": { + "deletes_at": { + "type": "string", + "format": "date-time" + }, + "feature": { + "$ref": "#/definitions/codersdk.CryptoKeyFeature" + }, + "secret": { + "type": "string" + }, + "sequence": { + "type": "integer" + }, + "starts_at": { + "type": "string", + "format": "date-time" + } + } + }, + "codersdk.CryptoKeyFeature": { + "type": "string", + "enum": [ + "workspace_apps", + "oidc_convert", + "tailnet_resume" + ], + "x-enum-varnames": [ + "CryptoKeyFeatureWorkspaceApp", + "CryptoKeyFeatureOIDCConvert", + "CryptoKeyFeatureTailnetResume" + ] + }, "codersdk.CustomRoleRequest": { "type": "object", "properties": { @@ -15983,46 +16018,13 @@ const docTemplate = `{ } } }, - "wsproxysdk.CryptoKey": { - "type": "object", - "properties": { - "deletes_at": { - "type": "string" - }, - "feature": { - "$ref": "#/definitions/wsproxysdk.CryptoKeyFeature" - }, - "secret": { - "type": "string" - }, - "sequence": { - "type": "integer" - }, - "starts_at": { - "type": "string" - } - } - }, - "wsproxysdk.CryptoKeyFeature": { - "type": "string", - "enum": [ - "workspace_apps", - "oidc_convert", - "tailnet_resume" - ], - "x-enum-varnames": [ - "CryptoKeyFeatureWorkspaceApp", - "CryptoKeyFeatureOIDCConvert", - "CryptoKeyFeatureTailnetResume" - ] - }, "wsproxysdk.CryptoKeysResponse": { "type": "object", "properties": { "crypto_keys": { "type": "array", "items": { - "$ref": "#/definitions/wsproxysdk.CryptoKey" + "$ref": "#/definitions/codersdk.CryptoKey" } } } diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 579cb33bfaa6e..7ac09bd2bd8b9 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -8768,6 +8768,37 @@ } } }, + "codersdk.CryptoKey": { + "type": "object", + "properties": { + "deletes_at": { + "type": "string", + "format": "date-time" + }, + "feature": { + "$ref": "#/definitions/codersdk.CryptoKeyFeature" + }, + "secret": { + "type": "string" + }, + "sequence": { + "type": "integer" + }, + "starts_at": { + "type": "string", + "format": "date-time" + } + } + }, + "codersdk.CryptoKeyFeature": { + "type": "string", + "enum": ["workspace_apps", "oidc_convert", "tailnet_resume"], + "x-enum-varnames": [ + "CryptoKeyFeatureWorkspaceApp", + "CryptoKeyFeatureOIDCConvert", + "CryptoKeyFeatureTailnetResume" + ] + }, "codersdk.CustomRoleRequest": { "type": "object", "properties": { @@ -14614,42 +14645,13 @@ } } }, - "wsproxysdk.CryptoKey": { - "type": "object", - "properties": { - "deletes_at": { - "type": "string" - }, - "feature": { - "$ref": "#/definitions/wsproxysdk.CryptoKeyFeature" - }, - "secret": { - "type": "string" - }, - "sequence": { - "type": "integer" - }, - "starts_at": { - "type": "string" - } - } - }, - "wsproxysdk.CryptoKeyFeature": { - "type": "string", - "enum": ["workspace_apps", "oidc_convert", "tailnet_resume"], - "x-enum-varnames": [ - "CryptoKeyFeatureWorkspaceApp", - "CryptoKeyFeatureOIDCConvert", - "CryptoKeyFeatureTailnetResume" - ] - }, "wsproxysdk.CryptoKeysResponse": { "type": "object", "properties": { "crypto_keys": { "type": "array", "items": { - "$ref": "#/definitions/wsproxysdk.CryptoKey" + "$ref": "#/definitions/codersdk.CryptoKey" } } } diff --git a/codersdk/deployment.go b/codersdk/deployment.go index 7189657029dba..950537286d735 100644 --- a/codersdk/deployment.go +++ b/codersdk/deployment.go @@ -3115,9 +3115,9 @@ const ( type CryptoKey struct { Feature CryptoKeyFeature `json:"feature"` Secret string `json:"secret"` - DeletesAt time.Time `json:"deletes_at"` + DeletesAt time.Time `json:"deletes_at" format:"date-time"` Sequence int32 `json:"sequence"` - StartsAt time.Time `json:"starts_at"` + StartsAt time.Time `json:"starts_at" format:"date-time"` } func (c CryptoKey) CanSign(now time.Time) bool { diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 757d0aaab4ccb..7e0c3bb45a6b2 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -1406,6 +1406,44 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o | `template_version_id` | string | false | | Template version ID can be used to specify a specific version of a template for creating the workspace. | | `ttl_ms` | integer | false | | | +## codersdk.CryptoKey + +```json +{ + "deletes_at": "2019-08-24T14:15:22Z", + "feature": "workspace_apps", + "secret": "string", + "sequence": 0, + "starts_at": "2019-08-24T14:15:22Z" +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +| ------------ | ------------------------------------------------------ | -------- | ------------ | ----------- | +| `deletes_at` | string | false | | | +| `feature` | [codersdk.CryptoKeyFeature](#codersdkcryptokeyfeature) | false | | | +| `secret` | string | false | | | +| `sequence` | integer | false | | | +| `starts_at` | string | false | | | + +## codersdk.CryptoKeyFeature + +```json +"workspace_apps" +``` + +### Properties + +#### Enumerated Values + +| Value | +| ---------------- | +| `workspace_apps` | +| `oidc_convert` | +| `tailnet_resume` | + ## codersdk.CustomRoleRequest ```json @@ -9768,55 +9806,17 @@ _None_ | `derp_map` | [tailcfg.DERPMap](#tailcfgderpmap) | false | | | | `disable_direct_connections` | boolean | false | | | -## wsproxysdk.CryptoKey - -```json -{ - "deletes_at": "string", - "feature": "workspace_apps", - "secret": "string", - "sequence": 0, - "starts_at": "string" -} -``` - -### Properties - -| Name | Type | Required | Restrictions | Description | -| ------------ | ---------------------------------------------------------- | -------- | ------------ | ----------- | -| `deletes_at` | string | false | | | -| `feature` | [wsproxysdk.CryptoKeyFeature](#wsproxysdkcryptokeyfeature) | false | | | -| `secret` | string | false | | | -| `sequence` | integer | false | | | -| `starts_at` | string | false | | | - -## wsproxysdk.CryptoKeyFeature - -```json -"workspace_apps" -``` - -### Properties - -#### Enumerated Values - -| Value | -| ---------------- | -| `workspace_apps` | -| `oidc_convert` | -| `tailnet_resume` | - ## wsproxysdk.CryptoKeysResponse ```json { "crypto_keys": [ { - "deletes_at": "string", + "deletes_at": "2019-08-24T14:15:22Z", "feature": "workspace_apps", "secret": "string", "sequence": 0, - "starts_at": "string" + "starts_at": "2019-08-24T14:15:22Z" } ] } @@ -9824,9 +9824,9 @@ _None_ ### Properties -| Name | Type | Required | Restrictions | Description | -| ------------- | ----------------------------------------------------- | -------- | ------------ | ----------- | -| `crypto_keys` | array of [wsproxysdk.CryptoKey](#wsproxysdkcryptokey) | false | | | +| Name | Type | Required | Restrictions | Description | +| ------------- | ------------------------------------------------- | -------- | ------------ | ----------- | +| `crypto_keys` | array of [codersdk.CryptoKey](#codersdkcryptokey) | false | | | ## wsproxysdk.DeregisterWorkspaceProxyRequest diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 35f393e567f1e..971a0d8a4ccdc 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -343,6 +343,15 @@ export interface CreateWorkspaceRequest { readonly automatic_updates?: AutomaticUpdates; } +// From codersdk/deployment.go +export interface CryptoKey { + readonly feature: CryptoKeyFeature; + readonly secret: string; + readonly deletes_at: string; + readonly sequence: number; + readonly starts_at: string; +} + // From codersdk/roles.go export interface CustomRoleRequest { readonly name: string; @@ -2074,6 +2083,10 @@ export const AutomaticUpdateses: AutomaticUpdates[] = ["always", "never"] export type BuildReason = "autostart" | "autostop" | "initiator" export const BuildReasons: BuildReason[] = ["autostart", "autostop", "initiator"] +// From codersdk/deployment.go +export type CryptoKeyFeature = "oidc_convert" | "tailnet_resume" | "workspace_apps" +export const CryptoKeyFeatures: CryptoKeyFeature[] = ["oidc_convert", "tailnet_resume", "workspace_apps"] + // From codersdk/workspaceagents.go export type DisplayApp = "port_forwarding_helper" | "ssh_helper" | "vscode" | "vscode_insiders" | "web_terminal" export const DisplayApps: DisplayApp[] = ["port_forwarding_helper", "ssh_helper", "vscode", "vscode_insiders", "web_terminal"] From 201347f4daf31a5a571888d8c423812140903979 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 26 Sep 2024 22:30:46 +0000 Subject: [PATCH 06/15] Refactor key verification with db2sdk conversion --- coderd/cryptokeys/dbkeycache.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index de6b7e2f0c1a6..83c8728bebb0e 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -113,14 +113,14 @@ func (d *DBCache) Latest(ctx context.Context) (codersdk.CryptoKey, error) { now := d.clock.Now().UTC() if latest.CanSign(now) { - return checkKey(latest, now) + return db2sdk.CryptoKey(latest), nil } d.cacheMu.Lock() defer d.cacheMu.Unlock() if latest.CanSign(now) { - return checkKey(latest, now) + return db2sdk.CryptoKey(latest), nil } // Refetch all keys for this feature so we can find the latest valid key. @@ -139,7 +139,7 @@ func (d *DBCache) Latest(ctx context.Context) (codersdk.CryptoKey, error) { d.cache, d.latestKey = cache, latest - return checkKey(latest, now) + return db2sdk.CryptoKey(latest), nil } func (d *DBCache) refresh(ctx context.Context) { From 5f9744ba0d7c0ef99035497a5012d4102ae3cb84 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Sun, 29 Sep 2024 20:35:22 +0000 Subject: [PATCH 07/15] pr comments --- coderd/cryptokeys/dbkeycache.go | 155 +++++++------- coderd/cryptokeys/dbkeycache_internal_test.go | 198 ++++++++++++------ coderd/cryptokeys/dbkeycache_test.go | 145 +------------ coderd/cryptokeys/keycache.go | 4 +- coderd/database/db2sdk/db2sdk.go | 4 +- coderd/database/modelmethods.go | 6 +- 6 files changed, 228 insertions(+), 284 deletions(-) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index 83c8728bebb0e..aaa31fa22ffb9 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -2,7 +2,6 @@ package cryptokeys import ( "context" - "database/sql" "sync" "time" @@ -23,9 +22,10 @@ type DBCache struct { clock quartz.Clock // The following are initialized by NewDBCache. - cacheMu sync.RWMutex - cache map[int32]database.CryptoKey + keysMu sync.RWMutex + keys map[int32]database.CryptoKey latestKey database.CryptoKey + fetched chan struct{} } type DBCacheOption func(*DBCache) @@ -39,131 +39,118 @@ func WithDBCacheClock(clock quartz.Clock) DBCacheOption { // NewDBCache creates a new DBCache. It starts a background // process that periodically refreshes the cache. The context should // be canceled to stop the background process. -func NewDBCache(ctx context.Context, logger slog.Logger, db database.Store, feature database.CryptoKeyFeature, opts ...func(*DBCache)) (*DBCache, error) { +func NewDBCache(ctx context.Context, logger slog.Logger, db database.Store, feature database.CryptoKeyFeature, opts ...func(*DBCache)) *DBCache { d := &DBCache{ db: db, feature: feature, clock: quartz.NewReal(), logger: logger, + fetched: make(chan struct{}), } + for _, opt := range opts { opt(d) } - cache, latest, err := d.newCache(ctx) - if err != nil { - return nil, xerrors.Errorf("new cache: %w", err) - } - d.cache, d.latestKey = cache, latest - - go d.refresh(ctx) - return d, nil + go d.clear(ctx) + return d } -// Version returns the CryptoKey with the given sequence number, provided that -// it is neither deleted nor has breached its deletion date. -func (d *DBCache) Version(ctx context.Context, sequence int32) (codersdk.CryptoKey, error) { - now := d.clock.Now().UTC() - d.cacheMu.RLock() - key, ok := d.cache[sequence] - d.cacheMu.RUnlock() +// Verifying returns the CryptoKey with the given sequence number, provided that +// it is neither deleted nor has breached its deletion date. It should only be +// used for verifying or decrypting payloads. To sign/encrypt call Signing. +func (d *DBCache) Verifying(ctx context.Context, sequence int32) (codersdk.CryptoKey, error) { + now := d.clock.Now() + d.keysMu.RLock() + key, ok := d.keys[sequence] + d.keysMu.RUnlock() if ok { return checkKey(key, now) } - d.cacheMu.Lock() - defer d.cacheMu.Unlock() + d.keysMu.Lock() + defer d.keysMu.Unlock() - key, ok = d.cache[sequence] + key, ok = d.keys[sequence] if ok { return checkKey(key, now) } - key, err := d.db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ - Feature: d.feature, - Sequence: sequence, - }) - if xerrors.Is(err, sql.ErrNoRows) { - return codersdk.CryptoKey{}, ErrKeyNotFound - } + cache, latest, err := d.fetch(ctx) if err != nil { - return codersdk.CryptoKey{}, err - } - - if !key.CanVerify(now) { - return codersdk.CryptoKey{}, ErrKeyInvalid + return codersdk.CryptoKey{}, xerrors.Errorf("new cache: %w", err) } + d.keys, d.latestKey = cache, latest - // If this key is valid for signing then mark it as the latest key. - if key.CanSign(now) && key.Sequence > d.latestKey.Sequence { - d.latestKey = key + key, ok = d.keys[sequence] + if !ok { + return codersdk.CryptoKey{}, ErrKeyNotFound } - d.cache[sequence] = key - - return db2sdk.CryptoKey(key), nil + return checkKey(key, now) } -// Latest returns the latest valid key for signing. A valid key is one that is +// Signing returns the latest valid key for signing. A valid key is one that is // both past its start time and before its deletion time. -func (d *DBCache) Latest(ctx context.Context) (codersdk.CryptoKey, error) { - d.cacheMu.RLock() +func (d *DBCache) Signing(ctx context.Context) (codersdk.CryptoKey, error) { + d.keysMu.RLock() latest := d.latestKey - d.cacheMu.RUnlock() + d.keysMu.RUnlock() - now := d.clock.Now().UTC() + now := d.clock.Now() if latest.CanSign(now) { return db2sdk.CryptoKey(latest), nil } - d.cacheMu.Lock() - defer d.cacheMu.Unlock() + d.keysMu.Lock() + defer d.keysMu.Unlock() - if latest.CanSign(now) { - return db2sdk.CryptoKey(latest), nil + if d.latestKey.CanSign(now) { + return db2sdk.CryptoKey(d.latestKey), nil } // Refetch all keys for this feature so we can find the latest valid key. - cache, latest, err := d.newCache(ctx) + cache, latest, err := d.fetch(ctx) if err != nil { - return codersdk.CryptoKey{}, xerrors.Errorf("new cache: %w", err) + return codersdk.CryptoKey{}, xerrors.Errorf("fetch: %w", err) } + d.keys, d.latestKey = cache, latest - if len(cache) == 0 { - return codersdk.CryptoKey{}, ErrKeyNotFound - } - - if !latest.CanSign(now) { - return codersdk.CryptoKey{}, ErrKeyInvalid - } - - d.cache, d.latestKey = cache, latest - - return db2sdk.CryptoKey(latest), nil + return db2sdk.CryptoKey(d.latestKey), nil } -func (d *DBCache) refresh(ctx context.Context) { - d.clock.TickerFunc(ctx, time.Minute*10, func() error { - cache, latest, err := d.newCache(ctx) - if err != nil { - d.logger.Error(ctx, "failed to refresh cache", slog.Error(err)) - return nil +func (d *DBCache) clear(ctx context.Context) { + for { + fired := make(chan struct{}) + timer := d.clock.AfterFunc(time.Minute*10, func() { + defer close(fired) + + // There's a small window where the timer fires as we're fetching + // keys that could result in us immediately invalidating the cache that we just populated. + d.keysMu.Lock() + defer d.keysMu.Unlock() + d.keys = nil + d.latestKey = database.CryptoKey{} + }) + + select { + case <-ctx.Done(): + return + case <-d.fetched: + timer.Stop() + case <-fired: } - d.cacheMu.Lock() - defer d.cacheMu.Unlock() - - d.cache, d.latestKey = cache, latest - return nil - }) + } } -// newCache fetches all keys for the given feature and determines the latest key. -func (d *DBCache) newCache(ctx context.Context) (map[int32]database.CryptoKey, database.CryptoKey, error) { - now := d.clock.Now().UTC() +// fetch fetches all keys for the given feature and determines the latest key. +func (d *DBCache) fetch(ctx context.Context) (map[int32]database.CryptoKey, database.CryptoKey, error) { + now := d.clock.Now() keys, err := d.db.GetCryptoKeysByFeature(ctx, d.feature) if err != nil { return nil, database.CryptoKey{}, xerrors.Errorf("get crypto keys by feature: %w", err) } + cache := make(map[int32]database.CryptoKey) var latest database.CryptoKey for _, key := range keys { @@ -173,6 +160,20 @@ func (d *DBCache) newCache(ctx context.Context) (map[int32]database.CryptoKey, d } } + if len(cache) == 0 { + return nil, database.CryptoKey{}, ErrKeyNotFound + } + + if !latest.CanSign(now) { + return nil, database.CryptoKey{}, ErrKeyInvalid + } + + select { + case <-ctx.Done(): + return nil, database.CryptoKey{}, ctx.Err() + case d.fetched <- struct{}{}: + } + return cache, latest, nil } diff --git a/coderd/cryptokeys/dbkeycache_internal_test.go b/coderd/cryptokeys/dbkeycache_internal_test.go index f7b4c08e29ac0..845f7da76742f 100644 --- a/coderd/cryptokeys/dbkeycache_internal_test.go +++ b/coderd/cryptokeys/dbkeycache_internal_test.go @@ -8,6 +8,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbmock" @@ -15,7 +17,7 @@ import ( "github.com/coder/quartz" ) -func Test_Version(t *testing.T) { +func Test_Verifying(t *testing.T) { t.Parallel() t.Run("HitsCache", func(t *testing.T) { @@ -44,11 +46,11 @@ func Test_Version(t *testing.T) { k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, - cache: cache, + keys: cache, clock: clock, } - got, err := k.Version(ctx, 32) + got, err := k.Verifying(ctx, 32) require.NoError(t, err) require.Equal(t, db2sdk.CryptoKey(expectedKey), got) }) @@ -61,31 +63,24 @@ func Test_Version(t *testing.T) { mockDB = dbmock.NewMockStore(ctrl) clock = quartz.NewMock(t) ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) ) expectedKey := database.CryptoKey{ Feature: database.CryptoKeyFeatureWorkspaceApps, Sequence: 33, + StartsAt: clock.Now().UTC(), Secret: sql.NullString{ String: "secret", Valid: true, }, - StartsAt: clock.Now().UTC(), } - mockDB.EXPECT().GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ - Feature: database.CryptoKeyFeatureWorkspaceApps, - Sequence: 33, - }).Return(expectedKey, nil) + mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{expectedKey}, nil) - k := &DBCache{ - db: mockDB, - feature: database.CryptoKeyFeatureWorkspaceApps, - cache: map[int32]database.CryptoKey{}, - clock: clock, - } + k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) - got, err := k.Version(ctx, 33) + got, err := k.Verifying(ctx, 33) require.NoError(t, err) require.Equal(t, db2sdk.CryptoKey(expectedKey), got) require.Equal(t, db2sdk.CryptoKey(expectedKey), db2sdk.CryptoKey(k.latestKey)) @@ -119,11 +114,11 @@ func Test_Version(t *testing.T) { k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, - cache: cache, + keys: cache, clock: clock, } - _, err := k.Version(ctx, 32) + _, err := k.Verifying(ctx, 32) require.ErrorIs(t, err, ErrKeyInvalid) }) @@ -149,24 +144,21 @@ func Test_Version(t *testing.T) { Valid: true, }, } - mockDB.EXPECT().GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ - Feature: database.CryptoKeyFeatureWorkspaceApps, - Sequence: 32, - }).Return(invalidKey, nil) + mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{invalidKey}, nil) k := &DBCache{ db: mockDB, feature: database.CryptoKeyFeatureWorkspaceApps, - cache: map[int32]database.CryptoKey{}, + keys: map[int32]database.CryptoKey{}, clock: clock, } - _, err := k.Version(ctx, 32) + _, err := k.Verifying(ctx, 32) require.ErrorIs(t, err, ErrKeyInvalid) }) } -func Test_Latest(t *testing.T) { +func Test_Signing(t *testing.T) { t.Parallel() t.Run("HitsCache", func(t *testing.T) { @@ -177,6 +169,7 @@ func Test_Latest(t *testing.T) { mockDB = dbmock.NewMockStore(ctrl) clock = quartz.NewMock(t) ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) ) latestKey := database.CryptoKey{ @@ -188,14 +181,10 @@ func Test_Latest(t *testing.T) { }, StartsAt: clock.Now().UTC(), } - k := &DBCache{ - db: mockDB, - feature: database.CryptoKeyFeatureWorkspaceApps, - clock: clock, - latestKey: latestKey, - } + k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + k.latestKey = latestKey - got, err := k.Latest(ctx) + got, err := k.Signing(ctx) require.NoError(t, err) require.Equal(t, db2sdk.CryptoKey(latestKey), got) }) @@ -208,6 +197,7 @@ func Test_Latest(t *testing.T) { mockDB = dbmock.NewMockStore(ctrl) clock = quartz.NewMock(t) ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) ) latestKey := database.CryptoKey{ @@ -220,28 +210,26 @@ func Test_Latest(t *testing.T) { StartsAt: clock.Now().UTC(), } - mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{latestKey}, nil) - - k := &DBCache{ - db: mockDB, - feature: database.CryptoKeyFeatureWorkspaceApps, - clock: clock, - latestKey: database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, - Sequence: 32, - Secret: sql.NullString{ - String: "secret", - Valid: true, - }, - StartsAt: clock.Now().UTC().Add(-time.Hour), - DeletesAt: sql.NullTime{ - Time: clock.Now().UTC(), - Valid: true, - }, + invalidKey := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now().UTC().Add(-time.Hour), + DeletesAt: sql.NullTime{ + Time: clock.Now().UTC(), + Valid: true, }, } - got, err := k.Latest(ctx) + mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{latestKey}, nil) + + k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + k.latestKey = invalidKey + + got, err := k.Signing(ctx) require.NoError(t, err) require.Equal(t, db2sdk.CryptoKey(latestKey), got) }) @@ -254,6 +242,7 @@ func Test_Latest(t *testing.T) { mockDB = dbmock.NewMockStore(ctrl) clock = quartz.NewMock(t) ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) ) inactiveKey := database.CryptoKey{ @@ -278,14 +267,9 @@ func Test_Latest(t *testing.T) { mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{inactiveKey, activeKey}, nil) - k := &DBCache{ - db: mockDB, - feature: database.CryptoKeyFeatureWorkspaceApps, - clock: clock, - cache: map[int32]database.CryptoKey{}, - } + k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) - got, err := k.Latest(ctx) + got, err := k.Signing(ctx) require.NoError(t, err) require.Equal(t, db2sdk.CryptoKey(activeKey), got) }) @@ -298,6 +282,7 @@ func Test_Latest(t *testing.T) { mockDB = dbmock.NewMockStore(ctrl) clock = quartz.NewMock(t) ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) ) inactiveKey := database.CryptoKey{ @@ -326,14 +311,99 @@ func Test_Latest(t *testing.T) { mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{inactiveKey, invalidKey}, nil) - k := &DBCache{ - db: mockDB, - feature: database.CryptoKeyFeatureWorkspaceApps, - clock: clock, - cache: map[int32]database.CryptoKey{}, - } + k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) - _, err := k.Latest(ctx) + _, err := k.Signing(ctx) require.ErrorIs(t, err, ErrKeyInvalid) }) } + +func Test_clear(t *testing.T) { + t.Parallel() + + t.Run("InvalidatesCache", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) + ) + + trap := clock.Trap().AfterFunc() + + k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + k.latestKey = database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + } + k.keys = map[int32]database.CryptoKey{ + 32: { + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + }, + } + trap.MustWait(ctx).Release() + _, wait := clock.AdvanceNext() + wait.MustWait(ctx) + require.Len(t, k.keys, 0) + require.Equal(t, database.CryptoKey{}, k.latestKey) + }) + + t.Run("ResetsTimer", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) + ) + + trap := clock.Trap().AfterFunc() + + k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + + key := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now(), + } + + mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{key}, nil) + + trap.MustWait(ctx).Release() + + // Advance it five minutes so that we can test that the + // time is reset and doesn't fire after another five minute and doesn't fire after another five minutes. + clock.Advance(time.Minute * 5) + + latest, err := k.Signing(ctx) + require.NoError(t, err) + require.Equal(t, db2sdk.CryptoKey(key), latest) + + trap.MustWait(ctx).Release() + // Advancing the clock now should require 10 minutes + // before the timer fires again. + dur, wait := clock.AdvanceNext() + wait.MustWait(ctx) + require.Equal(t, time.Minute*10, dur) + require.Len(t, k.keys, 0) + require.Equal(t, database.CryptoKey{}, k.latestKey) + }) +} diff --git a/coderd/cryptokeys/dbkeycache_test.go b/coderd/cryptokeys/dbkeycache_test.go index 2848309009367..d8517984e9de2 100644 --- a/coderd/cryptokeys/dbkeycache_test.go +++ b/coderd/cryptokeys/dbkeycache_test.go @@ -1,9 +1,7 @@ package cryptokeys_test import ( - "database/sql" "testing" - "time" "github.com/stretchr/testify/require" @@ -21,21 +19,7 @@ import ( func TestDBKeyCache(t *testing.T) { t.Parallel() - t.Run("NoKeys", func(t *testing.T) { - t.Parallel() - - var ( - db, _ = dbtestutil.NewDB(t) - clock = quartz.NewMock(t) - ctx = testutil.Context(t, testutil.WaitShort) - logger = slogtest.Make(t, nil) - ) - - _, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) - require.NoError(t, err) - }) - - t.Run("Version", func(t *testing.T) { + t.Run("Verifying", func(t *testing.T) { t.Parallel() t.Run("HitsCache", func(t *testing.T) { @@ -51,22 +35,17 @@ func TestDBKeyCache(t *testing.T) { key := dbgen.CryptoKey(t, db, database.CryptoKey{ Feature: database.CryptoKeyFeatureWorkspaceApps, Sequence: 1, - Secret: sql.NullString{ - String: "secret", - Valid: true, - }, StartsAt: clock.Now().UTC(), }) - k, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) - require.NoError(t, err) + k := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) - got, err := k.Version(ctx, key.Sequence) + got, err := k.Verifying(ctx, key.Sequence) require.NoError(t, err) require.Equal(t, db2sdk.CryptoKey(key), got) }) - t.Run("MissesCache", func(t *testing.T) { + t.Run("NotFound", func(t *testing.T) { t.Parallel() var ( @@ -76,36 +55,14 @@ func TestDBKeyCache(t *testing.T) { logger = slogtest.Make(t, nil) ) - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, - Sequence: 1, - Secret: sql.NullString{ - String: "secret", - Valid: true, - }, - StartsAt: clock.Now().UTC(), - }) - - k, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) - require.NoError(t, err) - - key := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, - Sequence: 3, - Secret: sql.NullString{ - String: "secret", - Valid: true, - }, - StartsAt: clock.Now().UTC(), - }) + k := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) - got, err := k.Version(ctx, key.Sequence) - require.NoError(t, err) - require.Equal(t, db2sdk.CryptoKey(key), got) + _, err := k.Verifying(ctx, 123) + require.ErrorIs(t, err, cryptokeys.ErrKeyNotFound) }) }) - t.Run("Latest", func(t *testing.T) { + t.Run("Signing", func(t *testing.T) { t.Parallel() var ( @@ -133,92 +90,10 @@ func TestDBKeyCache(t *testing.T) { StartsAt: clock.Now().UTC(), }) - k, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) - require.NoError(t, err) + k := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) - got, err := k.Latest(ctx) + got, err := k.Signing(ctx) require.NoError(t, err) require.Equal(t, db2sdk.CryptoKey(expectedKey), got) }) - - t.Run("CacheRefreshes", func(t *testing.T) { - t.Parallel() - - var ( - db, _ = dbtestutil.NewDB(t) - clock = quartz.NewMock(t) - ctx = testutil.Context(t, testutil.WaitShort) - logger = slogtest.Make(t, nil) - ) - - expiringKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, - Sequence: 12, - StartsAt: clock.Now().UTC(), - DeletesAt: sql.NullTime{ - Time: clock.Now().UTC().Add(time.Minute * 10), - Valid: true, - }, - }) - latest := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, - Sequence: 24, - StartsAt: clock.Now().UTC(), - DeletesAt: sql.NullTime{ - Time: clock.Now().UTC().Add(2 * 2 * time.Hour), - Valid: true, - }, - }) - - wrongFeature := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureOidcConvert, - Sequence: 30, - StartsAt: clock.Now().UTC(), - }) - - trap := clock.Trap().TickerFunc() - k, err := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) - require.NoError(t, err) - - // Should be able to fetch the expiring key since it's still valid. - got, err := k.Version(ctx, expiringKey.Sequence) - require.NoError(t, err) - require.Equal(t, db2sdk.CryptoKey(expiringKey), got) - - _, err = k.Version(ctx, wrongFeature.Sequence) - require.ErrorIs(t, err, cryptokeys.ErrKeyNotFound) - - newLatest := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, - Sequence: 25, - StartsAt: clock.Now().UTC(), - DeletesAt: sql.NullTime{ - Time: clock.Now().UTC().Add(2 * 2 * time.Hour), - Valid: true, - }, - }) - - // The latest key should not be the one we just generated. - got, err = k.Latest(ctx) - require.NoError(t, err) - require.Equal(t, db2sdk.CryptoKey(latest), got) - - // Wait for the ticker to fire and the cache to refresh. - trap.MustWait(ctx).Release() - _, wait := clock.AdvanceNext() - wait.MustWait(ctx) - - // The latest key should be the one we just generated. - got, err = k.Latest(ctx) - require.NoError(t, err) - require.Equal(t, db2sdk.CryptoKey(newLatest), got) - - // The expiring key should be invalid. - _, err = k.Version(ctx, expiringKey.Sequence) - require.ErrorIs(t, err, cryptokeys.ErrKeyInvalid) - - // Sanity check that the wrong feature is still not found. - _, err = k.Version(ctx, wrongFeature.Sequence) - require.ErrorIs(t, err, cryptokeys.ErrKeyNotFound) - }) } diff --git a/coderd/cryptokeys/keycache.go b/coderd/cryptokeys/keycache.go index 503edec2ca63c..f06cd425676ce 100644 --- a/coderd/cryptokeys/keycache.go +++ b/coderd/cryptokeys/keycache.go @@ -14,6 +14,6 @@ var ErrKeyInvalid = xerrors.New("key is invalid for use") // Keycache provides an abstraction for fetching signing keys. type Keycache interface { - Latest(ctx context.Context) (codersdk.CryptoKey, error) - Version(ctx context.Context, sequence int32) (codersdk.CryptoKey, error) + Signing(ctx context.Context) (codersdk.CryptoKey, error) + Verifying(ctx context.Context, sequence int32) (codersdk.CryptoKey, error) } diff --git a/coderd/database/db2sdk/db2sdk.go b/coderd/database/db2sdk/db2sdk.go index 59f683e6ce0e6..a0e8977ff8879 100644 --- a/coderd/database/db2sdk/db2sdk.go +++ b/coderd/database/db2sdk/db2sdk.go @@ -668,8 +668,8 @@ func CryptoKey(key database.CryptoKey) codersdk.CryptoKey { return codersdk.CryptoKey{ Feature: codersdk.CryptoKeyFeature(key.Feature), Sequence: key.Sequence, - StartsAt: key.StartsAt.UTC(), - DeletesAt: key.DeletesAt.Time.UTC(), + StartsAt: key.StartsAt, + DeletesAt: key.DeletesAt.Time, Secret: key.Secret.String, } } diff --git a/coderd/database/modelmethods.go b/coderd/database/modelmethods.go index d93a458dccee9..846de6e36aa47 100644 --- a/coderd/database/modelmethods.go +++ b/coderd/database/modelmethods.go @@ -458,14 +458,12 @@ func (k CryptoKey) DecodeString() ([]byte, error) { } func (k CryptoKey) CanSign(now time.Time) bool { - now = now.UTC() - isAfterStart := !k.StartsAt.IsZero() && !now.Before(k.StartsAt.UTC()) + isAfterStart := !k.StartsAt.IsZero() && !now.Before(k.StartsAt) return isAfterStart && k.CanVerify(now) } func (k CryptoKey) CanVerify(now time.Time) bool { - now = now.UTC() hasSecret := k.Secret.Valid - isBeforeDeletion := !k.DeletesAt.Valid || now.Before(k.DeletesAt.Time.UTC()) + isBeforeDeletion := !k.DeletesAt.Valid || now.Before(k.DeletesAt.Time) return hasSecret && isBeforeDeletion } From 580309267eabefd27f6f590acb7d29a86bc074fb Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Mon, 30 Sep 2024 01:45:10 +0000 Subject: [PATCH 08/15] Refactor error message for key fetching --- coderd/cryptokeys/dbkeycache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index aaa31fa22ffb9..5784952692fbe 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -78,7 +78,7 @@ func (d *DBCache) Verifying(ctx context.Context, sequence int32) (codersdk.Crypt cache, latest, err := d.fetch(ctx) if err != nil { - return codersdk.CryptoKey{}, xerrors.Errorf("new cache: %w", err) + return codersdk.CryptoKey{}, xerrors.Errorf("fetch: %w", err) } d.keys, d.latestKey = cache, latest From ea659b0abef9fe867ed9d4b7b942781044f646b1 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 1 Oct 2024 06:28:36 +0000 Subject: [PATCH 09/15] Refactor DBKeyCache with robust invalidation timing Introduce a timer-based cache invalidation system in DBCache to enhance reliability. The new implementation minimizes cache invalidation race conditions during key fetching, ensuring consistent cache state management. Additions include a 'Close' method for releasing resources, such as timers, and test improvements for timer behavior validation. --- coderd/cryptokeys/dbkeycache.go | 91 ++++----- coderd/cryptokeys/dbkeycache_internal_test.go | 174 ++++++++++++------ coderd/cryptokeys/dbkeycache_test.go | 9 +- coderd/database/dbgen/dbgen.go | 2 +- enterprise/coderd/workspaceproxy_test.go | 22 ++- 5 files changed, 188 insertions(+), 110 deletions(-) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index 5784952692fbe..610f08e834028 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -14,6 +14,9 @@ import ( "github.com/coder/quartz" ) +// never represents the maximum value for a time.Duration. +const never = 1<<63 - 1 + // DBCache implements Keycache for callers with access to the database. type DBCache struct { db database.Store @@ -25,7 +28,9 @@ type DBCache struct { keysMu sync.RWMutex keys map[int32]database.CryptoKey latestKey database.CryptoKey - fetched chan struct{} + timer *quartz.Timer + // invalidateAt is the time at which the keys cache should be invalidated. + invalidateAt time.Time } type DBCacheOption func(*DBCache) @@ -36,23 +41,22 @@ func WithDBCacheClock(clock quartz.Clock) DBCacheOption { } } -// NewDBCache creates a new DBCache. It starts a background -// process that periodically refreshes the cache. The context should -// be canceled to stop the background process. -func NewDBCache(ctx context.Context, logger slog.Logger, db database.Store, feature database.CryptoKeyFeature, opts ...func(*DBCache)) *DBCache { +// NewDBCache creates a new DBCache. Close should be called to +// release resources associated with its internal timer. +func NewDBCache(logger slog.Logger, db database.Store, feature database.CryptoKeyFeature, opts ...func(*DBCache)) *DBCache { d := &DBCache{ db: db, feature: feature, clock: quartz.NewReal(), logger: logger, - fetched: make(chan struct{}), } for _, opt := range opts { opt(d) } - go d.clear(ctx) + d.timer = d.clock.AfterFunc(never, d.clear) + return d } @@ -76,11 +80,10 @@ func (d *DBCache) Verifying(ctx context.Context, sequence int32) (codersdk.Crypt return checkKey(key, now) } - cache, latest, err := d.fetch(ctx) + err := d.fetch(ctx) if err != nil { return codersdk.CryptoKey{}, xerrors.Errorf("fetch: %w", err) } - d.keys, d.latestKey = cache, latest key, ok = d.keys[sequence] if !ok { @@ -110,47 +113,42 @@ func (d *DBCache) Signing(ctx context.Context) (codersdk.CryptoKey, error) { } // Refetch all keys for this feature so we can find the latest valid key. - cache, latest, err := d.fetch(ctx) + err := d.fetch(ctx) if err != nil { return codersdk.CryptoKey{}, xerrors.Errorf("fetch: %w", err) } - d.keys, d.latestKey = cache, latest return db2sdk.CryptoKey(d.latestKey), nil } -func (d *DBCache) clear(ctx context.Context) { - for { - fired := make(chan struct{}) - timer := d.clock.AfterFunc(time.Minute*10, func() { - defer close(fired) - - // There's a small window where the timer fires as we're fetching - // keys that could result in us immediately invalidating the cache that we just populated. - d.keysMu.Lock() - defer d.keysMu.Unlock() - d.keys = nil - d.latestKey = database.CryptoKey{} - }) - - select { - case <-ctx.Done(): - return - case <-d.fetched: - timer.Stop() - case <-fired: - } - } +// clear invalidates the cache. This forces the subsequent call to fetch fresh keys. +func (d *DBCache) clear() { + now := d.clock.Now("DBCache", "clear") + d.keysMu.Lock() + defer d.keysMu.Unlock() + // Check if we raced with a fetch. It's possible that the timer fired and we + // lost the race to the mutex. We want to avoid invalidating + // a cache that was just refetched. + if now.Before(d.invalidateAt) { + return + } + d.keys = nil + d.latestKey = database.CryptoKey{} } // fetch fetches all keys for the given feature and determines the latest key. -func (d *DBCache) fetch(ctx context.Context) (map[int32]database.CryptoKey, database.CryptoKey, error) { - now := d.clock.Now() +// It must be called while holding the keysMu lock. +func (d *DBCache) fetch(ctx context.Context) error { keys, err := d.db.GetCryptoKeysByFeature(ctx, d.feature) if err != nil { - return nil, database.CryptoKey{}, xerrors.Errorf("get crypto keys by feature: %w", err) + return xerrors.Errorf("get crypto keys by feature: %w", err) } + now := d.clock.Now() + d.timer.Stop() + d.timer = d.newTimer() + d.invalidateAt = now.Add(time.Minute * 10) + cache := make(map[int32]database.CryptoKey) var latest database.CryptoKey for _, key := range keys { @@ -161,20 +159,15 @@ func (d *DBCache) fetch(ctx context.Context) (map[int32]database.CryptoKey, data } if len(cache) == 0 { - return nil, database.CryptoKey{}, ErrKeyNotFound + return ErrKeyNotFound } if !latest.CanSign(now) { - return nil, database.CryptoKey{}, ErrKeyInvalid + return ErrKeyInvalid } - select { - case <-ctx.Done(): - return nil, database.CryptoKey{}, ctx.Err() - case d.fetched <- struct{}{}: - } - - return cache, latest, nil + d.keys, d.latestKey = cache, latest + return nil } func checkKey(key database.CryptoKey, now time.Time) (codersdk.CryptoKey, error) { @@ -184,3 +177,11 @@ func checkKey(key database.CryptoKey, now time.Time) (codersdk.CryptoKey, error) return db2sdk.CryptoKey(key), nil } + +func (d *DBCache) newTimer() *quartz.Timer { + return d.clock.AfterFunc(time.Minute*10, d.clear) +} + +func (d *DBCache) Close() { + d.timer.Stop() +} diff --git a/coderd/cryptokeys/dbkeycache_internal_test.go b/coderd/cryptokeys/dbkeycache_internal_test.go index 845f7da76742f..a3450f5f5e0d9 100644 --- a/coderd/cryptokeys/dbkeycache_internal_test.go +++ b/coderd/cryptokeys/dbkeycache_internal_test.go @@ -27,6 +27,7 @@ func Test_Verifying(t *testing.T) { ctrl = gomock.NewController(t) mockDB = dbmock.NewMockStore(ctrl) clock = quartz.NewMock(t) + logger = slogtest.Make(t, nil) ctx = testutil.Context(t, testutil.WaitShort) ) @@ -43,12 +44,9 @@ func Test_Verifying(t *testing.T) { 32: expectedKey, } - k := &DBCache{ - db: mockDB, - feature: database.CryptoKeyFeatureWorkspaceApps, - keys: cache, - clock: clock, - } + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() + k.keys = cache got, err := k.Verifying(ctx, 32) require.NoError(t, err) @@ -69,7 +67,7 @@ func Test_Verifying(t *testing.T) { expectedKey := database.CryptoKey{ Feature: database.CryptoKeyFeatureWorkspaceApps, Sequence: 33, - StartsAt: clock.Now().UTC(), + StartsAt: clock.Now(), Secret: sql.NullString{ String: "secret", Valid: true, @@ -78,7 +76,8 @@ func Test_Verifying(t *testing.T) { mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{expectedKey}, nil) - k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() got, err := k.Verifying(ctx, 33) require.NoError(t, err) @@ -94,6 +93,7 @@ func Test_Verifying(t *testing.T) { mockDB = dbmock.NewMockStore(ctrl) clock = quartz.NewMock(t) ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) ) cache := map[int32]database.CryptoKey{ @@ -105,18 +105,15 @@ func Test_Verifying(t *testing.T) { Valid: true, }, DeletesAt: sql.NullTime{ - Time: clock.Now().UTC(), + Time: clock.Now(), Valid: true, }, }, } - k := &DBCache{ - db: mockDB, - feature: database.CryptoKeyFeatureWorkspaceApps, - keys: cache, - clock: clock, - } + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() + k.keys = cache _, err := k.Verifying(ctx, 32) require.ErrorIs(t, err, ErrKeyInvalid) @@ -130,6 +127,7 @@ func Test_Verifying(t *testing.T) { mockDB = dbmock.NewMockStore(ctrl) clock = quartz.NewMock(t) ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) ) invalidKey := database.CryptoKey{ @@ -140,18 +138,14 @@ func Test_Verifying(t *testing.T) { Valid: true, }, DeletesAt: sql.NullTime{ - Time: clock.Now().UTC(), + Time: clock.Now(), Valid: true, }, } mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{invalidKey}, nil) - k := &DBCache{ - db: mockDB, - feature: database.CryptoKeyFeatureWorkspaceApps, - keys: map[int32]database.CryptoKey{}, - clock: clock, - } + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() _, err := k.Verifying(ctx, 32) require.ErrorIs(t, err, ErrKeyInvalid) @@ -179,9 +173,11 @@ func Test_Signing(t *testing.T) { String: "secret", Valid: true, }, - StartsAt: clock.Now().UTC(), + StartsAt: clock.Now(), } - k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() + k.latestKey = latestKey got, err := k.Signing(ctx) @@ -207,7 +203,7 @@ func Test_Signing(t *testing.T) { String: "secret", Valid: true, }, - StartsAt: clock.Now().UTC(), + StartsAt: clock.Now(), } invalidKey := database.CryptoKey{ @@ -217,16 +213,17 @@ func Test_Signing(t *testing.T) { String: "secret", Valid: true, }, - StartsAt: clock.Now().UTC().Add(-time.Hour), + StartsAt: clock.Now().Add(-time.Hour), DeletesAt: sql.NullTime{ - Time: clock.Now().UTC(), + Time: clock.Now(), Valid: true, }, } mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{latestKey}, nil) - k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() k.latestKey = invalidKey got, err := k.Signing(ctx) @@ -252,7 +249,7 @@ func Test_Signing(t *testing.T) { String: "secret", Valid: true, }, - StartsAt: clock.Now().UTC().Add(time.Hour), + StartsAt: clock.Now().Add(time.Hour), } activeKey := database.CryptoKey{ @@ -262,12 +259,13 @@ func Test_Signing(t *testing.T) { String: "secret", Valid: true, }, - StartsAt: clock.Now().UTC(), + StartsAt: clock.Now(), } mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{inactiveKey, activeKey}, nil) - k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() got, err := k.Signing(ctx) require.NoError(t, err) @@ -292,7 +290,7 @@ func Test_Signing(t *testing.T) { String: "secret", Valid: true, }, - StartsAt: clock.Now().UTC().Add(time.Hour), + StartsAt: clock.Now().Add(time.Hour), } invalidKey := database.CryptoKey{ @@ -302,16 +300,17 @@ func Test_Signing(t *testing.T) { String: "secret", Valid: true, }, - StartsAt: clock.Now().UTC().Add(-time.Hour), + StartsAt: clock.Now().Add(-time.Hour), DeletesAt: sql.NullTime{ - Time: clock.Now().UTC(), + Time: clock.Now(), Valid: true, }, } mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{inactiveKey, invalidKey}, nil) - k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() _, err := k.Signing(ctx) require.ErrorIs(t, err, ErrKeyInvalid) @@ -332,30 +331,27 @@ func Test_clear(t *testing.T) { logger = slogtest.Make(t, nil) ) - trap := clock.Trap().AfterFunc() + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() - k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) - k.latestKey = database.CryptoKey{ + activeKey := database.CryptoKey{ Feature: database.CryptoKeyFeatureWorkspaceApps, - Sequence: 32, + Sequence: 33, Secret: sql.NullString{ String: "secret", Valid: true, }, + StartsAt: clock.Now(), } - k.keys = map[int32]database.CryptoKey{ - 32: { - Feature: database.CryptoKeyFeatureWorkspaceApps, - Sequence: 32, - Secret: sql.NullString{ - String: "secret", - Valid: true, - }, - }, - } - trap.MustWait(ctx).Release() - _, wait := clock.AdvanceNext() + + mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{activeKey}, nil) + + _, err := k.Signing(ctx) + require.NoError(t, err) + + dur, wait := clock.AdvanceNext() wait.MustWait(ctx) + require.Equal(t, time.Minute*10, dur) require.Len(t, k.keys, 0) require.Equal(t, database.CryptoKey{}, k.latestKey) }) @@ -371,9 +367,8 @@ func Test_clear(t *testing.T) { logger = slogtest.Make(t, nil) ) - trap := clock.Trap().AfterFunc() - - k := NewDBCache(ctx, logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() key := database.CryptoKey{ Feature: database.CryptoKeyFeatureWorkspaceApps, @@ -387,17 +382,14 @@ func Test_clear(t *testing.T) { mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{key}, nil) - trap.MustWait(ctx).Release() - // Advance it five minutes so that we can test that the - // time is reset and doesn't fire after another five minute and doesn't fire after another five minutes. + // timer is reset and doesn't fire after another five minute. clock.Advance(time.Minute * 5) latest, err := k.Signing(ctx) require.NoError(t, err) require.Equal(t, db2sdk.CryptoKey(key), latest) - trap.MustWait(ctx).Release() // Advancing the clock now should require 10 minutes // before the timer fires again. dur, wait := clock.AdvanceNext() @@ -406,4 +398,70 @@ func Test_clear(t *testing.T) { require.Len(t, k.keys, 0) require.Equal(t, database.CryptoKey{}, k.latestKey) }) + + // InvalidateAt tests that we have accounted for the race condition where a + // timer fires to invalidate the cache at the same time we are fetching new + // keys. In such cases we want to skip invalidation. + t.Run("InvalidateAt", func(t *testing.T) { + t.Parallel() + + var ( + ctrl = gomock.NewController(t) + mockDB = dbmock.NewMockStore(ctrl) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) + ) + + trap := clock.Trap().Now("clear") + + k := NewDBCache(logger, mockDB, database.CryptoKeyFeatureWorkspaceApps, WithDBCacheClock(clock)) + defer k.Close() + + key := database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 32, + Secret: sql.NullString{ + String: "secret", + Valid: true, + }, + StartsAt: clock.Now(), + } + + mockDB.EXPECT().GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps).Return([]database.CryptoKey{key}, nil).Times(2) + + // Move us past the initial timer. + latest, err := k.Signing(ctx) + require.NoError(t, err) + require.Equal(t, db2sdk.CryptoKey(key), latest) + // Null these out so that we refetch. + k.keys = nil + k.latestKey = database.CryptoKey{} + + // Initiate firing the timer. + dur, wait := clock.AdvanceNext() + require.Equal(t, time.Minute*10, dur) + // Trap the function just before acquiring the mutex. + call := trap.MustWait(ctx) + + // Refetch keys. + latest, err = k.Signing(ctx) + require.NoError(t, err) + require.Equal(t, db2sdk.CryptoKey(key), latest) + + // Let the rest of the timer function run. + // It should see that we have refetched keys and + // not invalidate. + call.Release() + wait.MustWait(ctx) + require.Len(t, k.keys, 1) + require.Equal(t, key, k.latestKey) + trap.Close() + + // Refetching the keys should've instantiated a new timer. This one should invalidate keys. + _, wait = clock.AdvanceNext() + wait.MustWait(ctx) + require.Len(t, k.keys, 0) + require.Equal(t, database.CryptoKey{}, k.latestKey) + }) } diff --git a/coderd/cryptokeys/dbkeycache_test.go b/coderd/cryptokeys/dbkeycache_test.go index d8517984e9de2..6eb5de5182e4f 100644 --- a/coderd/cryptokeys/dbkeycache_test.go +++ b/coderd/cryptokeys/dbkeycache_test.go @@ -38,7 +38,8 @@ func TestDBKeyCache(t *testing.T) { StartsAt: clock.Now().UTC(), }) - k := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) + k := cryptokeys.NewDBCache(logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) + defer k.Close() got, err := k.Verifying(ctx, key.Sequence) require.NoError(t, err) @@ -55,7 +56,8 @@ func TestDBKeyCache(t *testing.T) { logger = slogtest.Make(t, nil) ) - k := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) + k := cryptokeys.NewDBCache(logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) + defer k.Close() _, err := k.Verifying(ctx, 123) require.ErrorIs(t, err, cryptokeys.ErrKeyNotFound) @@ -90,7 +92,8 @@ func TestDBKeyCache(t *testing.T) { StartsAt: clock.Now().UTC(), }) - k := cryptokeys.NewDBCache(ctx, logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) + k := cryptokeys.NewDBCache(logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) + defer k.Close() got, err := k.Signing(ctx) require.NoError(t, err) diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index d18da855be7b8..1a2f052a279b3 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -920,7 +920,7 @@ func CryptoKey(t testing.TB, db database.Store, seed database.CryptoKey) databas Secret: seed.Secret, SecretKeyID: takeFirst(seed.SecretKeyID, sql.NullString{}), Feature: seed.Feature, - StartsAt: takeFirst(seed.StartsAt, time.Now()), + StartsAt: takeFirst(seed.StartsAt, dbtime.Now()), }) require.NoError(t, err, "insert crypto key") diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index 4003cd55e503b..5231a0b0c4241 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -912,7 +912,7 @@ func TestGetCryptoKeys(t *testing.T) { }, }) - now := time.Now().UTC() + now := time.Now() expectedKey1 := dbgen.CryptoKey(t, db, database.CryptoKey{ Feature: database.CryptoKeyFeatureWorkspaceApps, @@ -959,8 +959,7 @@ func TestGetCryptoKeys(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, keys) require.Equal(t, 2, len(keys.CryptoKeys)) - require.Contains(t, keys.CryptoKeys, key1) - require.Contains(t, keys.CryptoKeys, key2) + requireContainsKeys(t, keys.CryptoKeys, key1, key2) }) t.Run("Unauthorized", func(t *testing.T) { @@ -995,3 +994,20 @@ func TestGetCryptoKeys(t *testing.T) { require.Equal(t, http.StatusUnauthorized, sdkErr.StatusCode()) }) } + +func requireContainsKeys(t *testing.T, keys []codersdk.CryptoKey, expected ...codersdk.CryptoKey) { + t.Helper() + + for _, expectedKey := range expected { + var found bool + for _, key := range keys { + if key.Feature == expectedKey.Feature && key.Sequence == expectedKey.Sequence { + require.True(t, expectedKey.StartsAt.Equal(key.StartsAt), "expected starts at %s, got %s", expectedKey.StartsAt, key.StartsAt) + require.Equal(t, expectedKey.Secret, key.Secret) + require.True(t, expectedKey.DeletesAt.Equal(key.DeletesAt), "expected deletes at %s, got %s", expectedKey.DeletesAt, key.DeletesAt) + found = true + } + } + require.True(t, found, "expected key %+v not found", expectedKey) + } +} From fe0b4e43ac764bafdf7e099bf776a1d828bf25f6 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 1 Oct 2024 06:55:15 +0000 Subject: [PATCH 10/15] Add leak detection to dbkeycache tests --- coderd/cryptokeys/dbkeycache_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/coderd/cryptokeys/dbkeycache_test.go b/coderd/cryptokeys/dbkeycache_test.go index 6eb5de5182e4f..f59821675ebe0 100644 --- a/coderd/cryptokeys/dbkeycache_test.go +++ b/coderd/cryptokeys/dbkeycache_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" + "go.uber.org/goleak" "cdr.dev/slog/sloggers/slogtest" @@ -16,6 +17,10 @@ import ( "github.com/coder/quartz" ) +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} + func TestDBKeyCache(t *testing.T) { t.Parallel() From 430e1c2e1329d6e5c28bdb74126e98ba60ee19c7 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 1 Oct 2024 07:05:47 +0000 Subject: [PATCH 11/15] Add mutex lock to DBCache Close method --- coderd/cryptokeys/dbkeycache.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index 610f08e834028..372c0e8ea5a2b 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -183,5 +183,7 @@ func (d *DBCache) newTimer() *quartz.Timer { } func (d *DBCache) Close() { + d.keysMu.Lock() + defer d.keysMu.Unlock() d.timer.Stop() } From 4dfdd4fe92cc262d16b68149b4609086e29cba9b Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 1 Oct 2024 15:15:33 +0000 Subject: [PATCH 12/15] Add closed state to DBCache for safe shutdown Introduce `closed` state to the DBCache to ensure operations such as Verifying and Signing are safely terminated when the cache is closed. This update prevents resource access and ensures proper shutdown sequences, enhancing system reliability. --- coderd/cryptokeys/dbkeycache.go | 20 +++++++++++++++- coderd/cryptokeys/dbkeycache_test.go | 36 ++++++++++++++++++++++++++++ coderd/cryptokeys/keycache.go | 8 ++++--- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index 372c0e8ea5a2b..fd3f803b23680 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -31,6 +31,7 @@ type DBCache struct { timer *quartz.Timer // invalidateAt is the time at which the keys cache should be invalidated. invalidateAt time.Time + closed bool } type DBCacheOption func(*DBCache) @@ -64,8 +65,13 @@ func NewDBCache(logger slog.Logger, db database.Store, feature database.CryptoKe // it is neither deleted nor has breached its deletion date. It should only be // used for verifying or decrypting payloads. To sign/encrypt call Signing. func (d *DBCache) Verifying(ctx context.Context, sequence int32) (codersdk.CryptoKey, error) { - now := d.clock.Now() d.keysMu.RLock() + if d.closed { + d.keysMu.RUnlock() + return codersdk.CryptoKey{}, ErrClosed + } + + now := d.clock.Now() key, ok := d.keys[sequence] d.keysMu.RUnlock() if ok { @@ -97,6 +103,12 @@ func (d *DBCache) Verifying(ctx context.Context, sequence int32) (codersdk.Crypt // both past its start time and before its deletion time. func (d *DBCache) Signing(ctx context.Context) (codersdk.CryptoKey, error) { d.keysMu.RLock() + + if d.closed { + d.keysMu.RUnlock() + return codersdk.CryptoKey{}, ErrClosed + } + latest := d.latestKey d.keysMu.RUnlock() @@ -185,5 +197,11 @@ func (d *DBCache) newTimer() *quartz.Timer { func (d *DBCache) Close() { d.keysMu.Lock() defer d.keysMu.Unlock() + + if d.closed { + return + } + d.timer.Stop() + d.closed = true } diff --git a/coderd/cryptokeys/dbkeycache_test.go b/coderd/cryptokeys/dbkeycache_test.go index f59821675ebe0..8c92cf3a90aa6 100644 --- a/coderd/cryptokeys/dbkeycache_test.go +++ b/coderd/cryptokeys/dbkeycache_test.go @@ -104,4 +104,40 @@ func TestDBKeyCache(t *testing.T) { require.NoError(t, err) require.Equal(t, db2sdk.CryptoKey(expectedKey), got) }) + + t.Run("Closed", func(t *testing.T) { + t.Parallel() + + var ( + db, _ = dbtestutil.NewDB(t) + clock = quartz.NewMock(t) + ctx = testutil.Context(t, testutil.WaitShort) + logger = slogtest.Make(t, nil) + ) + + expectedKey := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceApps, + Sequence: 10, + StartsAt: clock.Now(), + }) + + k := cryptokeys.NewDBCache(logger, db, database.CryptoKeyFeatureWorkspaceApps, cryptokeys.WithDBCacheClock(clock)) + defer k.Close() + + got, err := k.Signing(ctx) + require.NoError(t, err) + require.Equal(t, db2sdk.CryptoKey(expectedKey), got) + + got, err = k.Verifying(ctx, expectedKey.Sequence) + require.NoError(t, err) + require.Equal(t, db2sdk.CryptoKey(expectedKey), got) + + k.Close() + + _, err = k.Signing(ctx) + require.ErrorIs(t, err, cryptokeys.ErrClosed) + + _, err = k.Verifying(ctx, expectedKey.Sequence) + require.ErrorIs(t, err, cryptokeys.ErrClosed) + }) } diff --git a/coderd/cryptokeys/keycache.go b/coderd/cryptokeys/keycache.go index f06cd425676ce..8c4ebfa13f64e 100644 --- a/coderd/cryptokeys/keycache.go +++ b/coderd/cryptokeys/keycache.go @@ -8,9 +8,11 @@ import ( "github.com/coder/coder/v2/codersdk" ) -var ErrKeyNotFound = xerrors.New("key not found") - -var ErrKeyInvalid = xerrors.New("key is invalid for use") +var ( + ErrKeyNotFound = xerrors.New("key not found") + ErrKeyInvalid = xerrors.New("key is invalid for use") + ErrClosed = xerrors.New("closed") +) // Keycache provides an abstraction for fetching signing keys. type Keycache interface { From 5078c2a5288c8e28b9ff5e58cfbf8723b5f26e3c Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 1 Oct 2024 15:27:46 +0000 Subject: [PATCH 13/15] Optimize timer reset in fetch method --- coderd/cryptokeys/dbkeycache.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index fd3f803b23680..f9f76321cac3d 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -157,8 +157,7 @@ func (d *DBCache) fetch(ctx context.Context) error { } now := d.clock.Now() - d.timer.Stop() - d.timer = d.newTimer() + _ = d.timer.Reset(time.Minute * 10) d.invalidateAt = now.Add(time.Minute * 10) cache := make(map[int32]database.CryptoKey) From 280d5215c5f5c97b31ff24b8878bad6a2bd9baaf Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 1 Oct 2024 15:28:42 +0000 Subject: [PATCH 14/15] Refactor DBCache to remove unused timer function --- coderd/cryptokeys/dbkeycache.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index f9f76321cac3d..bd5015b4886b6 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -189,10 +189,6 @@ func checkKey(key database.CryptoKey, now time.Time) (codersdk.CryptoKey, error) return db2sdk.CryptoKey(key), nil } -func (d *DBCache) newTimer() *quartz.Timer { - return d.clock.AfterFunc(time.Minute*10, d.clear) -} - func (d *DBCache) Close() { d.keysMu.Lock() defer d.keysMu.Unlock() From 05dec8a8cc3a3e39bca42f77306abfcdf2bcdc0b Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 1 Oct 2024 15:50:21 +0000 Subject: [PATCH 15/15] finale --- coderd/cryptokeys/dbkeycache.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/coderd/cryptokeys/dbkeycache.go b/coderd/cryptokeys/dbkeycache.go index bd5015b4886b6..4986f1669c4e5 100644 --- a/coderd/cryptokeys/dbkeycache.go +++ b/coderd/cryptokeys/dbkeycache.go @@ -81,6 +81,10 @@ func (d *DBCache) Verifying(ctx context.Context, sequence int32) (codersdk.Crypt d.keysMu.Lock() defer d.keysMu.Unlock() + if d.closed { + return codersdk.CryptoKey{}, ErrClosed + } + key, ok = d.keys[sequence] if ok { return checkKey(key, now) @@ -120,6 +124,10 @@ func (d *DBCache) Signing(ctx context.Context) (codersdk.CryptoKey, error) { d.keysMu.Lock() defer d.keysMu.Unlock() + if d.closed { + return codersdk.CryptoKey{}, ErrClosed + } + if d.latestKey.CanSign(now) { return db2sdk.CryptoKey(d.latestKey), nil }