From ec148d8f61da9674856d26cb460ae25f737e3ca2 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Tue, 30 Jun 2026 22:34:02 +0000 Subject: [PATCH 01/16] feat(coderd/cryptokeys): add and serve the nats_ca cluster CA Add a nats_ca crypto-key feature holding the NATS cluster mTLS CA: a PEM cert+key bundle minted by the key rotator (generateCASecret/parseCASecret), sized so an old CA stays a valid trust root for the maximum leaf lifetime after rotation. The CA is served through the generic cryptokeys signing cache rather than a bespoke cache: idSecret decodes the PEM bundle into a *NATSCA for nats_ca (hex bytes otherwise), so SigningKey returns the active CA and VerifyingKey returns a specific CA by sequence, reusing the cache's fetch/refresh/rotation logic. The feature is experiment-gated. The rotator only mints nats_ca when opted in via WithFeatures (default rotation excludes it), and coderd.New opts it in and builds a real signing cache only when ExperimentNATSPubsub is enabled; otherwise the cache is a NoopSigningKeycache so callers still get a valid response (treated as mTLS-off). nats_ca is kept off the workspace-proxy crypto key allowlist so the CA private key is never served over the API. Co-authored-by: Mux --- coderd/apidoc/docs.go | 6 +- coderd/apidoc/swagger.json | 6 +- coderd/coderd.go | 36 +++- coderd/cryptokeys/ca.go | 149 +++++++++++++ coderd/cryptokeys/ca_internal_test.go | 251 ++++++++++++++++++++++ coderd/cryptokeys/cache.go | 51 ++++- coderd/cryptokeys/rotate.go | 48 ++++- coderd/cryptokeys/rotate_internal_test.go | 116 +++++++++- coderd/database/dbgen/dbgen.go | 43 ++++ coderd/database/modelmethods.go | 7 + codersdk/deployment.go | 6 + docs/reference/api/schemas.md | 6 +- enterprise/coderd/workspaceproxy_test.go | 6 + site/src/api/typesGenerated.ts | 2 + 14 files changed, 706 insertions(+), 27 deletions(-) create mode 100644 coderd/cryptokeys/ca.go create mode 100644 coderd/cryptokeys/ca_internal_test.go diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index b423dca3c7d..06dfc35b0ca 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -18937,13 +18937,15 @@ const docTemplate = `{ "workspace_apps_api_key", "workspace_apps_token", "oidc_convert", - "tailnet_resume" + "tailnet_resume", + "nats_ca" ], "x-enum-varnames": [ "CryptoKeyFeatureWorkspaceAppsAPIKey", "CryptoKeyFeatureWorkspaceAppsToken", "CryptoKeyFeatureOIDCConvert", - "CryptoKeyFeatureTailnetResume" + "CryptoKeyFeatureTailnetResume", + "CryptoKeyFeatureNATSCA" ] }, "codersdk.CustomNotificationContent": { diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 811633d02b0..1c4ed8d404a 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -17137,13 +17137,15 @@ "workspace_apps_api_key", "workspace_apps_token", "oidc_convert", - "tailnet_resume" + "tailnet_resume", + "nats_ca" ], "x-enum-varnames": [ "CryptoKeyFeatureWorkspaceAppsAPIKey", "CryptoKeyFeatureWorkspaceAppsToken", "CryptoKeyFeatureOIDCConvert", - "CryptoKeyFeatureTailnetResume" + "CryptoKeyFeatureTailnetResume", + "CryptoKeyFeatureNATSCA" ] }, "codersdk.CustomNotificationContent": { diff --git a/coderd/coderd.go b/coderd/coderd.go index f2d5e6f5df7..9a4674ff1e2 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -305,7 +305,12 @@ type Options struct { AppSigningKeyCache cryptokeys.SigningKeycache AppEncryptionKeyCache cryptokeys.EncryptionKeycache OIDCConvertKeyCache cryptokeys.SigningKeycache - Clock quartz.Clock + // NATSCACache serves the NATS cluster mTLS CA via the generic signing key + // cache for the nats_ca feature. SigningKey returns the active CA + // (a *NATSCA); VerifyingKey returns a specific CA by sequence. The key + // rotator is the sole creator of nats_ca rows, so this cache is read-only. + NATSCACache cryptokeys.SigningKeycache + Clock quartz.Clock // WebPushDispatcher is a way to send notifications over Web Push. WebPushDispatcher webpush.Dispatcher @@ -607,10 +612,34 @@ func New(options *Options) *API { updatesProvider := NewUpdatesProvider(options.Logger.Named("workspace_updates"), options.Pubsub, options.Database, options.Authorizer) + // The NATS cluster CA is only minted and served when NATS pubsub is in use. + // It is experiment-gated, so it is opted into rotation and backed by a real + // signing cache only when the experiment is enabled; otherwise the rotator + // leaves it alone and the cache is a noop, which still answers requests (the + // pubsub treats a missing CA as "mTLS off"). This avoids minting CA private + // keys on deployments that never run NATS clustering. + rotatedFeatures := cryptokeys.DefaultRotatedFeatures() + if experiments.Enabled(codersdk.ExperimentNATSPubsub) { + rotatedFeatures = append(rotatedFeatures, database.CryptoKeyFeatureNATSCA) + } + // Start a background process that rotates keys. We intentionally start this after the caches // are created to force initial requests for a key to populate the caches. This helps catch // bugs that may only occur when a key isn't precached in tests and the latency cost is minimal. - cryptokeys.StartRotator(ctx, options.Logger, options.Database) + cryptokeys.StartRotator(ctx, options.Logger, options.Database, cryptokeys.WithFeatures(rotatedFeatures)) + + // The NATS CA cache is read-only and depends on the rotator having minted + // the nats_ca CA, so it must be constructed after StartRotator. + if options.NATSCACache == nil { + if experiments.Enabled(codersdk.ExperimentNATSPubsub) { + options.NATSCACache, err = cryptokeys.NewSigningCache(ctx, options.Logger.Named("nats_ca_cache"), &cryptokeys.DBFetcher{DB: options.Database}, codersdk.CryptoKeyFeatureNATSCA) + if err != nil { + options.Logger.Fatal(ctx, "failed to instantiate NATS CA cache", slog.Error(err)) + } + } else { + options.NATSCACache = cryptokeys.NoopSigningKeycache{} + } + } // Ensure all system role permissions are current. //nolint:gocritic // Startup reconciliation reads/writes system roles. There is @@ -2398,6 +2427,9 @@ func (api *API) Close() error { _ = api.OIDCConvertKeyCache.Close() _ = api.AppSigningKeyCache.Close() _ = api.AppEncryptionKeyCache.Close() + if api.NATSCACache != nil { + _ = api.NATSCACache.Close() + } _ = api.UpdatesProvider.Close() api.workspaceAgentConnWatcher.Close() diff --git a/coderd/cryptokeys/ca.go b/coderd/cryptokeys/ca.go new file mode 100644 index 00000000000..20eb100fdb2 --- /dev/null +++ b/coderd/cryptokeys/ca.go @@ -0,0 +1,149 @@ +package cryptokeys + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "time" + + "golang.org/x/xerrors" +) + +const ( + caCertPEMBlockType = "CERTIFICATE" + caKeyPEMBlockType = "EC PRIVATE KEY" + + // clockSkewTolerance backdates the CA certificate's NotBefore and extends + // its NotAfter so that replicas with mildly skewed clocks still accept it. + clockSkewTolerance = time.Hour +) + +// NATSCA is the decoded form of a single nats_ca crypto key row, produced by +// the generic crypto key cache (see idSecret). The CA signs the ephemeral leaf +// certificates that replicas use for NATS cluster mTLS. +// +// The active CA is served by a SigningKeycache.SigningKey call for the nats_ca +// feature; a specific historical CA (for verifying a peer leaf minted under an +// earlier CA during a rotation overlap) is served by VerifyingKey with that +// row's sequence. +type NATSCA struct { + // Sequence is the crypto_keys sequence of the row this CA came from. + Sequence int32 + // Cert is the CA certificate used to sign or verify leaf certificates. + Cert *x509.Certificate + // Key is the CA private key, used to sign leaves. + Key crypto.Signer +} + +// generateCASecret generates a new self-signed CA certificate and private key +// for signing NATS cluster leaf certificates, PEM-encoded into a single +// bundle for storage in the crypto_keys secret column. It is exported so test +// helpers (for example coderd/database/dbgen) can produce nats_ca rows in the +// exact format the rotator writes, rather than duplicating the bundle format. +// +// anchorTime is the key row's starts_at (which may be in the future for a +// rotated-in key). keyDuration is the rotator's key duration: the row stays the +// active signer for that long. The certificate must outlive that window plus +// the longest leaf it could sign (NATSCAKeyRetention) plus clock-skew slack, so +// leaves minted just before rotation still chain to a valid CA. +func generateCASecret(anchorTime time.Time, keyDuration time.Duration) (string, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return "", xerrors.Errorf("generate key: %w", err) + } + + // 128-bit random serial per CA/Browser Forum conventions. + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return "", xerrors.Errorf("generate serial: %w", err) + } + + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: "coder-nats-ca", + }, + NotBefore: anchorTime.Add(-clockSkewTolerance), + NotAfter: anchorTime.Add(keyDuration + NATSCAKeyRetention + clockSkewTolerance), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLenZero: true, + } + + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + if err != nil { + return "", xerrors.Errorf("create certificate: %w", err) + } + + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return "", xerrors.Errorf("marshal private key: %w", err) + } + + var secret []byte + secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: caCertPEMBlockType, Bytes: der})...) + secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: caKeyPEMBlockType, Bytes: keyDER})...) + return string(secret), nil +} + +// parseCASecret parses a PEM bundle produced by generateCASecret back into +// the CA certificate and private key. +func parseCASecret(secret string) (*x509.Certificate, crypto.Signer, error) { + var ( + cert *x509.Certificate + key *ecdsa.PrivateKey + ) + rest := []byte(secret) + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + switch block.Type { + case caCertPEMBlockType: + if cert != nil { + return nil, nil, xerrors.New("multiple certificates in CA secret") + } + var err error + cert, err = x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, nil, xerrors.Errorf("parse certificate: %w", err) + } + case caKeyPEMBlockType: + if key != nil { + return nil, nil, xerrors.New("multiple private keys in CA secret") + } + var err error + key, err = x509.ParseECPrivateKey(block.Bytes) + if err != nil { + return nil, nil, xerrors.Errorf("parse private key: %w", err) + } + default: + return nil, nil, xerrors.Errorf("unexpected PEM block type: %q", block.Type) + } + } + if cert == nil { + return nil, nil, xerrors.New("no certificate in CA secret") + } + if key == nil { + return nil, nil, xerrors.New("no private key in CA secret") + } + if !key.PublicKey.Equal(cert.PublicKey) { + return nil, nil, xerrors.New("private key does not match certificate") + } + // Reject a structurally valid bundle whose certificate cannot act as a + // signing CA. Without this, a corrupted secret could yield a non-CA cert + // that silently becomes the active signer; leaves signed under it would + // then fail x509 verification on every replica. + if !cert.IsCA || !cert.BasicConstraintsValid || cert.KeyUsage&x509.KeyUsageCertSign == 0 { + return nil, nil, xerrors.New("certificate is not a valid signing CA") + } + return cert, key, nil +} diff --git a/coderd/cryptokeys/ca_internal_test.go b/coderd/cryptokeys/ca_internal_test.go new file mode 100644 index 00000000000..3fd93d14171 --- /dev/null +++ b/coderd/cryptokeys/ca_internal_test.go @@ -0,0 +1,251 @@ +package cryptokeys + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/database/dbtime" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/testutil" + "github.com/coder/quartz" +) + +func TestCASecretRoundTrip(t *testing.T) { + t.Parallel() + + // The certificate's NotAfter must track the supplied keyDuration, not a + // hardcoded default, so a CA stays valid for as long as it can be the + // active signer plus the longest leaf it signs. + for _, keyDuration := range []time.Duration{DefaultKeyDuration, DefaultKeyDuration * 3, time.Hour} { + now := time.Now().UTC().Truncate(time.Second) + secret, err := generateCASecret(now, keyDuration) + require.NoError(t, err) + + cert, signer, err := parseCASecret(secret) + require.NoError(t, err) + + require.True(t, cert.IsCA) + require.True(t, cert.BasicConstraintsValid) + require.True(t, cert.MaxPathLenZero) + require.Equal(t, x509.KeyUsageCertSign, cert.KeyUsage) + require.Equal(t, now.Add(-clockSkewTolerance), cert.NotBefore) + require.Equal(t, now.Add(keyDuration+NATSCAKeyRetention+clockSkewTolerance), cert.NotAfter) + require.Equal(t, cert.PublicKey, signer.Public()) + + // The cert must outlive its active-signer window so leaves signed at + // the end of that window still chain to a valid CA. + require.True(t, cert.NotAfter.After(now.Add(keyDuration)), + "cert must remain valid past the end of its active-signer window") + + // The cert must be able to verify itself as a trust root. + pool := x509.NewCertPool() + pool.AddCert(cert) + _, err = cert.Verify(x509.VerifyOptions{Roots: pool}) + require.NoError(t, err) + } +} + +func TestParseCASecretErrors(t *testing.T) { + t.Parallel() + + now := time.Now() + secretA, err := generateCASecret(now, DefaultKeyDuration) + require.NoError(t, err) + secretB, err := generateCASecret(now, DefaultKeyDuration) + require.NoError(t, err) + + certA, keyA := splitCAPEM(t, secretA) + _, keyB := splitCAPEM(t, secretB) + + nonCACert, nonCAKey := generateNonCAPEM(t, now) + + cases := []struct { + name string + secret string + errText string + }{ + {"Empty", "", "no certificate"}, + {"NotPEM", "not pem at all", "no certificate"}, + {"CertOnly", string(certA), "no private key"}, + {"KeyCertMismatch", string(certA) + string(keyB), "does not match certificate"}, + {"MultipleCertificates", string(certA) + string(certA) + string(keyA), "multiple certificates"}, + {"MultiplePrivateKeys", string(certA) + string(keyA) + string(keyA), "multiple private keys"}, + {"UnexpectedBlockType", string(pemBlock("RSA PRIVATE KEY", []byte("x"))), "unexpected PEM block type"}, + {"BadCertificateBytes", string(pemBlock(caCertPEMBlockType, []byte("garbage"))), "parse certificate"}, + {"BadPrivateKeyBytes", string(certA) + string(pemBlock(caKeyPEMBlockType, []byte("garbage"))), "parse private key"}, + {"NotASigningCA", string(nonCACert) + string(nonCAKey), "not a valid signing CA"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, _, err := parseCASecret(tc.secret) + require.ErrorContains(t, err, tc.errText) + }) + } +} + +// splitCAPEM splits a CA secret bundle into its certificate and private key +// PEM blocks so tests can recombine them into malformed bundles. +func splitCAPEM(t *testing.T, secret string) (certPEM, keyPEM []byte) { + t.Helper() + rest := []byte(secret) + for { + block, r := pem.Decode(rest) + if block == nil { + break + } + rest = r + switch block.Type { + case caCertPEMBlockType: + certPEM = pem.EncodeToMemory(block) + case caKeyPEMBlockType: + keyPEM = pem.EncodeToMemory(block) + } + } + require.NotNil(t, certPEM) + require.NotNil(t, keyPEM) + return certPEM, keyPEM +} + +func pemBlock(blockType string, der []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: der}) +} + +// generateNonCAPEM produces a structurally valid cert+key bundle whose +// certificate is not a CA (no IsCA, no KeyUsageCertSign). The key matches the +// cert, so it passes every parseCASecret check except the signing-CA check. +func generateNonCAPEM(t *testing.T, now time.Time) (certPEM, keyPEM []byte) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "not-a-ca"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + require.NoError(t, err) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + return pemBlock(caCertPEMBlockType, der), pemBlock(caKeyPEMBlockType, keyDER) +} + +// TestNATSCASigningCache exercises the nats_ca feature through the generic +// signing key cache: the PEM secret decodes into a *NATSCA, SigningKey serves +// the active CA, VerifyingKey serves a specific CA by sequence, and a rotation +// is picked up on the next refresh. +func TestNATSCASigningCache(t *testing.T) { + t.Parallel() + + t.Run("ActiveAndVerifyingByID", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + now := time.Now().UTC() + + current := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: 1, + StartsAt: now.Add(-time.Hour), + }) + + cache, err := NewSigningCache(ctx, testutil.Logger(t), &DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA) + require.NoError(t, err) + defer cache.Close() + + id, key, err := cache.SigningKey(ctx) + require.NoError(t, err) + + ca, ok := key.(*NATSCA) + require.True(t, ok, "signing key should decode to *NATSCA, got %T", key) + require.Equal(t, current.Sequence, ca.Sequence) + require.NotNil(t, ca.Cert) + require.NotNil(t, ca.Key) + + currentCert, _, err := parseCASecret(current.Secret.String) + require.NoError(t, err) + require.Equal(t, currentCert.Raw, ca.Cert.Raw) + + // VerifyingKey looks the CA up by the sequence embedded in id, which is + // how a peer leaf minted under this CA is verified. + verifying, err := cache.VerifyingKey(ctx, id) + require.NoError(t, err) + vca, ok := verifying.(*NATSCA) + require.True(t, ok, "verifying key should decode to *NATSCA, got %T", verifying) + require.Equal(t, currentCert.Raw, vca.Cert.Raw) + }) + + t.Run("RefreshesOnRotation", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + clock := quartz.NewMock(t) + now := dbtime.Now() + clock.Set(now) + + first := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: 1, + StartsAt: now.Add(-time.Hour), + }) + + cache, err := NewSigningCache(ctx, testutil.Logger(t), &DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA, WithCacheClock(clock)) + require.NoError(t, err) + defer cache.Close() + + _, key, err := cache.SigningKey(ctx) + require.NoError(t, err) + require.Equal(t, first.Sequence, key.(*NATSCA).Sequence) + + // Simulate a rotation by inserting a higher-sequence active CA. The old + // CA stays valid for verification by its sequence. + second := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: 2, + StartsAt: now.Add(-time.Minute), + }) + + // Fire the background refresher; the active CA advances to the new row. + clock.Advance(refreshInterval).MustWait(ctx) + + _, key, err = cache.SigningKey(ctx) + require.NoError(t, err) + require.Equal(t, second.Sequence, key.(*NATSCA).Sequence) + + oldVerifying, err := cache.VerifyingKey(ctx, "1") + require.NoError(t, err) + require.Equal(t, first.Sequence, oldVerifying.(*NATSCA).Sequence) + }) +} + +func TestNoopSigningKeycache(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + var cache SigningKeycache = NoopSigningKeycache{} + + _, _, err := cache.SigningKey(ctx) + require.ErrorIs(t, err, ErrKeyNotFound) + + _, err = cache.VerifyingKey(ctx, "1") + require.ErrorIs(t, err, ErrKeyNotFound) + + require.NoError(t, cache.Close()) +} diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index de40324df1a..75269d98395 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -55,6 +55,26 @@ type SigningKeycache interface { io.Closer } +// NoopSigningKeycache is a SigningKeycache that holds no keys: SigningKey and +// VerifyingKey always report ErrKeyNotFound. It lets a subsystem that only +// needs real keys once an optional feature is enabled (for example NATS +// cluster mTLS, which only signs leaves under enterprise HA) be constructed +// without a database dependency, then be swapped for a real cache when the +// feature turns on. +type NoopSigningKeycache struct{} + +var _ SigningKeycache = NoopSigningKeycache{} + +func (NoopSigningKeycache) SigningKey(context.Context) (string, interface{}, error) { + return "", nil, ErrKeyNotFound +} + +func (NoopSigningKeycache) VerifyingKey(context.Context, string) (interface{}, error) { + return nil, ErrKeyNotFound +} + +func (NoopSigningKeycache) Close() error { return nil } + const ( // latestSequence is a special sequence number that represents the latest key. latestSequence = -1 @@ -213,23 +233,42 @@ func isEncryptionKeyFeature(feature codersdk.CryptoKeyFeature) bool { func isSigningKeyFeature(feature codersdk.CryptoKeyFeature) bool { switch feature { - case codersdk.CryptoKeyFeatureTailnetResume, codersdk.CryptoKeyFeatureOIDCConvert, codersdk.CryptoKeyFeatureWorkspaceAppsToken: + case codersdk.CryptoKeyFeatureTailnetResume, codersdk.CryptoKeyFeatureOIDCConvert, codersdk.CryptoKeyFeatureWorkspaceAppsToken, codersdk.CryptoKeyFeatureNATSCA: return true default: return false } } -func idSecret(k codersdk.CryptoKey) (string, []byte, error) { +// idSecret materializes a stored crypto key into the in-memory key object the +// feature uses, returning it as an interface{} alongside the key's id (its +// sequence as a decimal string). Most features hex-decode the secret into raw +// bytes, but nats_ca stores a PEM cert+key bundle and decodes into a *NATSCA. +// +// TODO: this hard-coded switch on feature is the simplest way to support a +// second secret encoding, but it couples this generic cache to nats_ca +// specifics. Explore abstracting the decode step (for example a per-feature +// decoder injected at construction) so new key types can be added without +// editing this function. +func idSecret(k codersdk.CryptoKey) (string, interface{}, error) { + id := strconv.FormatInt(int64(k.Sequence), 10) + + if k.Feature == codersdk.CryptoKeyFeatureNATSCA { + cert, signer, err := parseCASecret(k.Secret) + if err != nil { + return "", nil, xerrors.Errorf("decode nats_ca key: %w", err) + } + return id, &NATSCA{Sequence: k.Sequence, Cert: cert, Key: signer}, nil + } + key, err := hex.DecodeString(k.Secret) if err != nil { return "", nil, xerrors.Errorf("decode key: %w", err) } - - return strconv.FormatInt(int64(k.Sequence), 10), key, nil + return id, key, nil } -func (c *cache) cryptoKey(ctx context.Context, sequence int32) (string, []byte, error) { +func (c *cache) cryptoKey(ctx context.Context, sequence int32) (string, interface{}, error) { c.logger.Debug(ctx, "request for key", slog.F("sequence", sequence)) c.mu.Lock() defer c.mu.Unlock() @@ -284,7 +323,7 @@ func (c *cache) key(sequence int32) (codersdk.CryptoKey, bool) { return key, ok } -func checkKey(key codersdk.CryptoKey, sequence int32, now time.Time) (string, []byte, error) { +func checkKey(key codersdk.CryptoKey, sequence int32, now time.Time) (string, interface{}, error) { if sequence == latestSequence { if !key.CanSign(now) { return "", nil, ErrKeyInvalid diff --git a/coderd/cryptokeys/rotate.go b/coderd/cryptokeys/rotate.go index 4f30f842597..db49cfce5c9 100644 --- a/coderd/cryptokeys/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -21,6 +21,14 @@ const ( WorkspaceAppsTokenDuration = time.Minute OIDCConvertTokenDuration = time.Minute * 5 TailnetResumeTokenDuration = time.Hour * 24 + // NATSCAKeyRetention is how long a rotated-out NATS cluster CA is kept as a + // valid trust root after it stops being the active signer. A replica may + // still present a leaf signed by the old CA until that leaf expires, so the + // old CA must remain verifiable for at least the leaf lifetime. This is a CA + // retention budget, not a leaf lifetime: it must be >= the leaf validity + // used when minting (coderd/x/nats leafCertValidity), which is enforced by a + // compile-time assertion there. + NATSCAKeyRetention = time.Hour * 24 * 30 // defaultRotationInterval is the default interval at which keys are checked for rotation. defaultRotationInterval = time.Minute * 10 @@ -70,6 +78,15 @@ func WithKeyDuration(keyDuration time.Duration) RotatorOption { } } +// WithFeatures sets the crypto key features the rotator manages, replacing the +// default set. Use this to opt experiment- or deployment-gated features (such +// as the NATS cluster CA) into rotation only when their owner is active. +func WithFeatures(features []database.CryptoKeyFeature) RotatorOption { + return func(r *rotator) { + r.features = slices.Clone(features) + } +} + // StartRotator starts a background process that rotates keys in the database. // It ensures there's at least one valid key per feature prior to returning. // Canceling the provided context will stop the background process. @@ -126,10 +143,7 @@ func (k *rotator) rotateKeys(ctx context.Context) error { return xerrors.Errorf("get keys: %w", err) } - featureKeys, err := keysByFeature(cryptokeys, k.features) - if err != nil { - return xerrors.Errorf("keys by feature: %w", err) - } + featureKeys := keysByFeature(cryptokeys, k.features) now := dbtime.Time(k.clock.Now().UTC()) for feature, keys := range featureKeys { @@ -189,7 +203,7 @@ func (k *rotator) rotateKeys(ctx context.Context) error { } func (k *rotator) insertNewKey(ctx context.Context, tx database.Store, feature database.CryptoKeyFeature, startsAt time.Time) (database.CryptoKey, error) { - secret, err := generateNewSecret(feature) + secret, err := generateNewSecret(feature, startsAt, k.keyDuration) if err != nil { return database.CryptoKey{}, xerrors.Errorf("generate new secret: %w", err) } @@ -246,7 +260,11 @@ func (k *rotator) rotateKey(ctx context.Context, tx database.Store, key database return []database.CryptoKey{updatedKey, newKey}, nil } -func generateNewSecret(feature database.CryptoKeyFeature) (string, error) { +// generateNewSecret generates the secret for a new key of the given feature. +// keyDuration is the rotator's key duration; it is only used by features whose +// secret encodes its own validity window (currently only the NATS CA, whose +// certificate must outlive the key row's active-signer period). +func generateNewSecret(feature database.CryptoKeyFeature, startsAt time.Time, keyDuration time.Duration) (string, error) { switch feature { case database.CryptoKeyFeatureWorkspaceAppsAPIKey: return generateKey(32) @@ -256,6 +274,8 @@ func generateNewSecret(feature database.CryptoKeyFeature) (string, error) { return generateKey(64) case database.CryptoKeyFeatureTailnetResume: return generateKey(64) + case database.CryptoKeyFeatureNATSCA: + return generateCASecret(startsAt, keyDuration) } return "", xerrors.Errorf("unknown feature: %s", feature) } @@ -279,6 +299,8 @@ func tokenDuration(feature database.CryptoKeyFeature) time.Duration { return OIDCConvertTokenDuration case database.CryptoKeyFeatureTailnetResume: return TailnetResumeTokenDuration + case database.CryptoKeyFeatureNATSCA: + return NATSCAKeyRetention default: return 0 } @@ -297,19 +319,25 @@ func shouldRotateKey(key database.CryptoKey, keyDuration time.Duration, now time return !now.Add(time.Hour).UTC().Before(expirationTime) } -func keysByFeature(keys []database.CryptoKey, features []database.CryptoKeyFeature) (map[database.CryptoKeyFeature][]database.CryptoKey, error) { +// keysByFeature groups keys by feature, restricted to the managed feature set. +// GetCryptoKeys returns rows for every feature, but the rotator only manages a +// subset (features can be gated, e.g. nats_ca behind an experiment). Keys for +// features outside the managed set belong to features this rotator is not +// responsible for and are skipped, so their presence (for example nats_ca rows +// left over from a prior experiment-on run) does not abort rotation of the +// managed features. +func keysByFeature(keys []database.CryptoKey, features []database.CryptoKeyFeature) map[database.CryptoKeyFeature][]database.CryptoKey { m := map[database.CryptoKeyFeature][]database.CryptoKey{} for _, feature := range features { m[feature] = []database.CryptoKey{} } for _, key := range keys { if _, ok := m[key.Feature]; !ok { - return nil, xerrors.Errorf("unknown feature: %s", key.Feature) + continue } - m[key.Feature] = append(m[key.Feature], key) } - return m, nil + return m } // minStartsAt ensures the minimum starts_at time we use for a new diff --git a/coderd/cryptokeys/rotate_internal_test.go b/coderd/cryptokeys/rotate_internal_test.go index 4d43f24e187..0bcf7b7a9d7 100644 --- a/coderd/cryptokeys/rotate_internal_test.go +++ b/coderd/cryptokeys/rotate_internal_test.go @@ -104,6 +104,111 @@ func Test_rotateKeys(t *testing.T) { require.Equal(t, newKey, keys[0]) }) + t.Run("RotatesNATSCA", func(t *testing.T) { + t.Parallel() + + var ( + db, _ = dbtestutil.NewDB(t) + clock = quartz.NewMock(t) + keyDuration = time.Hour * 24 * 7 + logger = testutil.Logger(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + kr := &rotator{ + db: db, + keyDuration: keyDuration, + clock: clock, + logger: logger, + features: []database.CryptoKeyFeature{ + database.CryptoKeyFeatureNATSCA, + }, + } + + now := dbnow(clock) + + oldKey := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + StartsAt: now, + Sequence: 4, + }) + + // Advance the window to just inside rotation time. + _ = clock.Advance(keyDuration - time.Minute*59) + err := kr.rotateKeys(ctx) + require.NoError(t, err) + + // The old CA must remain a valid trust root for the maximum leaf + // lifetime after rotation. + expectedDeletesAt := oldKey.ExpiresAt(keyDuration).Add(NATSCAKeyRetention + time.Hour) + oldKey, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ + Feature: oldKey.Feature, + Sequence: oldKey.Sequence, + }) + require.NoError(t, err) + require.Equal(t, expectedDeletesAt, oldKey.DeletesAt.Time.UTC()) + + newKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: oldKey.Sequence + 1, + }) + require.NoError(t, err) + requireKey(t, newKey, database.CryptoKeyFeatureNATSCA, oldKey.ExpiresAt(keyDuration), nullTime, oldKey.Sequence+1) + }) + + t.Run("IgnoresUnmanagedFeatureKeys", func(t *testing.T) { + t.Parallel() + + // Regression: a rotator managing a subset of features (e.g. after the + // nats_ca experiment is toggled off) must still rotate its managed + // features even when the DB holds keys for features it does not manage, + // such as nats_ca rows left over from a prior experiment-on run. + // Previously such rows aborted every rotation. + var ( + db, _ = dbtestutil.NewDB(t) + clock = quartz.NewMock(t) + keyDuration = time.Hour * 24 * 7 + logger = testutil.Logger(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + kr := &rotator{ + db: db, + keyDuration: keyDuration, + clock: clock, + logger: logger, + // Manages only tailnet resume; nats_ca is intentionally not managed, + // mirroring the experiment being off. + features: []database.CryptoKeyFeature{ + database.CryptoKeyFeatureTailnetResume, + }, + } + + now := dbnow(clock) + + // A leftover nats_ca row the rotator does not manage. + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + StartsAt: now, + Sequence: 1, + }) + + // No managed key exists yet, so rotation must insert one for the managed + // feature and must not error on the unmanaged nats_ca row. + err := kr.rotateKeys(ctx) + require.NoError(t, err) + + newKey, err := db.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureTailnetResume) + require.NoError(t, err) + require.Equal(t, database.CryptoKeyFeatureTailnetResume, newKey.Feature) + + // The unmanaged nats_ca row is untouched (no rotation, no delete). + natsKeys, err := db.GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureNATSCA) + require.NoError(t, err) + require.Len(t, natsKeys, 1) + require.False(t, natsKeys[0].DeletesAt.Valid) + }) + t.Run("DoesNotRotateValidKeys", func(t *testing.T) { t.Parallel() @@ -409,8 +514,7 @@ func Test_rotateKeys(t *testing.T) { require.NoError(t, err) require.Len(t, keys, 5) - kbf, err := keysByFeature(keys, defaultRotatedFeatures) - require.NoError(t, err) + kbf := keysByFeature(keys, defaultRotatedFeatures) // No actions on OIDC convert. require.Len(t, kbf[database.CryptoKeyFeatureOIDCConvert], 1) @@ -586,6 +690,14 @@ func requireKey(t *testing.T, key database.CryptoKey, feature database.CryptoKey require.Equal(t, deletesAt.Time.UTC(), key.DeletesAt.Time.UTC()) require.Equal(t, sequence, key.Sequence) + // The NATS CA secret is a PEM bundle rather than hex-encoded bytes. + if key.Feature == database.CryptoKeyFeatureNATSCA { + cert, _, err := parseCASecret(key.Secret.String) + require.NoError(t, err) + require.True(t, cert.IsCA) + return + } + secret, err := hex.DecodeString(key.Secret.String) require.NoError(t, err) diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index 97d3094ea06..db7d8e6f0cb 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -2,13 +2,19 @@ package dbgen import ( "context" + "crypto/ecdsa" + "crypto/elliptic" "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" "database/sql" "encoding/hex" "encoding/json" + "encoding/pem" "errors" "fmt" "maps" + "math/big" "net" "strings" "testing" @@ -2219,10 +2225,47 @@ func newCryptoKeySecret(feature database.CryptoKeyFeature) (string, error) { return generateCryptoKey(64) case database.CryptoKeyFeatureTailnetResume: return generateCryptoKey(64) + case database.CryptoKeyFeatureNATSCA: + return generateCACryptoKeySecret() } return "", xerrors.Errorf("unknown feature: %s", feature) } +// generateCACryptoKeySecret generates a self-signed CA certificate and private +// key as a PEM bundle, matching the secret format that coderd/cryptokeys +// produces for the nats_ca feature. It intentionally duplicates +// cryptokeys.generateCASecret rather than calling it: coderd/cryptokeys's +// internal tests import dbgen, so importing cryptokeys here would create a +// test-build import cycle. +func generateCACryptoKeySecret() (string, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return "", xerrors.Errorf("generate key: %w", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "dbgen-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLenZero: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + if err != nil { + return "", xerrors.Errorf("create certificate: %w", err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return "", xerrors.Errorf("marshal private key: %w", err) + } + var secret []byte + secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...) + secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})...) + return string(secret), nil +} + func generateCryptoKey(length int) (string, error) { b := make([]byte, length) _, err := rand.Read(b) diff --git a/coderd/database/modelmethods.go b/coderd/database/modelmethods.go index cec3e1d291b..ae3c3075131 100644 --- a/coderd/database/modelmethods.go +++ b/coderd/database/modelmethods.go @@ -869,6 +869,13 @@ func (k CryptoKey) ExpiresAt(keyDuration time.Duration) time.Time { return k.StartsAt.Add(keyDuration).UTC() } +// DecodeString hex-decodes the key's secret. It is only valid for features +// whose secret is hex-encoded bytes; it must NOT be used for nats_ca, whose +// secret is a PEM certificate+key bundle (see coderd/cryptokeys parseCASecret). +// +// TODO: this method currently has no callers (the keycache hex-decodes via its +// own helper). Investigate removing it, or making it feature-aware, so the +// secret column's dual format (hex bytes vs PEM bundle) cannot be misdecoded. func (k CryptoKey) DecodeString() ([]byte, error) { return hex.DecodeString(k.Secret.String) } diff --git a/codersdk/deployment.go b/codersdk/deployment.go index 2edd295dbc6..608878b8c2d 100644 --- a/codersdk/deployment.go +++ b/codersdk/deployment.go @@ -5512,6 +5512,12 @@ const ( CryptoKeyFeatureWorkspaceAppsToken CryptoKeyFeature = "workspace_apps_token" CryptoKeyFeatureOIDCConvert CryptoKeyFeature = "oidc_convert" CryptoKeyFeatureTailnetResume CryptoKeyFeature = "tailnet_resume" + // CryptoKeyFeatureNATSCA is the CA that signs NATS cluster mTLS leaf + // certificates. Its secret is a PEM cert+key bundle (not a hex secret like + // the other features) and contains a private key, so it must never be + // served over the API. It is deliberately excluded from + // whitelistedCryptoKeyFeatures in enterprise/coderd/workspaceproxy.go. + CryptoKeyFeatureNATSCA CryptoKeyFeature = "nats_ca" ) type CryptoKey struct { diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 4d9124782bd..3aa7488e41c 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -5202,9 +5202,9 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o #### Enumerated Values -| Value(s) | -|------------------------------------------------------------------------------------| -| `oidc_convert`, `tailnet_resume`, `workspace_apps_api_key`, `workspace_apps_token` | +| Value(s) | +|-----------------------------------------------------------------------------------------------| +| `nats_ca`, `oidc_convert`, `tailnet_resume`, `workspace_apps_api_key`, `workspace_apps_token` | ## codersdk.CustomNotificationContent diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index 41956485521..50bb7cf68c5 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -1092,6 +1092,12 @@ func TestGetCryptoKeys(t *testing.T) { require.Error(t, err) require.ErrorAs(t, err, &sdkErr) require.Equal(t, http.StatusBadRequest, sdkErr.StatusCode()) + // The NATS cluster CA bundle contains a private key and must never be + // served to workspace proxies. + _, err = proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeature(database.CryptoKeyFeatureNATSCA)) + require.Error(t, err) + require.ErrorAs(t, err, &sdkErr) + require.Equal(t, http.StatusBadRequest, sdkErr.StatusCode()) _, err = proxy.SDKClient.CryptoKeys(ctx, "invalid") require.Error(t, err) require.ErrorAs(t, err, &sdkErr) diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 84a8a8c3ae4..3af72190b41 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -4044,12 +4044,14 @@ export interface CryptoKey { // From codersdk/deployment.go export type CryptoKeyFeature = + | "nats_ca" | "oidc_convert" | "tailnet_resume" | "workspace_apps_api_key" | "workspace_apps_token"; export const CryptoKeyFeatures: CryptoKeyFeature[] = [ + "nats_ca", "oidc_convert", "tailnet_resume", "workspace_apps_api_key", From f24b4802647f3a840fb73d25d350b3e328ecf961 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Tue, 30 Jun 2026 22:35:26 +0000 Subject: [PATCH 02/16] feat(coderd/x/nats): mutual TLS for the embedded NATS cluster Add cluster-route mTLS driven by tls.Config callbacks that read the nats_ca CA cache on each handshake, so a CA rotation is tracked with no restart. Each replica mints an ephemeral leaf from the active CA, stamping the signing CA's crypto_keys sequence into the leaf so a verifier loads exactly that CA (rotation overlap works in both directions). InsecureSkipVerify is set so verification runs in VerifyConnection against the live CA instead of a static, rotation-blind RootCAs pool; the connection is still mutually verified (RequireAnyClientCert). The leaf carries the replica's relay IP as an IP SAN. On the accepting side, where the dialing peer's source address is available, verification also requires the leaf SAN to match the connection source IP; the dialing side has no equivalent hook in Go and verifies the chain only. mTLS is optional: the pubsub boots with a noop CA cache (no leaf can be minted, so no route forms) and zero CA dependency. Enterprise HA swaps the real nats_ca cache plus the relay IP in via SetClusterCA when the high-availability feature is enabled, and reverts to noop when disabled. mTLS complements the existing shared route token (defense in depth). Co-authored-by: Mux --- cli/server.go | 7 + coderd/x/nats/cluster.go | 14 + coderd/x/nats/pubsub.go | 34 ++ coderd/x/nats/tls.go | 395 ++++++++++++++++++ .../x/nats/tls_integration_internal_test.go | 86 ++++ coderd/x/nats/tls_internal_test.go | 355 ++++++++++++++++ enterprise/coderd/coderd.go | 30 ++ 7 files changed, 921 insertions(+) create mode 100644 coderd/x/nats/tls.go create mode 100644 coderd/x/nats/tls_integration_internal_test.go create mode 100644 coderd/x/nats/tls_internal_test.go diff --git a/cli/server.go b/cli/server.go index 21f26c1dc03..ce423e1fa38 100644 --- a/cli/server.go +++ b/cli/server.go @@ -66,6 +66,7 @@ import ( "github.com/coder/coder/v2/coderd/aibridged" "github.com/coder/coder/v2/coderd/authlink" "github.com/coder/coder/v2/coderd/autobuild" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/awsiamrds" "github.com/coder/coder/v2/coderd/database/dbauthz" @@ -846,6 +847,12 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. token := fmt.Sprintf("%x", sha256.Sum256([]byte(dbURL))) natsps, err := nats.New(ctx, logger.Named("nats_pubsub"), nats.Options{ ClusterAuthToken: token, + // Install the cluster TLS callbacks with a noop CA cache so a + // single node (or pre-license deployment) boots without a CA + // dependency and forms no routes. Enterprise HA swaps in the + // real nats_ca cache plus the relay IP via Pubsub.SetClusterCA + // once clustering is licensed. + ClusterCA: cryptokeys.NoopSigningKeycache{}, }) if err != nil { return xerrors.Errorf("create nats pubsub: %w", err) diff --git a/coderd/x/nats/cluster.go b/coderd/x/nats/cluster.go index d70de4dc3a2..3779aa2c9b4 100644 --- a/coderd/x/nats/cluster.go +++ b/coderd/x/nats/cluster.go @@ -49,6 +49,20 @@ func (p *Pubsub) SetPeerFetcher(fetcher PeerFetcher) { p.RefreshPeers() } +// SetClusterCA swaps the cluster mTLS CA cache and this replica's leaf IP SAN, +// then triggers a peer refresh so any route blocked by the previous (for +// example noop) cache is retried. It is a no-op unless the pubsub was started +// with cluster TLS enabled (Options.ClusterCA set, which installs the TLS +// callbacks). Passing a noop cache reverts to no mTLS: new route handshakes +// can no longer mint a leaf and will not form. +func (p *Pubsub) SetClusterCA(ca ClusterCAKeycache, ip net.IP) { + if p.clusterTLS == nil { + return + } + p.clusterTLS.setClusterCA(ca, ip) + p.RefreshPeers() +} + // RefreshPeers signals the peer refresh worker to fetch and apply the latest // peer route addresses. Multiple pending refreshes are coalesced. func (p *Pubsub) RefreshPeers() { diff --git a/coderd/x/nats/pubsub.go b/coderd/x/nats/pubsub.go index 57ced2aeb39..1638be54271 100644 --- a/coderd/x/nats/pubsub.go +++ b/coderd/x/nats/pubsub.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "hash/fnv" + "net" "net/url" "sync" "time" @@ -16,6 +17,7 @@ import ( "cdr.dev/slog/v3" "github.com/coder/coder/v2/coderd/database/pubsub" + "github.com/coder/quartz" ) // DefaultServerMaxPendingBytes caps how many bytes the embedded NATS server will @@ -121,6 +123,21 @@ type Options struct { // clustered embedded NATS servers. Empty disables route auth. ClusterAuthToken string + // ClusterCA enables mutual TLS on the cluster route listener. When set + // (and cluster mode is enabled), each replica mints an ephemeral leaf + // certificate from the active nats_ca CA and verifies peers against the + // CA fetched from this cache on each handshake. Nil keeps routes + // plaintext (token auth only). cryptokeys.SigningKeycache satisfies this. + ClusterCA ClusterCAKeycache + + // ClusterTLSIP is this replica's relay IP, embedded as an IP SAN in the + // leaf certificate and matched against the dialed host when verifying a + // peer. Required when ClusterCA is set. + ClusterTLSIP net.IP + + // clusterTLSClock overrides the cluster TLS clock, for tests. + clusterTLSClock quartz.Clock + // PeerFetcher provides the current set of peer route addresses. // RefreshPeers uses it to update the configured cluster routes. PeerFetcher PeerFetcher @@ -182,6 +199,9 @@ type Pubsub struct { clustered bool serverOpts *natsserver.Options currentRoutes []*url.URL + // clusterTLS is non-nil when the cluster route listener runs mutual TLS. + // Its valid-peer-IP set is kept in sync with currentRoutes. + clusterTLS *clusterTLS peerFetcher PeerFetcher peerRefresh chan struct{} @@ -304,6 +324,19 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub, return nil, err } + // When ClusterCA is set, install the cluster TLS callbacks at boot so the + // route listener can negotiate mTLS. The callbacks read the CA cache on + // each handshake, so the default noop cache keeps routes inert (no leaf can + // be minted) until SetClusterCA swaps in a real cache. A real cache also + // requires ClusterTLSIP; leaf minting enforces that. ClusterCA == nil keeps + // routes plaintext (token auth only). + var ct *clusterTLS + if !opts.disableCluster && opts.ClusterCA != nil { + ct = newClusterTLS(ctx, logger, opts.clusterTLSClock, opts.ClusterCA, opts.ClusterTLSIP) + sopts.Cluster.TLSConfig = ct.tlsConfig() + sopts.Cluster.TLSTimeout = clusterTLSTimeout.Seconds() + } + ns, err := startEmbeddedServer(sopts) if err != nil { return nil, err @@ -333,6 +366,7 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub, p.clustered = !opts.disableCluster p.serverOpts = sopts.Clone() p.currentRoutes = cloneRouteURLs(sopts.Routes) + p.clusterTLS = ct handlers := p.buildConnHandlers() publishPool, err := newConnPool(ns, opts, handlers, opts.PublishConns, "coder-pubsub-pub") diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go new file mode 100644 index 00000000000..b9599d40a3a --- /dev/null +++ b/coderd/x/nats/tls.go @@ -0,0 +1,395 @@ +package nats + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "slices" + "strconv" + "sync" + "time" + + "golang.org/x/xerrors" + + "cdr.dev/slog/v3" + "github.com/coder/coder/v2/coderd/cryptokeys" + "github.com/coder/quartz" +) + +const ( + // leafCertValidity is the lifetime of an ephemeral cluster leaf + // certificate. Leaves are re-minted before expiry and whenever the active + // CA rotates, so this can be well under the CA retention window + // (cryptokeys.NATSCAKeyRetention). + leafCertValidity = 24 * time.Hour + // leafRenewBefore re-mints the leaf this long before it expires so an + // in-flight handshake never races expiry. + leafRenewBefore = time.Hour + // clusterTLSTimeout is the route TLS handshake timeout. NATS defaults to a + // tight 2s, which is flaky under load and in CI. + clusterTLSTimeout = 10 * time.Second + // leafSerialBits is the entropy of a leaf certificate serial number. + leafSerialBits = 128 + // clockSkewToleranceTLS backdates a leaf's NotBefore so a peer with a + // mildly skewed clock still accepts a freshly minted leaf. + clockSkewToleranceTLS = time.Hour +) + +// Leaves must never outlive the CA retention window, or a rotated-out CA could +// be deleted while an in-flight leaf still chains to it, breaking verification +// during a rotation overlap. Converting a negative duration to uint fails to +// compile, mechanically enforcing leafCertValidity <= NATSCAKeyRetention. +const _ = uint(cryptokeys.NATSCAKeyRetention - leafCertValidity) + +// ClusterCAKeycache is the read-only view of the nats_ca signing key cache that +// the cluster TLS layer needs. cryptokeys.SigningKeycache satisfies it, so the +// nats_ca cache is passed straight through with no adapter. +// +// SigningKey returns the active CA used to mint this replica's leaf; +// VerifyingKey returns a specific CA by its crypto_keys sequence, used to +// verify a peer leaf that was minted under that (possibly older) CA during a +// rotation overlap. Both return a *cryptokeys.NATSCA. +type ClusterCAKeycache interface { + SigningKey(ctx context.Context) (id string, key interface{}, err error) + VerifyingKey(ctx context.Context, id string) (key interface{}, err error) +} + +// clusterTLS builds the cluster route *tls.Config. Certificate selection and +// peer verification are tls.Config callbacks that consult the CA cache on each +// use, so a CA rotation is tracked without restarting or reloading the server. +type clusterTLS struct { + ctx context.Context + logger slog.Logger + clock quartz.Clock + + mu sync.Mutex + // ca and ip are swapped together by setClusterCA: under the default noop + // cache no leaf can be minted (so no route forms), and the real cache plus + // this replica's relay IP are installed once cluster mTLS is enabled. + ca ClusterCAKeycache + ip net.IP + // leaf is the cached leaf certificate. leafSeq is the active CA sequence it + // was minted under; a change means the CA rotated and the leaf is stale. + leaf *tls.Certificate + leafSeq string + // verifyPools caches the root pool used to verify a peer leaf, keyed by the + // CA sequence stamped in the leaf. A CA cert is immutable for a given + // sequence, so the pool is built once and reused across handshakes. Expired + // entries are pruned on insert to bound the map across rotations. + verifyPools map[string]cachedVerifyPool +} + +// cachedVerifyPool is a verify root pool plus the NotAfter of the CA cert it +// holds; the entry is dropped once the clock passes notAfter. +type cachedVerifyPool struct { + pool *x509.CertPool + notAfter time.Time +} + +func newClusterTLS(ctx context.Context, logger slog.Logger, clock quartz.Clock, ca ClusterCAKeycache, ip net.IP) *clusterTLS { + if clock == nil { + clock = quartz.NewReal() + } + return &clusterTLS{ + ctx: ctx, + logger: logger.Named("cluster_tls"), + clock: clock, + ca: ca, + ip: ip, + } +} + +// setClusterCA swaps the CA cache and this replica's leaf IP SAN. Because the +// tls.Config callbacks read these on each handshake, the swap takes effect +// without a server restart or route reload: installing the real cache lets +// routes negotiate mTLS, and reverting to a noop cache makes leaf minting fail +// so no new route can form. A swap clears the cached leaf so the next handshake +// re-mints under the new CA/IP. +func (t *clusterTLS) setClusterCA(ca ClusterCAKeycache, ip net.IP) { + t.mu.Lock() + defer t.mu.Unlock() + t.ca = ca + t.ip = ip + t.leaf = nil + t.leafSeq = "" + // Verify pools follow the CA source: drop them so stale roots are not + // reused after a swap to a noop or different CA. + t.verifyPools = nil +} + +// caCache returns the current CA cache under lock so callers do not hold the +// lock across cache I/O. +func (t *clusterTLS) caCache() ClusterCAKeycache { + t.mu.Lock() + defer t.mu.Unlock() + return t.ca +} + +// tlsConfig returns the *tls.Config for the embedded server's cluster route +// listener. The same config is used by NATS for both accepting inbound routes +// (TLS server) and soliciting outbound routes (TLS client), so it sets both +// GetCertificate and GetClientCertificate. +// +// Verification is done in VerifyConnection against the CA fetched fresh from +// the cache, not against a static RootCAs/ClientCAs pool that cannot follow a +// rotating CA. InsecureSkipVerify disables Go's default static-root check on +// the dialing side ONLY so verifyConnection can run instead; it does not make +// the connection unauthenticated. Every connection is still mutually verified +// (ClientAuth requires a peer certificate) against live CA material. +// +// GetConfigForClient runs only when accepting a route (TLS server side), where +// the dialing peer's source IP is available on the underlying connection. It +// returns a per-connection config whose VerifyConnection additionally requires +// the peer leaf's IP SAN to match that source IP, binding the certificate to +// the network origin. The dialing side has no equivalent hook (Go does not +// expose the connection in client-certificate callbacks), so it relies on the +// base VerifyConnection: chain + membership against the known peer set. +func (t *clusterTLS) tlsConfig() *tls.Config { + return &tls.Config{ + MinVersion: tls.VersionTLS13, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return t.currentLeafLogged() }, + GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return t.currentLeafLogged() }, + ClientAuth: tls.RequireAnyClientCert, + //nolint:gosec // Not insecure: verifyConnection performs full chain + // verification against the live CA cache. Go's static RootCAs cannot + // track a rotating CA, so default verification is replaced, not removed. + InsecureSkipVerify: true, + VerifyConnection: func(cs tls.ConnectionState) error { return t.verifyLogged(cs, nil) }, + GetConfigForClient: t.configForClient, + } +} + +// verifyLogged runs verify and debug-logs a rejection. The embedded NATS server +// runs with NoLog set, so it swallows its own TLS handshake errors; logging here +// gives deployments a way to see why a cluster peer was rejected. +func (t *clusterTLS) verifyLogged(cs tls.ConnectionState, sourceIP net.IP) error { + err := t.verify(cs, sourceIP) + if err != nil { + t.logger.Debug(t.ctx, "rejected nats cluster peer certificate", slog.Error(err)) + } + return err +} + +// currentLeafLogged mints (or returns the cached) leaf and debug-logs a +// failure. Like verifyLogged, this exists because the embedded NATS server runs +// with NoLog: a currentLeaf error (CA cache error, wrong key type, mint failure) +// is otherwise swallowed by the TLS stack, so a broken CA cache produces zero +// routes with no diagnostic. Used by the GetCertificate callbacks. +func (t *clusterTLS) currentLeafLogged() (*tls.Certificate, error) { + leaf, err := t.currentLeaf() + if err != nil { + t.logger.Debug(t.ctx, "failed to mint nats cluster leaf", slog.Error(err)) + } + return leaf, err +} + +// configForClient builds the per-connection config used when accepting a route. +// It captures the dialing peer's source IP from the underlying connection so +// VerifyConnection can require the peer leaf's IP SAN to match it. NATS calls +// this on each inbound handshake, so a fresh config is allocated per accepted +// connection; that is fine at cluster-route cardinality (a handful of peers). +func (t *clusterTLS) configForClient(chi *tls.ClientHelloInfo) (*tls.Config, error) { + var sourceIP net.IP + if chi.Conn != nil { + if remote := chi.Conn.RemoteAddr(); remote != nil { + if host, _, err := net.SplitHostPort(remote.String()); err == nil { + sourceIP = net.ParseIP(host) + } + } + } + cfg := &tls.Config{ + MinVersion: tls.VersionTLS13, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return t.currentLeafLogged() }, + ClientAuth: tls.RequireAnyClientCert, + //nolint:gosec // See tlsConfig: verification is performed in VerifyConnection. + InsecureSkipVerify: true, + VerifyConnection: func(cs tls.ConnectionState) error { return t.verifyLogged(cs, sourceIP) }, + } + return cfg, nil +} + +// currentLeaf returns the cached leaf, re-minting it when it is missing, near +// expiry, or signed by a CA that is no longer the active one (a rotation). +// +// The whole method holds t.mu so the CA cache, IP, and cached leaf are read as +// a consistent set: a concurrent setClusterCA cannot swap the CA out from under +// the IP we mint with. The lock is therefore held across the SigningKey lookup +// and the (rare) mint. Mints happen only at startup, when the leaf nears expiry +// (~daily), and on CA rotation, so the keygen+sign cost on the lock is +// acceptable; the SigningKey lookup is normally an in-memory cache hit. +func (t *clusterTLS) currentLeaf() (*tls.Certificate, error) { + t.mu.Lock() + defer t.mu.Unlock() + + id, key, err := t.ca.SigningKey(t.ctx) + if err != nil { + return nil, xerrors.Errorf("get signing CA: %w", err) + } + ca, ok := key.(*cryptokeys.NATSCA) + if !ok { + return nil, xerrors.Errorf("unexpected signing key type %T", key) + } + + now := t.clock.Now() + if t.leaf != nil && t.leafSeq == id && now.Before(t.leaf.Leaf.NotAfter.Add(-leafRenewBefore)) { + return t.leaf, nil + } + + leaf, err := mintLeaf(ca, t.ip, now) + if err != nil { + return nil, err + } + t.leaf = leaf + t.leafSeq = id + t.logger.Debug(t.ctx, "minted nats cluster leaf", slog.F("ca_sequence", id)) + return leaf, nil +} + +// mintLeaf creates an ephemeral leaf certificate signed by the active CA. The +// signing CA's sequence is stamped into the leaf's Subject SerialNumber so a +// verifying peer can look up exactly that CA (see verifyConnection), and the +// replica's relay IP is embedded as an IP SAN so a dialing peer can confirm it +// reached the host it intended. The leaf is usable as both a TLS server and +// client certificate because each replica both accepts and dials cluster +// routes. +func mintLeaf(ca *cryptokeys.NATSCA, ip net.IP, now time.Time) (*tls.Certificate, error) { + if len(ip) == 0 { + return nil, xerrors.New("leaf IP SAN is required") + } + + leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, xerrors.Errorf("generate leaf key: %w", err) + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), leafSerialBits)) + if err != nil { + return nil, xerrors.Errorf("generate serial: %w", err) + } + + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: "coder-nats-cluster-leaf", + // SerialNumber carries the sequence of the CA that signed this + // leaf, letting a verifier fetch exactly that CA from its cache. + SerialNumber: strconv.FormatInt(int64(ca.Sequence), 10), + }, + IPAddresses: []net.IP{ip}, + NotBefore: now.Add(-clockSkewToleranceTLS), + NotAfter: now.Add(leafCertValidity), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + + leafDER, err := x509.CreateCertificate(rand.Reader, template, ca.Cert, &leafKey.PublicKey, ca.Key) + if err != nil { + return nil, xerrors.Errorf("create leaf certificate: %w", err) + } + leaf, err := x509.ParseCertificate(leafDER) + if err != nil { + return nil, xerrors.Errorf("parse leaf certificate: %w", err) + } + + return &tls.Certificate{ + Certificate: [][]byte{leafDER}, + PrivateKey: leafKey, + Leaf: leaf, + }, nil +} + +// verify verifies a peer's leaf certificate. It reads the signing CA sequence +// the peer stamped into its leaf, fetches that exact CA from the cache, and +// confirms the leaf chains to it. Using the stamped sequence is not a trust +// decision: the leaf must still chain to OUR trusted copy of that CA, and a CA +// that has been retired is no longer returned by the cache, so leaves from a +// deleted CA are rejected. +// +// It then enforces source binding: when sourceIP is set (the accept side, where +// the dialing peer's connection address is available), the leaf must carry that +// source IP as an IP SAN, binding the certificate to the network origin. Go's +// default hostname verification, which InsecureSkipVerify disables, cannot do +// this because Go does not populate cs.ServerName for IP-based routes. On the +// dial side sourceIP is nil (Go does not expose the connection in the +// client-certificate callbacks), so only the chain is verified there. +func (t *clusterTLS) verify(cs tls.ConnectionState, sourceIP net.IP) error { + if len(cs.PeerCertificates) == 0 { + return xerrors.New("no peer certificate presented") + } + leaf := cs.PeerCertificates[0] + + seq := leaf.Subject.SerialNumber + if seq == "" { + return xerrors.New("peer leaf missing signing CA sequence") + } + + key, err := t.caCache().VerifyingKey(t.ctx, seq) + if err != nil { + return xerrors.Errorf("get CA for sequence %q: %w", seq, err) + } + ca, ok := key.(*cryptokeys.NATSCA) + if !ok { + return xerrors.Errorf("unexpected verifying key type %T", key) + } + + // Leaves carry both ServerAuth and ClientAuth, since each replica is both a + // route server and client. Requiring those specific usages rejects a leaf + // with some unexpected EKU rather than accepting any usage. + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: t.verifyPool(seq, ca.Cert), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + CurrentTime: t.clock.Now(), + }); err != nil { + return xerrors.Errorf("verify peer leaf against CA sequence %q: %w", seq, err) + } + + // On the accept side, confirm the leaf's IP SAN matches the address the + // peer actually connected from. + if len(sourceIP) != 0 && !slices.ContainsFunc(leaf.IPAddresses, sourceIP.Equal) { + return xerrors.Errorf("peer leaf IP SANs %v do not match source IP %s", leaf.IPAddresses, sourceIP) + } + return nil +} + +// verifyPool returns the root pool used to verify a peer leaf minted under the +// given CA sequence, building it once and caching it for reuse. It is called +// from verify on every route handshake; cluster routes are long-lived, so a +// handshake is a rare event, and the common case here is a cache hit (a single +// map lookup). +// +// A miss occurs only the first time a sequence is seen (startup, and once per +// CA rotation), which is the only moment the map can grow, so pruning of expired +// entries is attached to the miss path rather than run on every handshake. An +// entry is dropped once the clock passes the CA cert's NotAfter: no valid leaf +// can chain to an expired CA, and the CA outlives every leaf it signed, so this +// is always safe and bounds the map across rotations. +func (t *clusterTLS) verifyPool(seq string, cert *x509.Certificate) *x509.CertPool { + t.mu.Lock() + defer t.mu.Unlock() + + if cp, ok := t.verifyPools[seq]; ok { + return cp.pool + } + + now := t.clock.Now() + for s, cp := range t.verifyPools { + if now.After(cp.notAfter) { + delete(t.verifyPools, s) + } + } + + pool := x509.NewCertPool() + pool.AddCert(cert) + if t.verifyPools == nil { + t.verifyPools = map[string]cachedVerifyPool{} + } + t.verifyPools[seq] = cachedVerifyPool{pool: pool, notAfter: cert.NotAfter} + return pool +} diff --git a/coderd/x/nats/tls_integration_internal_test.go b/coderd/x/nats/tls_integration_internal_test.go new file mode 100644 index 00000000000..05ef24d79db --- /dev/null +++ b/coderd/x/nats/tls_integration_internal_test.go @@ -0,0 +1,86 @@ +package nats + +import ( + "context" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "cdr.dev/slog/v3/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/cryptokeys" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/testutil" +) + +// TestPubsub_ClusterTLS_RealCA stands up a three-node TLS mesh whose trust root +// is a real CA served by the cryptokeys signing cache against a real DB, then +// verifies a cross-route publish/subscribe round-trip. This exercises the +// integration seam between the cryptokeys CA cache and the x/nats cluster TLS +// callbacks, including the real PEM/x509 round-trip that the synthetic +// generateTestCA helper does not cover. Nodes form a direct full mesh to avoid +// depending on multi-hop route gossip. +func TestPubsub_ClusterTLS_RealCA(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + // Seed an active nats_ca crypto key, mirroring the row the key rotator + // mints in production. The signing cache decodes the PEM secret into a + // *cryptokeys.NATSCA the same way production reads it. + dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: 1, + StartsAt: time.Now().UTC().Add(-time.Hour), + }) + + newNode := func() *Pubsub { + // A real signing cache per node, as each replica builds in coderd.New. + cache, err := cryptokeys.NewSigningCache(ctx, slogtest.Make(t, nil), &cryptokeys.DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + // Nodes mesh on loopback, so the leaf IP SAN must be 127.0.0.1. + return newTLSPubsub(t, cache, net.IPv4(127, 0, 0, 1)) + } + + a := newNode() + b := newNode() + c := newNode() + + addrA := clusterRouteAddress(t, a) + addrB := clusterRouteAddress(t, b) + addrC := clusterRouteAddress(t, c) + require.NoError(t, a.setPeerAddresses([]string{addrB, addrC})) + require.NoError(t, b.setPeerAddresses([]string{addrA, addrC})) + require.NoError(t, c.setPeerAddresses([]string{addrA, addrB})) + + received := make(chan string, 4) + cancelSub, err := c.Subscribe("tls-realca", func(_ context.Context, msg []byte) { + select { + case received <- string(msg): + default: + } + }) + require.NoError(t, err) + defer cancelSub() + + // Routes and subscription interest propagate asynchronously after the + // servers report ready, so retry rather than gate on a one-shot check. + require.Eventually(t, func() bool { + if err := b.Publish("tls-realca", []byte("hello")); err != nil { + return false + } + select { + case msg := <-received: + require.Equal(t, "hello", msg) + return true + case <-time.After(testutil.IntervalMedium): + return false + } + }, testutil.WaitLong, testutil.IntervalFast) +} diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go new file mode 100644 index 00000000000..9bd74e7d44e --- /dev/null +++ b/coderd/x/nats/tls_internal_test.go @@ -0,0 +1,355 @@ +package nats + +import ( + "context" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "strconv" + "testing" + "time" + + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/stretchr/testify/require" + + "cdr.dev/slog/v3/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/cryptokeys" + "github.com/coder/coder/v2/testutil" + "github.com/coder/quartz" +) + +// fakeCACache is an in-memory ClusterCAKeycache for tests. active is returned +// by SigningKey (the CA this replica mints leaves under); byID is consulted by +// VerifyingKey (the CAs this replica trusts when verifying peers). +type fakeCACache struct { + active *cryptokeys.NATSCA + byID map[string]*cryptokeys.NATSCA +} + +func (f *fakeCACache) SigningKey(context.Context) (string, interface{}, error) { + if f.active == nil { + return "", nil, cryptokeys.ErrKeyNotFound + } + return strconv.FormatInt(int64(f.active.Sequence), 10), f.active, nil +} + +func (f *fakeCACache) VerifyingKey(_ context.Context, id string) (interface{}, error) { + ca, ok := f.byID[id] + if !ok { + return nil, cryptokeys.ErrKeyNotFound + } + return ca, nil +} + +func generateTestCA(t *testing.T, sequence int32) *cryptokeys.NATSCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + now := time.Now() + template := &x509.Certificate{ + SerialNumber: big.NewInt(int64(sequence)), + Subject: pkix.Name{CommonName: "coder-nats-ca-test"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(72 * time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + require.NoError(t, err) + cert, err := x509.ParseCertificate(der) + require.NoError(t, err) + + return &cryptokeys.NATSCA{Sequence: sequence, Cert: cert, Key: crypto.Signer(key)} +} + +// newTLSPubsub builds a clustered pubsub whose route listener requires mTLS, +// using the supplied CA cache and leaf IP SAN. Peers dial each other on +// 127.0.0.1 (clusterRouteAddress), so ip must be 127.0.0.1 for routes to form. +func newTLSPubsub(t *testing.T, ca ClusterCAKeycache, ip net.IP) *Pubsub { + t.Helper() + logger := slogtest.Make(t, nil) + ctx := testutil.Context(t, testutil.WaitLong) + ps, err := New(ctx, logger, Options{ + ClusterHost: "127.0.0.1", + ClusterPort: natsserver.RANDOM_PORT, + disableCluster: false, + ClusterCA: ca, + ClusterTLSIP: ip, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = ps.Close() }) + return ps +} + +func numRoutes(t *testing.T, ps *Pubsub) int { + t.Helper() + routes, err := ps.Server.Routez(&natsserver.RoutezOptions{}) + require.NoError(t, err) + return routes.NumRoutes +} + +// TestPubsub_ClusterTLS validates that the embedded NATS server honors the +// tls.Config callbacks on cluster routes: leaves minted from the CA cache form +// a verified mesh, peers under unrelated CAs are rejected, and peers on either +// side of a CA rotation still verify each other. +func TestPubsub_ClusterTLS(t *testing.T) { + t.Parallel() + + t.Run("Mesh", func(t *testing.T) { + t.Parallel() + + ca := generateTestCA(t, 1) + cache := func() *fakeCACache { + return &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + } + a := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + c := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + + addrB := clusterRouteAddress(t, b) + addrC := clusterRouteAddress(t, c) + // Full mesh so any pair can exchange messages over an mTLS route. + require.NoError(t, a.setPeerAddresses([]string{addrB, addrC})) + require.NoError(t, b.setPeerAddresses([]string{addrC})) + + event := "tls-mesh" + got := make(chan []byte, 8) + cancel, err := c.Subscribe(event, func(_ context.Context, msg []byte) { got <- msg }) + require.NoError(t, err) + defer cancel() + + // Retry publishes until the route subscription has propagated. + require.Eventually(t, func() bool { + if err := b.Publish(event, []byte("hello")); err != nil { + return false + } + if err := b.Flush(); err != nil { + return false + } + select { + case msg := <-got: + return string(msg) == "hello" + case <-time.After(testutil.IntervalMedium): + return false + } + }, testutil.WaitLong, testutil.IntervalFast) + }) + + t.Run("WrongCARejected", func(t *testing.T) { + t.Parallel() + + caX := generateTestCA(t, 1) + caY := generateTestCA(t, 1) + a := newTLSPubsub(t, &fakeCACache{active: caX, byID: map[string]*cryptokeys.NATSCA{"1": caX}}, net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, &fakeCACache{active: caY, byID: map[string]*cryptokeys.NATSCA{"1": caY}}, net.IPv4(127, 0, 0, 1)) + + require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) + + // Each side only trusts its own CA, so the route handshake never + // completes and no route is established. + require.Never(t, func() bool { + return numRoutes(t, a) > 0 || numRoutes(t, b) > 0 + }, testutil.WaitShort, testutil.IntervalFast) + }) + + t.Run("RotationOverlap", func(t *testing.T) { + t.Parallel() + + ca1 := generateTestCA(t, 1) + ca2 := generateTestCA(t, 2) + bundle := map[string]*cryptokeys.NATSCA{"1": ca1, "2": ca2} + // a still mints under the old CA; b has already rotated to the new CA. + // Both trust both CAs, so the mesh forms across the rotation overlap. + a := newTLSPubsub(t, &fakeCACache{active: ca1, byID: bundle}, net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, &fakeCACache{active: ca2, byID: bundle}, net.IPv4(127, 0, 0, 1)) + + require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) + + require.Eventually(t, func() bool { + return numRoutes(t, a) > 0 && numRoutes(t, b) > 0 + }, testutil.WaitLong, testutil.IntervalFast) + }) + + t.Run("SANMismatch", func(t *testing.T) { + t.Parallel() + + ca := generateTestCA(t, 1) + cache := func() *fakeCACache { + return &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + } + // b mints its leaf for the wrong IP (not the loopback it actually + // connects from). When b dials a, a's accept-side source binding sees + // b's source IP (127.0.0.1) does not match b's leaf SAN (10.99.99.99) + // and rejects it, so no route forms even though both share a CA. Only b + // dials, so there is no other handshake direction. + a := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, cache(), net.IPv4(10, 99, 99, 99)) + + require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)})) + + require.Never(t, func() bool { + return numRoutes(t, a) > 0 || numRoutes(t, b) > 0 + }, testutil.WaitShort, testutil.IntervalFast) + }) + + t.Run("MixedTLSAndPlaintext", func(t *testing.T) { + t.Parallel() + + ca := generateTestCA(t, 1) + // a requires mTLS on its route listener; b is a plaintext node + // (newTestPubsub leaves ClusterCA nil). Routes must not form in either + // direction: a rollout has to enable TLS on every replica at once. + a := newTLSPubsub(t, &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}, net.IPv4(127, 0, 0, 1)) + b := newTestPubsub(t, clusterTestOptions(t)) + + require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) + require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)})) + + require.Never(t, func() bool { + return numRoutes(t, a) > 0 || numRoutes(t, b) > 0 + }, testutil.WaitShort, testutil.IntervalFast) + }) +} + +// TestPubsub_ClusterTLS_CacheSwap covers the Part C optional-mTLS model: a node +// that boots with the noop CA cache forms no route, and swapping in a real cache +// via SetClusterCA lets routes form over mTLS with no server restart. +func TestPubsub_ClusterTLS_CacheSwap(t *testing.T) { + t.Parallel() + + t.Run("NoopFormsNoRoute", func(t *testing.T) { + t.Parallel() + + ca := generateTestCA(t, 1) + // a boots with the noop cache (production default); b has a real cache. + // a cannot mint a leaf, so its route handshakes fail and no route forms. + a := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, nil) + b := newTLSPubsub(t, &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}, net.IPv4(127, 0, 0, 1)) + + require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) + require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)})) + + require.Never(t, func() bool { + return numRoutes(t, a) > 0 || numRoutes(t, b) > 0 + }, testutil.WaitShort, testutil.IntervalFast) + }) + + t.Run("SwapToRealFormsRoute", func(t *testing.T) { + t.Parallel() + + ca := generateTestCA(t, 1) + realCache := func() *fakeCACache { + return &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + } + // Both boot with the noop cache, then both get the real cache swapped in + // (mirroring the enterprise HA enable path) without a server restart. + a := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, nil) + b := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, nil) + + // Drive peers through fetchers, as production does, rather than calling + // setPeerAddresses directly: SetClusterCA and SetPeerFetcher both trigger + // a peer refresh that reads the current fetcher, so routes converge on + // the fetcher's addresses without racing a manual call. + a.SetClusterCA(realCache(), net.IPv4(127, 0, 0, 1)) + b.SetClusterCA(realCache(), net.IPv4(127, 0, 0, 1)) + + a.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, b)}}) + b.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, a)}}) + + require.Eventually(t, func() bool { + return numRoutes(t, a) > 0 && numRoutes(t, b) > 0 + }, testutil.WaitLong, testutil.IntervalFast) + }) +} + +// TestClusterTLS_verify unit-tests the verifier directly, isolating chain +// verification and source-IP binding that the mesh tests exercise only +// indirectly. +func TestClusterTLS_verify(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ca := generateTestCA(t, 1) + cache := &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + + leafIP := net.IPv4(10, 0, 0, 5) + ct := newClusterTLS(ctx, slogtest.Make(t, nil), nil, cache, net.IPv4(10, 0, 0, 1)) + + // A leaf bound to leafIP, signed by the trusted CA. + leafCert, err := mintLeaf(ca, leafIP, time.Now()) + require.NoError(t, err) + leaf, err := x509.ParseCertificate(leafCert.Certificate[0]) + require.NoError(t, err) + cs := tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}} + + t.Run("DialSideChainOnly", func(t *testing.T) { + t.Parallel() + // No source IP (dial side): only the chain is verified. + require.NoError(t, ct.verify(cs, nil)) + }) + + t.Run("AcceptSideSourceMatches", func(t *testing.T) { + t.Parallel() + // Source IP equals the leaf SAN: accepted. + require.NoError(t, ct.verify(cs, leafIP)) + }) + + t.Run("AcceptSideSourceMismatch", func(t *testing.T) { + t.Parallel() + // The leaf is bound to leafIP, so a connection from a different source + // is rejected even though the chain is valid. + err := ct.verify(cs, net.IPv4(10, 0, 0, 1)) + require.ErrorContains(t, err, "do not match source IP") + }) + + t.Run("UntrustedCARejected", func(t *testing.T) { + t.Parallel() + otherCA := generateTestCA(t, 9) + strangerCert, err := mintLeaf(otherCA, leafIP, time.Now()) + require.NoError(t, err) + stranger, err := x509.ParseCertificate(strangerCert.Certificate[0]) + require.NoError(t, err) + // The stamped sequence (9) is not in the cache, so the CA lookup fails. + err = ct.verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{stranger}}, nil) + require.Error(t, err) + }) +} + +// TestClusterTLS_verifyPool asserts the verify-pool cache reuses a pool for a +// given CA sequence and prunes entries whose CA cert has expired, so the map +// does not grow unbounded across rotations. +func TestClusterTLS_verifyPool(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + clock := quartz.NewMock(t) + clock.Set(time.Now()) + + ca1 := generateTestCA(t, 1) + ca2 := generateTestCA(t, 2) + cache := &fakeCACache{byID: map[string]*cryptokeys.NATSCA{"1": ca1, "2": ca2}} + ct := newClusterTLS(ctx, slogtest.Make(t, nil), clock, cache, net.IPv4(10, 0, 0, 1)) + + // First build for seq 1 caches the pool; a second call returns the same one. + p1 := ct.verifyPool("1", ca1.Cert) + require.Same(t, p1, ct.verifyPool("1", ca1.Cert)) + require.Len(t, ct.verifyPools, 1) + + // Advance past ca1's NotAfter. Building a pool for a new sequence prunes the + // now-expired seq 1 entry, leaving only seq 2. + clock.Set(ca1.Cert.NotAfter.Add(time.Minute)) + ct.verifyPool("2", ca2.Cert) + require.Len(t, ct.verifyPools, 1) + _, ok := ct.verifyPools["1"] + require.False(t, ok, "expired seq 1 pool should be pruned") + _, ok = ct.verifyPools["2"] + require.True(t, ok) +} diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 9f3860f3ca4..58e445ff42e 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "math" + "net" "net/http" "net/url" "strconv" @@ -29,6 +30,7 @@ import ( agplaudit "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/boundaryusage" agplconnectionlog "github.com/coder/coder/v2/coderd/connectionlog" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" agpldbauthz "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/database/dbtime" @@ -881,6 +883,27 @@ func (api *API) Close() error { return api.AGPL.Close() } +// configureNATSClusterTLS swaps the real nats_ca CA cache and this replica's +// relay IP into the NATS pubsub, enabling cluster mTLS. The relay URL host is +// the IP SAN peers verify, so without an IP-based relay URL the cache is left +// as the boot-time noop and routes stay plaintext (token auth only). The CA is +// read lazily by the TLS callbacks on each handshake, so nothing reads it here. +func (api *API) configureNATSClusterTLS(natsPubsub *nats.Pubsub) { + relayURL := api.Options.DeploymentValues.DERP.Server.RelayURL.Value() + if relayURL == nil || relayURL.String() == "" { + api.Logger.Warn(api.ctx, "nats cluster mTLS disabled: no DERP relay URL configured; cluster routes use token auth only") + return + } + ip := net.ParseIP(relayURL.Hostname()) + if ip == nil { + api.Logger.Warn(api.ctx, "nats cluster mTLS disabled: relay URL host is not an IP", + slog.F("relay_host", relayURL.Hostname())) + return + } + natsPubsub.SetClusterCA(api.AGPL.NATSCACache, ip) + api.Logger.Info(api.ctx, "nats cluster mTLS enabled", slog.F("leaf_ip", ip.String())) +} + func (api *API) updateEntitlements(ctx context.Context) error { return api.Entitlements.Update(ctx, func(ctx context.Context) (codersdk.Entitlements, error) { replicas := api.replicaManager.AllPrimary() @@ -1008,6 +1031,9 @@ func (api *API) updateEntitlements(ctx context.Context) error { } if natsPubsub, ok := api.Pubsub.(*nats.Pubsub); ok { + // Swap the real nats_ca CA cache in before peers are known + // so the first route handshake can negotiate mTLS. + api.configureNATSClusterTLS(natsPubsub) natsPubsub.SetPeerFetcher(api.replicaManager) api.replicaManager.SetCallback("nats", natsPubsub.RefreshPeers) } @@ -1040,6 +1066,10 @@ func (api *API) updateEntitlements(ctx context.Context) error { if natsPubsub, ok := api.Pubsub.(*nats.Pubsub); ok { natsPubsub.SetPeerFetcher(nats.NopPeerFetcher{}) + // Revert to the noop CA cache: new route handshakes can no + // longer mint a leaf, so the cluster mesh stops forming. + natsPubsub.SetClusterCA(cryptokeys.NoopSigningKeycache{}, nil) + api.Logger.Info(api.ctx, "nats cluster mTLS disabled") api.replicaManager.SetCallback("nats", nil) } } From 071f40f09b4e79166b456804065656355956566f Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Tue, 7 Jul 2026 01:04:51 +0000 Subject: [PATCH 03/16] feat(coderd): setup for constrained rotation overlap timings --- coderd/cryptokeys/ca.go | 14 ++--- coderd/cryptokeys/ca_internal_test.go | 2 +- coderd/cryptokeys/rotate.go | 22 ++++---- coderd/cryptokeys/rotate_internal_test.go | 7 +-- coderd/x/nats/tls.go | 41 +++++++++----- coderd/x/nats/tls_internal_test.go | 66 +++++++++++++++++++++++ 6 files changed, 120 insertions(+), 32 deletions(-) diff --git a/coderd/cryptokeys/ca.go b/coderd/cryptokeys/ca.go index 20eb100fdb2..7103a08c9e1 100644 --- a/coderd/cryptokeys/ca.go +++ b/coderd/cryptokeys/ca.go @@ -42,15 +42,15 @@ type NATSCA struct { // generateCASecret generates a new self-signed CA certificate and private key // for signing NATS cluster leaf certificates, PEM-encoded into a single -// bundle for storage in the crypto_keys secret column. It is exported so test -// helpers (for example coderd/database/dbgen) can produce nats_ca rows in the -// exact format the rotator writes, rather than duplicating the bundle format. +// bundle for storage in the crypto_keys secret column. // // anchorTime is the key row's starts_at (which may be in the future for a // rotated-in key). keyDuration is the rotator's key duration: the row stays the -// active signer for that long. The certificate must outlive that window plus -// the longest leaf it could sign (NATSCAKeyRetention) plus clock-skew slack, so -// leaves minted just before rotation still chain to a valid CA. +// active signer for that long. The certificate stays valid for NATSCAOverlap +// past that window so that, once the next CA becomes the active signer, this CA +// is still valid while replicas' key caches refresh onto the new one. Leaves +// are separately clamped to expire before this NotAfter (see coderd/x/nats +// mintLeaf), so the overlap only needs to cover the cache-refresh transition. func generateCASecret(anchorTime time.Time, keyDuration time.Duration) (string, error) { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { @@ -69,7 +69,7 @@ func generateCASecret(anchorTime time.Time, keyDuration time.Duration) (string, CommonName: "coder-nats-ca", }, NotBefore: anchorTime.Add(-clockSkewTolerance), - NotAfter: anchorTime.Add(keyDuration + NATSCAKeyRetention + clockSkewTolerance), + NotAfter: anchorTime.Add(keyDuration + NATSCAOverlap), KeyUsage: x509.KeyUsageCertSign, BasicConstraintsValid: true, IsCA: true, diff --git a/coderd/cryptokeys/ca_internal_test.go b/coderd/cryptokeys/ca_internal_test.go index 3fd93d14171..419f52f1cdd 100644 --- a/coderd/cryptokeys/ca_internal_test.go +++ b/coderd/cryptokeys/ca_internal_test.go @@ -41,7 +41,7 @@ func TestCASecretRoundTrip(t *testing.T) { require.True(t, cert.MaxPathLenZero) require.Equal(t, x509.KeyUsageCertSign, cert.KeyUsage) require.Equal(t, now.Add(-clockSkewTolerance), cert.NotBefore) - require.Equal(t, now.Add(keyDuration+NATSCAKeyRetention+clockSkewTolerance), cert.NotAfter) + require.Equal(t, now.Add(keyDuration+NATSCAOverlap), cert.NotAfter) require.Equal(t, cert.PublicKey, signer.Public()) // The cert must outlive its active-signer window so leaves signed at diff --git a/coderd/cryptokeys/rotate.go b/coderd/cryptokeys/rotate.go index db49cfce5c9..775131185bf 100644 --- a/coderd/cryptokeys/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -21,14 +21,15 @@ const ( WorkspaceAppsTokenDuration = time.Minute OIDCConvertTokenDuration = time.Minute * 5 TailnetResumeTokenDuration = time.Hour * 24 - // NATSCAKeyRetention is how long a rotated-out NATS cluster CA is kept as a - // valid trust root after it stops being the active signer. A replica may - // still present a leaf signed by the old CA until that leaf expires, so the - // old CA must remain verifiable for at least the leaf lifetime. This is a CA - // retention budget, not a leaf lifetime: it must be >= the leaf validity - // used when minting (coderd/x/nats leafCertValidity), which is enforced by a - // compile-time assertion there. - NATSCAKeyRetention = time.Hour * 24 * 30 + // NATSCAOverlap is how long a NATS cluster CA certificate stays valid past + // the end of its active-signing window (startsAt + keyDuration). The next CA + // becomes the active signer at the window's end, but replicas keep minting + // leaves with the old CA until their key cache refreshes onto the new one. + // This overlap keeps the old CA valid through that transition, so it must + // exceed the cache refresh interval (plus a small leaf clamp buffer). Leaf + // lifetime imposes nothing here: leaves are clamped to just before their + // signing CA's NotAfter (see coderd/x/nats mintLeaf). + NATSCAOverlap = time.Minute * 30 // defaultRotationInterval is the default interval at which keys are checked for rotation. defaultRotationInterval = time.Minute * 10 @@ -300,7 +301,10 @@ func tokenDuration(feature database.CryptoKeyFeature) time.Duration { case database.CryptoKeyFeatureTailnetResume: return TailnetResumeTokenDuration case database.CryptoKeyFeatureNATSCA: - return NATSCAKeyRetention + // The old CA row only needs to outlive its own certificate, which stays + // valid for NATSCAOverlap past the active-signing window. Keeping the + // row (and thus its trust-root status) beyond cert expiry is pointless. + return NATSCAOverlap default: return 0 } diff --git a/coderd/cryptokeys/rotate_internal_test.go b/coderd/cryptokeys/rotate_internal_test.go index 0bcf7b7a9d7..89216cf7089 100644 --- a/coderd/cryptokeys/rotate_internal_test.go +++ b/coderd/cryptokeys/rotate_internal_test.go @@ -138,9 +138,10 @@ func Test_rotateKeys(t *testing.T) { err := kr.rotateKeys(ctx) require.NoError(t, err) - // The old CA must remain a valid trust root for the maximum leaf - // lifetime after rotation. - expectedDeletesAt := oldKey.ExpiresAt(keyDuration).Add(NATSCAKeyRetention + time.Hour) + // The old CA row is retained roughly as long as its certificate is + // valid: NATSCAOverlap past the active-signing window, plus the + // rotator's standard 1h propagation buffer. + expectedDeletesAt := oldKey.ExpiresAt(keyDuration).Add(NATSCAOverlap + time.Hour) oldKey, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ Feature: oldKey.Feature, Sequence: oldKey.Sequence, diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go index b9599d40a3a..637f942a163 100644 --- a/coderd/x/nats/tls.go +++ b/coderd/x/nats/tls.go @@ -23,14 +23,16 @@ import ( ) const ( - // leafCertValidity is the lifetime of an ephemeral cluster leaf - // certificate. Leaves are re-minted before expiry and whenever the active - // CA rotates, so this can be well under the CA retention window - // (cryptokeys.NATSCAKeyRetention). + // leafCertValidity is the desired lifetime of an ephemeral cluster leaf + // certificate. The actual NotAfter is clamped to expire before the signing + // CA's own NotAfter (see mintLeaf), so near a CA's end a leaf is shorter. leafCertValidity = 24 * time.Hour - // leafRenewBefore re-mints the leaf this long before it expires so an - // in-flight handshake never races expiry. + // leafRenewBefore re-mints the cached leaf this long before it expires so an + // in-flight handshake never races expiry. It is unrelated to the CA clamp. leafRenewBefore = time.Hour + // leafClampBuffer is how far before the signing CA's NotAfter a leaf is + // forced to expire, so a leaf never outlives the CA that signed it. + leafClampBuffer = time.Minute // clusterTLSTimeout is the route TLS handshake timeout. NATS defaults to a // tight 2s, which is flaky under load and in CI. clusterTLSTimeout = 10 * time.Second @@ -41,11 +43,12 @@ const ( clockSkewToleranceTLS = time.Hour ) -// Leaves must never outlive the CA retention window, or a rotated-out CA could -// be deleted while an in-flight leaf still chains to it, breaking verification -// during a rotation overlap. Converting a negative duration to uint fails to -// compile, mechanically enforcing leafCertValidity <= NATSCAKeyRetention. -const _ = uint(cryptokeys.NATSCAKeyRetention - leafCertValidity) +// Sanity check that a CA's active-signing window comfortably exceeds a leaf's +// lifetime; otherwise a freshly minted leaf would always be clamped short. +// Converting a negative duration to an unsigned type fails to compile. uint64 +// (not uint) is required because the positive difference exceeds 32-bit uint on +// 32-bit build targets. +const _ = uint64(cryptokeys.DefaultKeyDuration - leafCertValidity) // ClusterCAKeycache is the read-only view of the nats_ca signing key cache that // the cluster TLS layer needs. cryptokeys.SigningKeycache satisfies it, so the @@ -273,6 +276,20 @@ func mintLeaf(ca *cryptokeys.NATSCA, ip net.IP, now time.Time) (*tls.Certificate return nil, xerrors.Errorf("generate serial: %w", err) } + // Clamp the leaf's expiry to just before the signing CA's own NotAfter so a + // leaf never outlives the CA that signed it (which would fail chain + // verification once the CA expires). Near a CA's end the leaf is simply + // shorter; the cache switches to the newer CA before then under healthy + // rotation. + notAfter := now.Add(leafCertValidity) + if caLimit := ca.Cert.NotAfter.Add(-leafClampBuffer); notAfter.After(caLimit) { + notAfter = caLimit + } + if !notAfter.After(now) { + return nil, xerrors.Errorf("signing CA (seq %d) expires too soon to mint a leaf: CA NotAfter %s", + ca.Sequence, ca.Cert.NotAfter) + } + template := &x509.Certificate{ SerialNumber: serial, Subject: pkix.Name{ @@ -283,7 +300,7 @@ func mintLeaf(ca *cryptokeys.NATSCA, ip net.IP, now time.Time) (*tls.Certificate }, IPAddresses: []net.IP{ip}, NotBefore: now.Add(-clockSkewToleranceTLS), - NotAfter: now.Add(leafCertValidity), + NotAfter: notAfter, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, BasicConstraintsValid: true, diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go index 9bd74e7d44e..330a4254b9f 100644 --- a/coderd/x/nats/tls_internal_test.go +++ b/coderd/x/nats/tls_internal_test.go @@ -353,3 +353,69 @@ func TestClusterTLS_verifyPool(t *testing.T) { _, ok = ct.verifyPools["2"] require.True(t, ok) } + +// generateTestCAWithValidity is like generateTestCA but lets a test control the +// CA certificate's NotAfter, so leaf-clamp behavior near CA expiry is testable. +func generateTestCAWithValidity(t *testing.T, sequence int32, notAfter time.Time) *cryptokeys.NATSCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := &x509.Certificate{ + SerialNumber: big.NewInt(int64(sequence)), + Subject: pkix.Name{CommonName: "coder-nats-ca-test"}, + NotBefore: notAfter.Add(-90 * 24 * time.Hour), + NotAfter: notAfter, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + require.NoError(t, err) + cert, err := x509.ParseCertificate(der) + require.NoError(t, err) + + return &cryptokeys.NATSCA{Sequence: sequence, Cert: cert, Key: crypto.Signer(key)} +} + +// TestMintLeaf_ClampsToCA asserts a leaf's NotAfter is the lesser of the desired +// leaf validity and just before the signing CA's NotAfter, and that minting +// against an already-expired CA fails rather than emitting a dead leaf. +func TestMintLeaf_ClampsToCA(t *testing.T) { + t.Parallel() + + ip := net.IPv4(127, 0, 0, 1) + + t.Run("FullValidityWhenCAHasHeadroom", func(t *testing.T) { + t.Parallel() + now := time.Now() + // CA good for well over a leaf lifetime: leaf gets its full validity. + ca := generateTestCAWithValidity(t, 1, now.Add(30*24*time.Hour)) + leaf, err := mintLeaf(ca, ip, now) + require.NoError(t, err) + require.WithinDuration(t, now.Add(leafCertValidity), leaf.Leaf.NotAfter, time.Second) + }) + + t.Run("ClampedNearCAExpiry", func(t *testing.T) { + t.Parallel() + now := time.Now() + // CA expires in 2h, well under the 24h desired leaf validity. + caNotAfter := now.Add(2 * time.Hour) + ca := generateTestCAWithValidity(t, 1, caNotAfter) + leaf, err := mintLeaf(ca, ip, now) + require.NoError(t, err) + require.WithinDuration(t, caNotAfter.Add(-leafClampBuffer), leaf.Leaf.NotAfter, time.Second) + require.True(t, leaf.Leaf.NotAfter.Before(ca.Cert.NotAfter), + "leaf must expire before its signing CA") + }) + + t.Run("ErrorsWhenCAAlreadyExpired", func(t *testing.T) { + t.Parallel() + now := time.Now() + // CA is within the clamp buffer of expiry: no usable leaf can be minted. + ca := generateTestCAWithValidity(t, 1, now.Add(leafClampBuffer/2)) + _, err := mintLeaf(ca, ip, now) + require.Error(t, err) + require.ErrorContains(t, err, "expires too soon") + }) +} From 227f3e30038e6c9be4f81b9309ecf6533cdfc33c Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Wed, 8 Jul 2026 18:03:54 +0000 Subject: [PATCH 04/16] feat(coderd/x/nats): derive cluster leaf validity from its signing CA A cluster route leaf only authenticates a handshake, so it needs no independent lifetime: its NotAfter now tracks the signing CA's, and re-minting is driven purely by CA rotation. --- coderd/x/nats/tls.go | 53 +++++++++++------------------- coderd/x/nats/tls_internal_test.go | 38 ++++++++------------- 2 files changed, 32 insertions(+), 59 deletions(-) diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go index 637f942a163..eeb67e15e49 100644 --- a/coderd/x/nats/tls.go +++ b/coderd/x/nats/tls.go @@ -23,16 +23,6 @@ import ( ) const ( - // leafCertValidity is the desired lifetime of an ephemeral cluster leaf - // certificate. The actual NotAfter is clamped to expire before the signing - // CA's own NotAfter (see mintLeaf), so near a CA's end a leaf is shorter. - leafCertValidity = 24 * time.Hour - // leafRenewBefore re-mints the cached leaf this long before it expires so an - // in-flight handshake never races expiry. It is unrelated to the CA clamp. - leafRenewBefore = time.Hour - // leafClampBuffer is how far before the signing CA's NotAfter a leaf is - // forced to expire, so a leaf never outlives the CA that signed it. - leafClampBuffer = time.Minute // clusterTLSTimeout is the route TLS handshake timeout. NATS defaults to a // tight 2s, which is flaky under load and in CI. clusterTLSTimeout = 10 * time.Second @@ -43,13 +33,6 @@ const ( clockSkewToleranceTLS = time.Hour ) -// Sanity check that a CA's active-signing window comfortably exceeds a leaf's -// lifetime; otherwise a freshly minted leaf would always be clamped short. -// Converting a negative duration to an unsigned type fails to compile. uint64 -// (not uint) is required because the positive difference exceeds 32-bit uint on -// 32-bit build targets. -const _ = uint64(cryptokeys.DefaultKeyDuration - leafCertValidity) - // ClusterCAKeycache is the read-only view of the nats_ca signing key cache that // the cluster TLS layer needs. cryptokeys.SigningKeycache satisfies it, so the // nats_ca cache is passed straight through with no adapter. @@ -217,15 +200,17 @@ func (t *clusterTLS) configForClient(chi *tls.ClientHelloInfo) (*tls.Config, err return cfg, nil } -// currentLeaf returns the cached leaf, re-minting it when it is missing, near -// expiry, or signed by a CA that is no longer the active one (a rotation). +// currentLeaf returns the cached leaf, re-minting it when it is missing or +// signed by a CA that is no longer the active, still-valid one (a rotation). +// A leaf carries no independent lifetime: it is valid exactly as long as its +// signing CA (see mintLeaf), so re-minting is driven purely by CA rotation. // // The whole method holds t.mu so the CA cache, IP, and cached leaf are read as // a consistent set: a concurrent setClusterCA cannot swap the CA out from under // the IP we mint with. The lock is therefore held across the SigningKey lookup -// and the (rare) mint. Mints happen only at startup, when the leaf nears expiry -// (~daily), and on CA rotation, so the keygen+sign cost on the lock is -// acceptable; the SigningKey lookup is normally an in-memory cache hit. +// and the (rare) mint. Mints happen only at startup and on CA rotation, so the +// keygen+sign cost on the lock is acceptable; the SigningKey lookup is normally +// an in-memory cache hit. func (t *clusterTLS) currentLeaf() (*tls.Certificate, error) { t.mu.Lock() defer t.mu.Unlock() @@ -240,7 +225,10 @@ func (t *clusterTLS) currentLeaf() (*tls.Certificate, error) { } now := t.clock.Now() - if t.leaf != nil && t.leafSeq == id && now.Before(t.leaf.Leaf.NotAfter.Add(-leafRenewBefore)) { + // Reuse the cached leaf while it was signed by the still-active, still-valid + // CA. A CA rotation (sequence change) or the active CA expiring forces a + // re-mint; there is no separate leaf lifetime to track. + if t.leaf != nil && t.leafSeq == id && now.Before(ca.Cert.NotAfter) { return t.leaf, nil } @@ -276,19 +264,16 @@ func mintLeaf(ca *cryptokeys.NATSCA, ip net.IP, now time.Time) (*tls.Certificate return nil, xerrors.Errorf("generate serial: %w", err) } - // Clamp the leaf's expiry to just before the signing CA's own NotAfter so a - // leaf never outlives the CA that signed it (which would fail chain - // verification once the CA expires). Near a CA's end the leaf is simply - // shorter; the cache switches to the newer CA before then under healthy - // rotation. - notAfter := now.Add(leafCertValidity) - if caLimit := ca.Cert.NotAfter.Add(-leafClampBuffer); notAfter.After(caLimit) { - notAfter = caLimit - } - if !notAfter.After(now) { - return nil, xerrors.Errorf("signing CA (seq %d) expires too soon to mint a leaf: CA NotAfter %s", + // A leaf is only ever used to authenticate a handshake, so it need only be + // valid as long as the CA that signed it. Tie the leaf's NotAfter to the + // CA's so a leaf never outlives its CA and carries no independent lifetime. + // An expired active CA means a fully-dead rotator; fail loud rather than + // mint a dead leaf. + if !ca.Cert.NotAfter.After(now) { + return nil, xerrors.Errorf("signing CA (seq %d) is expired: NotAfter %s", ca.Sequence, ca.Cert.NotAfter) } + notAfter := ca.Cert.NotAfter template := &x509.Certificate{ SerialNumber: serial, diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go index 330a4254b9f..962429c8bf7 100644 --- a/coderd/x/nats/tls_internal_test.go +++ b/coderd/x/nats/tls_internal_test.go @@ -378,44 +378,32 @@ func generateTestCAWithValidity(t *testing.T, sequence int32, notAfter time.Time return &cryptokeys.NATSCA{Sequence: sequence, Cert: cert, Key: crypto.Signer(key)} } -// TestMintLeaf_ClampsToCA asserts a leaf's NotAfter is the lesser of the desired -// leaf validity and just before the signing CA's NotAfter, and that minting -// against an already-expired CA fails rather than emitting a dead leaf. -func TestMintLeaf_ClampsToCA(t *testing.T) { +// TestMintLeaf asserts a leaf's NotAfter is exactly its signing CA's NotAfter +// (a leaf carries no independent lifetime), and that minting against an +// already-expired CA fails rather than emitting a dead leaf. +func TestMintLeaf(t *testing.T) { t.Parallel() ip := net.IPv4(127, 0, 0, 1) - t.Run("FullValidityWhenCAHasHeadroom", func(t *testing.T) { + t.Run("MatchesCAValidity", func(t *testing.T) { t.Parallel() now := time.Now() - // CA good for well over a leaf lifetime: leaf gets its full validity. - ca := generateTestCAWithValidity(t, 1, now.Add(30*24*time.Hour)) - leaf, err := mintLeaf(ca, ip, now) - require.NoError(t, err) - require.WithinDuration(t, now.Add(leafCertValidity), leaf.Leaf.NotAfter, time.Second) - }) - - t.Run("ClampedNearCAExpiry", func(t *testing.T) { - t.Parallel() - now := time.Now() - // CA expires in 2h, well under the 24h desired leaf validity. - caNotAfter := now.Add(2 * time.Hour) - ca := generateTestCAWithValidity(t, 1, caNotAfter) + // generateTestCA mints a 72h CA; the leaf's NotAfter tracks it exactly. + ca := generateTestCA(t, 1) leaf, err := mintLeaf(ca, ip, now) require.NoError(t, err) - require.WithinDuration(t, caNotAfter.Add(-leafClampBuffer), leaf.Leaf.NotAfter, time.Second) - require.True(t, leaf.Leaf.NotAfter.Before(ca.Cert.NotAfter), - "leaf must expire before its signing CA") + require.WithinDuration(t, ca.Cert.NotAfter, leaf.Leaf.NotAfter, time.Second) + require.WithinDuration(t, now.Add(-clockSkewToleranceTLS), leaf.Leaf.NotBefore, time.Second) }) - t.Run("ErrorsWhenCAAlreadyExpired", func(t *testing.T) { + t.Run("ErrorsWhenCAExpired", func(t *testing.T) { t.Parallel() now := time.Now() - // CA is within the clamp buffer of expiry: no usable leaf can be minted. - ca := generateTestCAWithValidity(t, 1, now.Add(leafClampBuffer/2)) + // CA's NotAfter is already in the past: no usable leaf can be minted. + ca := generateTestCAWithValidity(t, 1, now.Add(-time.Minute)) _, err := mintLeaf(ca, ip, now) require.Error(t, err) - require.ErrorContains(t, err, "expires too soon") + require.ErrorContains(t, err, "expired") }) } From e92ab37fb3fcc75703779664e5cbc9dcb513e642 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Wed, 8 Jul 2026 21:13:34 +0000 Subject: [PATCH 05/16] feat(coderd/x/nats): reject cluster routes from unknown replica IPs Accept-side verification now requires the connection source IP to be one of this replica's configured cluster-route peers (from the replicas table via the NATS peer fetcher) in addition to matching the peer leaf's IP SAN. --- coderd/x/nats/cluster.go | 37 ++++++++++++++++++ coderd/x/nats/pubsub.go | 8 ++++ coderd/x/nats/tls.go | 21 ++++++++++ coderd/x/nats/tls_internal_test.go | 63 +++++++++++++++++++++++++++++- 4 files changed, 127 insertions(+), 2 deletions(-) diff --git a/coderd/x/nats/cluster.go b/coderd/x/nats/cluster.go index 3779aa2c9b4..2beb0a41867 100644 --- a/coderd/x/nats/cluster.go +++ b/coderd/x/nats/cluster.go @@ -63,6 +63,39 @@ func (p *Pubsub) SetClusterCA(ca ClusterCAKeycache, ip net.IP) { p.RefreshPeers() } +// knownPeerIPs returns the IPs of the currently configured cluster routes, so +// the set a route may be accepted from is exactly the set this replica dials. +// It reads the snapshot published by setPeerAddresses without taking clusterMu, +// so the accept-side handshake path (which runs while setPeerAddresses may hold +// clusterMu across a server reload) never blocks on route reconfiguration. +func (p *Pubsub) knownPeerIPs() []net.IP { + if ips := p.peerIPs.Load(); ips != nil { + return *ips + } + return nil +} + +// routeIPs derives the accept-side peer IP set from configured routes, skipping +// non-IP hosts. It reuses the same route set setPeerAddresses applies, so the +// accepted-from set stays in lockstep with the dialed set (both from the peer +// fetcher, ultimately the replicas table). +func routeIPs(routes []*url.URL) []net.IP { + ips := make([]net.IP, 0, len(routes)) + for _, route := range routes { + if route == nil { + continue + } + host, _, err := net.SplitHostPort(route.Host) + if err != nil { + continue + } + if ip := net.ParseIP(host); ip != nil { + ips = append(ips, ip) + } + } + return ips +} + // RefreshPeers signals the peer refresh worker to fetch and apply the latest // peer route addresses. Multiple pending refreshes are coalesced. func (p *Pubsub) RefreshPeers() { @@ -131,6 +164,10 @@ func (p *Pubsub) setPeerAddresses(addresses []string) error { } p.serverOpts = newOpts.Clone() p.currentRoutes = cloneRouteURLs(routes) + // Publish the accept-side peer IP set in lockstep with the routes so the + // handshake path reads it lock-free (see knownPeerIPs). + ips := routeIPs(routes) + p.peerIPs.Store(&ips) return nil } diff --git a/coderd/x/nats/pubsub.go b/coderd/x/nats/pubsub.go index 1638be54271..064013145e3 100644 --- a/coderd/x/nats/pubsub.go +++ b/coderd/x/nats/pubsub.go @@ -8,6 +8,7 @@ import ( "net" "net/url" "sync" + "sync/atomic" "time" natsserver "github.com/nats-io/nats-server/v2/server" @@ -199,6 +200,10 @@ type Pubsub struct { clustered bool serverOpts *natsserver.Options currentRoutes []*url.URL + // peerIPs holds the IPs of the currently configured cluster routes, + // published by setPeerAddresses (under clusterMu) and read lock-free on the + // accept-side handshake path so verification never contends on clusterMu. + peerIPs atomic.Pointer[[]net.IP] // clusterTLS is non-nil when the cluster route listener runs mutual TLS. // Its valid-peer-IP set is kept in sync with currentRoutes. clusterTLS *clusterTLS @@ -367,6 +372,9 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub, p.serverOpts = sopts.Clone() p.currentRoutes = cloneRouteURLs(sopts.Routes) p.clusterTLS = ct + if ct != nil { + ct.peerIPs = p.knownPeerIPs + } handlers := p.buildConnHandlers() publishPool, err := newConnPool(ns, opts, handlers, opts.PublishConns, "coder-pubsub-pub") diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go index eeb67e15e49..a11fa2faae2 100644 --- a/coderd/x/nats/tls.go +++ b/coderd/x/nats/tls.go @@ -69,6 +69,13 @@ type clusterTLS struct { // sequence, so the pool is built once and reused across handshakes. Expired // entries are pruned on insert to bound the map across rotations. verifyPools map[string]cachedVerifyPool + + // peerIPs returns the current set of replica cluster IPs a route may be + // accepted from. It is the same set this replica dials (the NATS peer + // fetcher, ultimately the replicas table), queried live per handshake so it + // tracks replicas joining and leaving without a cached copy. Handshakes are + // rare (cluster routes are long-lived), so a live query is cheap. + peerIPs func() []net.IP } // cachedVerifyPool is a verify root pool plus the NotAfter of the CA cert it @@ -357,6 +364,20 @@ func (t *clusterTLS) verify(cs tls.ConnectionState, sourceIP net.IP) error { if len(sourceIP) != 0 && !slices.ContainsFunc(leaf.IPAddresses, sourceIP.Equal) { return xerrors.Errorf("peer leaf IP SANs %v do not match source IP %s", leaf.IPAddresses, sourceIP) } + + // On the accept side, the source must also be a currently-known replica, so + // a valid leaf presented from an address outside the cluster is rejected. + // The replica set is the source of truth: an empty or unavailable set + // rejects, rather than falling open. + if len(sourceIP) != 0 { + var known []net.IP + if t.peerIPs != nil { + known = t.peerIPs() + } + if !slices.ContainsFunc(known, sourceIP.Equal) { + return xerrors.Errorf("source IP %s is not a known replica", sourceIP) + } + } return nil } diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go index 962429c8bf7..22b734cd849 100644 --- a/coderd/x/nats/tls_internal_test.go +++ b/coderd/x/nats/tls_internal_test.go @@ -114,11 +114,14 @@ func TestPubsub_ClusterTLS(t *testing.T) { b := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) c := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + addrA := clusterRouteAddress(t, a) addrB := clusterRouteAddress(t, b) addrC := clusterRouteAddress(t, c) - // Full mesh so any pair can exchange messages over an mTLS route. + // Full symmetric mesh: every node must know a peer to accept a route + // from it (accept-side membership), so each is given the other two. require.NoError(t, a.setPeerAddresses([]string{addrB, addrC})) - require.NoError(t, b.setPeerAddresses([]string{addrC})) + require.NoError(t, b.setPeerAddresses([]string{addrA, addrC})) + require.NoError(t, c.setPeerAddresses([]string{addrA, addrB})) event := "tls-mesh" got := make(chan []byte, 8) @@ -171,7 +174,9 @@ func TestPubsub_ClusterTLS(t *testing.T) { a := newTLSPubsub(t, &fakeCACache{active: ca1, byID: bundle}, net.IPv4(127, 0, 0, 1)) b := newTLSPubsub(t, &fakeCACache{active: ca2, byID: bundle}, net.IPv4(127, 0, 0, 1)) + // Symmetric peers so each side accepts a route from the other. require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) + require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)})) require.Eventually(t, func() bool { return numRoutes(t, a) > 0 && numRoutes(t, b) > 0 @@ -282,6 +287,9 @@ func TestClusterTLS_verify(t *testing.T) { leafIP := net.IPv4(10, 0, 0, 5) ct := newClusterTLS(ctx, slogtest.Make(t, nil), nil, cache, net.IPv4(10, 0, 0, 1)) + // leafIP is a known replica so accept-side membership passes; the source + // binding and CA checks are what these subtests exercise. + ct.peerIPs = func() []net.IP { return []net.IP{leafIP} } // A leaf bound to leafIP, signed by the trusted CA. leafCert, err := mintLeaf(ca, leafIP, time.Now()) @@ -323,6 +331,57 @@ func TestClusterTLS_verify(t *testing.T) { }) } +// TestClusterTLS_verifyReplicaMembership asserts that, on the accept side, the +// connection source IP must belong to the current peer (replica) set. A valid +// leaf with a matching SAN is still rejected when its source is not a known +// replica, and an empty set rejects rather than falling open. +func TestClusterTLS_verifyReplicaMembership(t *testing.T) { + t.Parallel() + + // newVerifier builds a fresh clusterTLS plus a valid leaf/connection state + // bound to leafIP, signed by a trusted CA, with peerIPs returning peers. + leafIP := net.IPv4(10, 0, 0, 5) + newVerifier := func(t *testing.T, peers ...net.IP) (*clusterTLS, tls.ConnectionState) { + t.Helper() + ctx := testutil.Context(t, testutil.WaitShort) + ca := generateTestCA(t, 1) + cache := &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + ct := newClusterTLS(ctx, slogtest.Make(t, nil), nil, cache, leafIP) + ct.peerIPs = func() []net.IP { return peers } + leafCert, err := mintLeaf(ca, leafIP, time.Now()) + require.NoError(t, err) + leaf, err := x509.ParseCertificate(leafCert.Certificate[0]) + require.NoError(t, err) + return ct, tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}} + } + + t.Run("NotAKnownReplica", func(t *testing.T) { + t.Parallel() + // Peer set does NOT include leafIP: even with a valid chain and a SAN + // matching the source, a source IP that is not a known replica is + // rejected. + ct, cs := newVerifier(t, net.IPv4(10, 0, 0, 9)) + err := ct.verify(cs, leafIP) + require.ErrorContains(t, err, "not a known replica") + }) + + t.Run("KnownReplica", func(t *testing.T) { + t.Parallel() + // leafIP is in the peer set and matches the SAN + source: accepted. + ct, cs := newVerifier(t, leafIP) + require.NoError(t, ct.verify(cs, leafIP)) + }) + + t.Run("EmptySetRejects", func(t *testing.T) { + t.Parallel() + // The replica set is the source of truth: with no known peers, a route + // is rejected rather than accepted. + ct, cs := newVerifier(t) + err := ct.verify(cs, leafIP) + require.ErrorContains(t, err, "not a known replica") + }) +} + // TestClusterTLS_verifyPool asserts the verify-pool cache reuses a pool for a // given CA sequence and prunes entries whose CA cert has expired, so the map // does not grow unbounded across rotations. From ec5533707186e430b3c171dbd312157d647b2013 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Wed, 8 Jul 2026 22:15:45 +0000 Subject: [PATCH 06/16] refactor(coderd): tidy up NATS cluster mTLS after review Drop the redundant ClusterCAKeycache interface in favor of cryptokeys.SigningKeycache, remove the unused CryptoKey.DecodeString method, merge the NATS TLS integration test into the internal test file, and note the CA-cache boot refactor as a TODO. --- cli/server.go | 7 ++ coderd/database/modelmethods.go | 12 --- coderd/x/nats/cluster.go | 3 +- coderd/x/nats/pubsub.go | 3 +- coderd/x/nats/tls.go | 21 +---- .../x/nats/tls_integration_internal_test.go | 86 ------------------- coderd/x/nats/tls_internal_test.go | 78 ++++++++++++++++- 7 files changed, 91 insertions(+), 119 deletions(-) delete mode 100644 coderd/x/nats/tls_integration_internal_test.go diff --git a/cli/server.go b/cli/server.go index ce423e1fa38..7886c4fdd86 100644 --- a/cli/server.go +++ b/cli/server.go @@ -852,6 +852,13 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. // dependency and forms no routes. Enterprise HA swaps in the // real nats_ca cache plus the relay IP via Pubsub.SetClusterCA // once clustering is licensed. + // + // TODO: the real CA cache cannot be built here because + // options.Database is not yet fully instantiated (it is + // wrapped with metrics/dbauthz downstream). This split boot + // (noop here, real cache swapped in by enterprise) wants a + // refactor so the CA cache can be constructed once alongside + // the database. ClusterCA: cryptokeys.NoopSigningKeycache{}, }) if err != nil { diff --git a/coderd/database/modelmethods.go b/coderd/database/modelmethods.go index ae3c3075131..67a6c7812d9 100644 --- a/coderd/database/modelmethods.go +++ b/coderd/database/modelmethods.go @@ -2,7 +2,6 @@ package database import ( "database/sql" - "encoding/hex" "fmt" "slices" "sort" @@ -869,17 +868,6 @@ func (k CryptoKey) ExpiresAt(keyDuration time.Duration) time.Time { return k.StartsAt.Add(keyDuration).UTC() } -// DecodeString hex-decodes the key's secret. It is only valid for features -// whose secret is hex-encoded bytes; it must NOT be used for nats_ca, whose -// secret is a PEM certificate+key bundle (see coderd/cryptokeys parseCASecret). -// -// TODO: this method currently has no callers (the keycache hex-decodes via its -// own helper). Investigate removing it, or making it feature-aware, so the -// secret column's dual format (hex bytes vs PEM bundle) cannot be misdecoded. -func (k CryptoKey) DecodeString() ([]byte, error) { - return hex.DecodeString(k.Secret.String) -} - func (k CryptoKey) CanSign(now time.Time) bool { isAfterStart := !k.StartsAt.IsZero() && !now.Before(k.StartsAt) return isAfterStart && k.CanVerify(now) diff --git a/coderd/x/nats/cluster.go b/coderd/x/nats/cluster.go index 2beb0a41867..5cc73792728 100644 --- a/coderd/x/nats/cluster.go +++ b/coderd/x/nats/cluster.go @@ -11,6 +11,7 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog/v3" + "github.com/coder/coder/v2/coderd/cryptokeys" ) const defaultClusterTokenUsername = "coder" @@ -55,7 +56,7 @@ func (p *Pubsub) SetPeerFetcher(fetcher PeerFetcher) { // with cluster TLS enabled (Options.ClusterCA set, which installs the TLS // callbacks). Passing a noop cache reverts to no mTLS: new route handshakes // can no longer mint a leaf and will not form. -func (p *Pubsub) SetClusterCA(ca ClusterCAKeycache, ip net.IP) { +func (p *Pubsub) SetClusterCA(ca cryptokeys.SigningKeycache, ip net.IP) { if p.clusterTLS == nil { return } diff --git a/coderd/x/nats/pubsub.go b/coderd/x/nats/pubsub.go index 064013145e3..7cd4913fbdc 100644 --- a/coderd/x/nats/pubsub.go +++ b/coderd/x/nats/pubsub.go @@ -17,6 +17,7 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog/v3" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database/pubsub" "github.com/coder/quartz" ) @@ -129,7 +130,7 @@ type Options struct { // certificate from the active nats_ca CA and verifies peers against the // CA fetched from this cache on each handshake. Nil keeps routes // plaintext (token auth only). cryptokeys.SigningKeycache satisfies this. - ClusterCA ClusterCAKeycache + ClusterCA cryptokeys.SigningKeycache // ClusterTLSIP is this replica's relay IP, embedded as an IP SAN in the // leaf certificate and matched against the dialed host when verifying a diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go index a11fa2faae2..f199c9a51fe 100644 --- a/coderd/x/nats/tls.go +++ b/coderd/x/nats/tls.go @@ -33,19 +33,6 @@ const ( clockSkewToleranceTLS = time.Hour ) -// ClusterCAKeycache is the read-only view of the nats_ca signing key cache that -// the cluster TLS layer needs. cryptokeys.SigningKeycache satisfies it, so the -// nats_ca cache is passed straight through with no adapter. -// -// SigningKey returns the active CA used to mint this replica's leaf; -// VerifyingKey returns a specific CA by its crypto_keys sequence, used to -// verify a peer leaf that was minted under that (possibly older) CA during a -// rotation overlap. Both return a *cryptokeys.NATSCA. -type ClusterCAKeycache interface { - SigningKey(ctx context.Context) (id string, key interface{}, err error) - VerifyingKey(ctx context.Context, id string) (key interface{}, err error) -} - // clusterTLS builds the cluster route *tls.Config. Certificate selection and // peer verification are tls.Config callbacks that consult the CA cache on each // use, so a CA rotation is tracked without restarting or reloading the server. @@ -58,7 +45,7 @@ type clusterTLS struct { // ca and ip are swapped together by setClusterCA: under the default noop // cache no leaf can be minted (so no route forms), and the real cache plus // this replica's relay IP are installed once cluster mTLS is enabled. - ca ClusterCAKeycache + ca cryptokeys.SigningKeycache ip net.IP // leaf is the cached leaf certificate. leafSeq is the active CA sequence it // was minted under; a change means the CA rotated and the leaf is stale. @@ -85,7 +72,7 @@ type cachedVerifyPool struct { notAfter time.Time } -func newClusterTLS(ctx context.Context, logger slog.Logger, clock quartz.Clock, ca ClusterCAKeycache, ip net.IP) *clusterTLS { +func newClusterTLS(ctx context.Context, logger slog.Logger, clock quartz.Clock, ca cryptokeys.SigningKeycache, ip net.IP) *clusterTLS { if clock == nil { clock = quartz.NewReal() } @@ -104,7 +91,7 @@ func newClusterTLS(ctx context.Context, logger slog.Logger, clock quartz.Clock, // routes negotiate mTLS, and reverting to a noop cache makes leaf minting fail // so no new route can form. A swap clears the cached leaf so the next handshake // re-mints under the new CA/IP. -func (t *clusterTLS) setClusterCA(ca ClusterCAKeycache, ip net.IP) { +func (t *clusterTLS) setClusterCA(ca cryptokeys.SigningKeycache, ip net.IP) { t.mu.Lock() defer t.mu.Unlock() t.ca = ca @@ -118,7 +105,7 @@ func (t *clusterTLS) setClusterCA(ca ClusterCAKeycache, ip net.IP) { // caCache returns the current CA cache under lock so callers do not hold the // lock across cache I/O. -func (t *clusterTLS) caCache() ClusterCAKeycache { +func (t *clusterTLS) caCache() cryptokeys.SigningKeycache { t.mu.Lock() defer t.mu.Unlock() return t.ca diff --git a/coderd/x/nats/tls_integration_internal_test.go b/coderd/x/nats/tls_integration_internal_test.go deleted file mode 100644 index 05ef24d79db..00000000000 --- a/coderd/x/nats/tls_integration_internal_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package nats - -import ( - "context" - "net" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "cdr.dev/slog/v3/sloggers/slogtest" - "github.com/coder/coder/v2/coderd/cryptokeys" - "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/dbgen" - "github.com/coder/coder/v2/coderd/database/dbtestutil" - "github.com/coder/coder/v2/codersdk" - "github.com/coder/coder/v2/testutil" -) - -// TestPubsub_ClusterTLS_RealCA stands up a three-node TLS mesh whose trust root -// is a real CA served by the cryptokeys signing cache against a real DB, then -// verifies a cross-route publish/subscribe round-trip. This exercises the -// integration seam between the cryptokeys CA cache and the x/nats cluster TLS -// callbacks, including the real PEM/x509 round-trip that the synthetic -// generateTestCA helper does not cover. Nodes form a direct full mesh to avoid -// depending on multi-hop route gossip. -func TestPubsub_ClusterTLS_RealCA(t *testing.T) { - t.Parallel() - - db, _ := dbtestutil.NewDB(t) - ctx := testutil.Context(t, testutil.WaitLong) - - // Seed an active nats_ca crypto key, mirroring the row the key rotator - // mints in production. The signing cache decodes the PEM secret into a - // *cryptokeys.NATSCA the same way production reads it. - dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureNATSCA, - Sequence: 1, - StartsAt: time.Now().UTC().Add(-time.Hour), - }) - - newNode := func() *Pubsub { - // A real signing cache per node, as each replica builds in coderd.New. - cache, err := cryptokeys.NewSigningCache(ctx, slogtest.Make(t, nil), &cryptokeys.DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA) - require.NoError(t, err) - t.Cleanup(func() { _ = cache.Close() }) - // Nodes mesh on loopback, so the leaf IP SAN must be 127.0.0.1. - return newTLSPubsub(t, cache, net.IPv4(127, 0, 0, 1)) - } - - a := newNode() - b := newNode() - c := newNode() - - addrA := clusterRouteAddress(t, a) - addrB := clusterRouteAddress(t, b) - addrC := clusterRouteAddress(t, c) - require.NoError(t, a.setPeerAddresses([]string{addrB, addrC})) - require.NoError(t, b.setPeerAddresses([]string{addrA, addrC})) - require.NoError(t, c.setPeerAddresses([]string{addrA, addrB})) - - received := make(chan string, 4) - cancelSub, err := c.Subscribe("tls-realca", func(_ context.Context, msg []byte) { - select { - case received <- string(msg): - default: - } - }) - require.NoError(t, err) - defer cancelSub() - - // Routes and subscription interest propagate asynchronously after the - // servers report ready, so retry rather than gate on a one-shot check. - require.Eventually(t, func() bool { - if err := b.Publish("tls-realca", []byte("hello")); err != nil { - return false - } - select { - case msg := <-received: - require.Equal(t, "hello", msg) - return true - case <-time.After(testutil.IntervalMedium): - return false - } - }, testutil.WaitLong, testutil.IntervalFast) -} diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go index 22b734cd849..4ae30fe676b 100644 --- a/coderd/x/nats/tls_internal_test.go +++ b/coderd/x/nats/tls_internal_test.go @@ -20,11 +20,15 @@ import ( "cdr.dev/slog/v3/sloggers/slogtest" "github.com/coder/coder/v2/coderd/cryptokeys" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" "github.com/coder/quartz" ) -// fakeCACache is an in-memory ClusterCAKeycache for tests. active is returned +// fakeCACache is an in-memory cryptokeys.SigningKeycache for tests. active is returned // by SigningKey (the CA this replica mints leaves under); byID is consulted by // VerifyingKey (the CAs this replica trusts when verifying peers). type fakeCACache struct { @@ -47,6 +51,8 @@ func (f *fakeCACache) VerifyingKey(_ context.Context, id string) (interface{}, e return ca, nil } +func (*fakeCACache) Close() error { return nil } + func generateTestCA(t *testing.T, sequence int32) *cryptokeys.NATSCA { t.Helper() key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) @@ -73,7 +79,7 @@ func generateTestCA(t *testing.T, sequence int32) *cryptokeys.NATSCA { // newTLSPubsub builds a clustered pubsub whose route listener requires mTLS, // using the supplied CA cache and leaf IP SAN. Peers dial each other on // 127.0.0.1 (clusterRouteAddress), so ip must be 127.0.0.1 for routes to form. -func newTLSPubsub(t *testing.T, ca ClusterCAKeycache, ip net.IP) *Pubsub { +func newTLSPubsub(t *testing.T, ca cryptokeys.SigningKeycache, ip net.IP) *Pubsub { t.Helper() logger := slogtest.Make(t, nil) ctx := testutil.Context(t, testutil.WaitLong) @@ -466,3 +472,71 @@ func TestMintLeaf(t *testing.T) { require.ErrorContains(t, err, "expired") }) } + +// TestPubsub_ClusterTLS_RealCA stands up a three-node TLS mesh whose trust root +// is a real CA served by the cryptokeys signing cache against a real DB, then +// verifies a cross-route publish/subscribe round-trip. This exercises the +// integration seam between the cryptokeys CA cache and the x/nats cluster TLS +// callbacks, including the real PEM/x509 round-trip that the synthetic +// generateTestCA helper does not cover. Nodes form a direct full mesh to avoid +// depending on multi-hop route gossip. +func TestPubsub_ClusterTLS_RealCA(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + // Seed an active nats_ca crypto key, mirroring the row the key rotator + // mints in production. The signing cache decodes the PEM secret into a + // *cryptokeys.NATSCA the same way production reads it. + dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: 1, + StartsAt: time.Now().UTC().Add(-time.Hour), + }) + + newNode := func() *Pubsub { + // A real signing cache per node, as each replica builds in coderd.New. + cache, err := cryptokeys.NewSigningCache(ctx, slogtest.Make(t, nil), &cryptokeys.DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + // Nodes mesh on loopback, so the leaf IP SAN must be 127.0.0.1. + return newTLSPubsub(t, cache, net.IPv4(127, 0, 0, 1)) + } + + a := newNode() + b := newNode() + c := newNode() + + addrA := clusterRouteAddress(t, a) + addrB := clusterRouteAddress(t, b) + addrC := clusterRouteAddress(t, c) + require.NoError(t, a.setPeerAddresses([]string{addrB, addrC})) + require.NoError(t, b.setPeerAddresses([]string{addrA, addrC})) + require.NoError(t, c.setPeerAddresses([]string{addrA, addrB})) + + received := make(chan string, 4) + cancelSub, err := c.Subscribe("tls-realca", func(_ context.Context, msg []byte) { + select { + case received <- string(msg): + default: + } + }) + require.NoError(t, err) + defer cancelSub() + + // Routes and subscription interest propagate asynchronously after the + // servers report ready, so retry rather than gate on a one-shot check. + require.Eventually(t, func() bool { + if err := b.Publish("tls-realca", []byte("hello")); err != nil { + return false + } + select { + case msg := <-received: + require.Equal(t, "hello", msg) + return true + case <-time.After(testutil.IntervalMedium): + return false + } + }, testutil.WaitLong, testutil.IntervalFast) +} From 8229557fcfae4fe279db236142728a5092c372f2 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Wed, 8 Jul 2026 23:15:09 +0000 Subject: [PATCH 07/16] feat(coderd): source NATS cluster mTLS identity from cluster host Derive the leaf IP SAN and accept-side binding from the replica's ClusterHost (the CODER_CLUSTER_HOST argument) fixed at construction, replacing the DERP-relay-URL source, and rename Pubsub.SetClusterCA to SetCACache now that it only swaps the cache. --- cli/server.go | 8 +++-- coderd/x/nats/cluster.go | 17 +++++----- coderd/x/nats/pubsub.go | 18 +++++------ coderd/x/nats/tls.go | 25 ++++++++------- coderd/x/nats/tls_internal_test.go | 51 ++++++++++++++++++++---------- enterprise/coderd/coderd.go | 27 ++++++---------- 6 files changed, 81 insertions(+), 65 deletions(-) diff --git a/cli/server.go b/cli/server.go index 7886c4fdd86..323913821b8 100644 --- a/cli/server.go +++ b/cli/server.go @@ -847,11 +847,15 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. token := fmt.Sprintf("%x", sha256.Sum256([]byte(dbURL))) natsps, err := nats.New(ctx, logger.Named("nats_pubsub"), nats.Options{ ClusterAuthToken: token, + // ClusterHost is this replica's routable cluster address. It + // is the NATS route listener host and, when it is an IP, the + // leaf certificate's IP SAN for cluster mTLS. + ClusterHost: options.DeploymentValues.Cluster.Host.String(), // Install the cluster TLS callbacks with a noop CA cache so a // single node (or pre-license deployment) boots without a CA // dependency and forms no routes. Enterprise HA swaps in the - // real nats_ca cache plus the relay IP via Pubsub.SetClusterCA - // once clustering is licensed. + // real nats_ca cache via Pubsub.SetCACache once clustering is + // licensed. // // TODO: the real CA cache cannot be built here because // options.Database is not yet fully instantiated (it is diff --git a/coderd/x/nats/cluster.go b/coderd/x/nats/cluster.go index 5cc73792728..3126ab031cf 100644 --- a/coderd/x/nats/cluster.go +++ b/coderd/x/nats/cluster.go @@ -50,17 +50,18 @@ func (p *Pubsub) SetPeerFetcher(fetcher PeerFetcher) { p.RefreshPeers() } -// SetClusterCA swaps the cluster mTLS CA cache and this replica's leaf IP SAN, -// then triggers a peer refresh so any route blocked by the previous (for -// example noop) cache is retried. It is a no-op unless the pubsub was started -// with cluster TLS enabled (Options.ClusterCA set, which installs the TLS -// callbacks). Passing a noop cache reverts to no mTLS: new route handshakes -// can no longer mint a leaf and will not form. -func (p *Pubsub) SetClusterCA(ca cryptokeys.SigningKeycache, ip net.IP) { +// SetCACache swaps the cluster mTLS CA cache, then triggers a peer refresh so +// any route blocked by the previous (for example noop) cache is retried. It is +// a no-op unless the pubsub was started with cluster TLS enabled +// (Options.ClusterCA set, which installs the TLS callbacks). Passing a noop +// cache reverts to no mTLS: new route handshakes can no longer mint a leaf and +// will not form. The leaf IP SAN is fixed at construction from ClusterHost, so +// it is not passed here. +func (p *Pubsub) SetCACache(ca cryptokeys.SigningKeycache) { if p.clusterTLS == nil { return } - p.clusterTLS.setClusterCA(ca, ip) + p.clusterTLS.setCACache(ca) p.RefreshPeers() } diff --git a/coderd/x/nats/pubsub.go b/coderd/x/nats/pubsub.go index 7cd4913fbdc..88dff3afbd3 100644 --- a/coderd/x/nats/pubsub.go +++ b/coderd/x/nats/pubsub.go @@ -130,13 +130,11 @@ type Options struct { // certificate from the active nats_ca CA and verifies peers against the // CA fetched from this cache on each handshake. Nil keeps routes // plaintext (token auth only). cryptokeys.SigningKeycache satisfies this. + // + // The leaf's IP SAN (and the accept-side source binding) is this replica's + // ClusterHost, so ClusterHost must be an IP for mTLS to activate. ClusterCA cryptokeys.SigningKeycache - // ClusterTLSIP is this replica's relay IP, embedded as an IP SAN in the - // leaf certificate and matched against the dialed host when verifying a - // peer. Required when ClusterCA is set. - ClusterTLSIP net.IP - // clusterTLSClock overrides the cluster TLS clock, for tests. clusterTLSClock quartz.Clock @@ -333,12 +331,14 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub, // When ClusterCA is set, install the cluster TLS callbacks at boot so the // route listener can negotiate mTLS. The callbacks read the CA cache on // each handshake, so the default noop cache keeps routes inert (no leaf can - // be minted) until SetClusterCA swaps in a real cache. A real cache also - // requires ClusterTLSIP; leaf minting enforces that. ClusterCA == nil keeps - // routes plaintext (token auth only). + // be minted) until SetCACache swaps in a real cache. The leaf IP SAN is this + // replica's ClusterHost, fixed here at construction; leaf minting enforces + // that it is an IP. ClusterCA == nil keeps routes plaintext (token auth + // only). var ct *clusterTLS if !opts.disableCluster && opts.ClusterCA != nil { - ct = newClusterTLS(ctx, logger, opts.clusterTLSClock, opts.ClusterCA, opts.ClusterTLSIP) + selfIP := net.ParseIP(opts.ClusterHost) + ct = newClusterTLS(ctx, logger, opts.clusterTLSClock, opts.ClusterCA, selfIP) sopts.Cluster.TLSConfig = ct.tlsConfig() sopts.Cluster.TLSTimeout = clusterTLSTimeout.Seconds() } diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go index f199c9a51fe..a91dd8137bb 100644 --- a/coderd/x/nats/tls.go +++ b/coderd/x/nats/tls.go @@ -42,9 +42,10 @@ type clusterTLS struct { clock quartz.Clock mu sync.Mutex - // ca and ip are swapped together by setClusterCA: under the default noop - // cache no leaf can be minted (so no route forms), and the real cache plus - // this replica's relay IP are installed once cluster mTLS is enabled. + // ca is swapped by setCACache: the default noop cache mints no leaf (so no + // route forms) until the real cache is installed once cluster mTLS is + // enabled. ip is this replica's cluster host, fixed at construction and + // embedded as the leaf IP SAN. ca cryptokeys.SigningKeycache ip net.IP // leaf is the cached leaf certificate. leafSeq is the active CA sequence it @@ -85,17 +86,17 @@ func newClusterTLS(ctx context.Context, logger slog.Logger, clock quartz.Clock, } } -// setClusterCA swaps the CA cache and this replica's leaf IP SAN. Because the -// tls.Config callbacks read these on each handshake, the swap takes effect -// without a server restart or route reload: installing the real cache lets -// routes negotiate mTLS, and reverting to a noop cache makes leaf minting fail -// so no new route can form. A swap clears the cached leaf so the next handshake -// re-mints under the new CA/IP. -func (t *clusterTLS) setClusterCA(ca cryptokeys.SigningKeycache, ip net.IP) { +// setCACache swaps the CA cache. Because the tls.Config callbacks read it on +// each handshake, the swap takes effect without a server restart or route +// reload: installing the real cache lets routes negotiate mTLS, and reverting +// to a noop cache makes leaf minting fail so no new route can form. The leaf IP +// SAN is fixed at construction (this replica's cluster host does not change), so +// it is not touched here. A swap clears the cached leaf so the next handshake +// re-mints under the new CA. +func (t *clusterTLS) setCACache(ca cryptokeys.SigningKeycache) { t.mu.Lock() defer t.mu.Unlock() t.ca = ca - t.ip = ip t.leaf = nil t.leafSeq = "" // Verify pools follow the CA source: drop them so stale roots are not @@ -200,7 +201,7 @@ func (t *clusterTLS) configForClient(chi *tls.ClientHelloInfo) (*tls.Config, err // signing CA (see mintLeaf), so re-minting is driven purely by CA rotation. // // The whole method holds t.mu so the CA cache, IP, and cached leaf are read as -// a consistent set: a concurrent setClusterCA cannot swap the CA out from under +// a consistent set: a concurrent setCACache cannot swap the CA out from under // the IP we mint with. The lock is therefore held across the SigningKey lookup // and the (rare) mint. Mints happen only at startup and on CA rotation, so the // keygen+sign cost on the lock is acceptable; the SigningKey lookup is normally diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go index 4ae30fe676b..c0e3b487943 100644 --- a/coderd/x/nats/tls_internal_test.go +++ b/coderd/x/nats/tls_internal_test.go @@ -77,24 +77,36 @@ func generateTestCA(t *testing.T, sequence int32) *cryptokeys.NATSCA { } // newTLSPubsub builds a clustered pubsub whose route listener requires mTLS, -// using the supplied CA cache and leaf IP SAN. Peers dial each other on -// 127.0.0.1 (clusterRouteAddress), so ip must be 127.0.0.1 for routes to form. +// using the supplied CA cache. ip is this node's cluster host: the route +// listener bind host and the leaf IP SAN. Peers dial each other on that host +// (clusterRouteAddress), so ip must be 127.0.0.1 for routes to form. func newTLSPubsub(t *testing.T, ca cryptokeys.SigningKeycache, ip net.IP) *Pubsub { t.Helper() logger := slogtest.Make(t, nil) ctx := testutil.Context(t, testutil.WaitLong) ps, err := New(ctx, logger, Options{ - ClusterHost: "127.0.0.1", + ClusterHost: ip.String(), ClusterPort: natsserver.RANDOM_PORT, disableCluster: false, ClusterCA: ca, - ClusterTLSIP: ip, }) require.NoError(t, err) t.Cleanup(func() { _ = ps.Close() }) return ps } +// setLeafSAN overrides a node's leaf IP SAN after construction, for tests that +// need the minted SAN to differ from the loopback address the node binds and +// connects on (which are otherwise both the node's ClusterHost). Clearing the +// cached leaf forces the next handshake to re-mint under the new SAN. +func setLeafSAN(ps *Pubsub, ip net.IP) { + ps.clusterTLS.mu.Lock() + defer ps.clusterTLS.mu.Unlock() + ps.clusterTLS.ip = ip + ps.clusterTLS.leaf = nil + ps.clusterTLS.leafSeq = "" +} + func numRoutes(t *testing.T, ps *Pubsub) int { t.Helper() routes, err := ps.Server.Routez(&natsserver.RoutezOptions{}) @@ -196,14 +208,19 @@ func TestPubsub_ClusterTLS(t *testing.T) { cache := func() *fakeCACache { return &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} } - // b mints its leaf for the wrong IP (not the loopback it actually - // connects from). When b dials a, a's accept-side source binding sees - // b's source IP (127.0.0.1) does not match b's leaf SAN (10.99.99.99) - // and rejects it, so no route forms even though both share a CA. Only b - // dials, so there is no other handshake direction. + // Both nodes bind and connect on loopback and know each other as peers, + // so the CA and source-membership checks pass. But both mint their leaf + // with a SAN that does not match the loopback address they connect from, + // so every handshake is rejected on the SAN binding alone and no route + // forms. This isolates the SAN check: a valid CA-signed leaf presented + // from a known replica is still rejected when the cert is not bound to + // the address it connects from (e.g. a stolen or mis-minted leaf). a := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) - b := newTLSPubsub(t, cache(), net.IPv4(10, 99, 99, 99)) + b := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + setLeafSAN(a, net.IPv4(10, 99, 99, 99)) + setLeafSAN(b, net.IPv4(10, 99, 99, 99)) + require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)})) require.Never(t, func() bool { @@ -232,7 +249,7 @@ func TestPubsub_ClusterTLS(t *testing.T) { // TestPubsub_ClusterTLS_CacheSwap covers the Part C optional-mTLS model: a node // that boots with the noop CA cache forms no route, and swapping in a real cache -// via SetClusterCA lets routes form over mTLS with no server restart. +// via SetCACache lets routes form over mTLS with no server restart. func TestPubsub_ClusterTLS_CacheSwap(t *testing.T) { t.Parallel() @@ -242,7 +259,7 @@ func TestPubsub_ClusterTLS_CacheSwap(t *testing.T) { ca := generateTestCA(t, 1) // a boots with the noop cache (production default); b has a real cache. // a cannot mint a leaf, so its route handshakes fail and no route forms. - a := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, nil) + a := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, net.IPv4(127, 0, 0, 1)) b := newTLSPubsub(t, &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}, net.IPv4(127, 0, 0, 1)) require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) @@ -262,15 +279,15 @@ func TestPubsub_ClusterTLS_CacheSwap(t *testing.T) { } // Both boot with the noop cache, then both get the real cache swapped in // (mirroring the enterprise HA enable path) without a server restart. - a := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, nil) - b := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, nil) + a := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, net.IPv4(127, 0, 0, 1)) // Drive peers through fetchers, as production does, rather than calling - // setPeerAddresses directly: SetClusterCA and SetPeerFetcher both trigger + // setPeerAddresses directly: SetCACache and SetPeerFetcher both trigger // a peer refresh that reads the current fetcher, so routes converge on // the fetcher's addresses without racing a manual call. - a.SetClusterCA(realCache(), net.IPv4(127, 0, 0, 1)) - b.SetClusterCA(realCache(), net.IPv4(127, 0, 0, 1)) + a.SetCACache(realCache()) + b.SetCACache(realCache()) a.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, b)}}) b.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, a)}}) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 58e445ff42e..ceab87864c6 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -883,25 +883,18 @@ func (api *API) Close() error { return api.AGPL.Close() } -// configureNATSClusterTLS swaps the real nats_ca CA cache and this replica's -// relay IP into the NATS pubsub, enabling cluster mTLS. The relay URL host is -// the IP SAN peers verify, so without an IP-based relay URL the cache is left -// as the boot-time noop and routes stay plaintext (token auth only). The CA is +// configureNATSClusterTLS swaps the real nats_ca CA cache into the NATS pubsub, +// enabling cluster mTLS. The leaf IP SAN peers verify is this replica's cluster +// host, fixed when the pubsub was constructed, so without an IP cluster host no +// leaf can be minted and routes stay plaintext (token auth only). The CA is // read lazily by the TLS callbacks on each handshake, so nothing reads it here. func (api *API) configureNATSClusterTLS(natsPubsub *nats.Pubsub) { - relayURL := api.Options.DeploymentValues.DERP.Server.RelayURL.Value() - if relayURL == nil || relayURL.String() == "" { - api.Logger.Warn(api.ctx, "nats cluster mTLS disabled: no DERP relay URL configured; cluster routes use token auth only") - return - } - ip := net.ParseIP(relayURL.Hostname()) - if ip == nil { - api.Logger.Warn(api.ctx, "nats cluster mTLS disabled: relay URL host is not an IP", - slog.F("relay_host", relayURL.Hostname())) - return + if net.ParseIP(api.Options.ClusterHost) == nil { + api.Logger.Warn(api.ctx, "nats cluster mTLS inactive: cluster host is not an IP; cluster routes use token auth only", + slog.F("cluster_host", api.Options.ClusterHost)) } - natsPubsub.SetClusterCA(api.AGPL.NATSCACache, ip) - api.Logger.Info(api.ctx, "nats cluster mTLS enabled", slog.F("leaf_ip", ip.String())) + natsPubsub.SetCACache(api.AGPL.NATSCACache) + api.Logger.Info(api.ctx, "nats cluster mTLS enabled") } func (api *API) updateEntitlements(ctx context.Context) error { @@ -1068,7 +1061,7 @@ func (api *API) updateEntitlements(ctx context.Context) error { natsPubsub.SetPeerFetcher(nats.NopPeerFetcher{}) // Revert to the noop CA cache: new route handshakes can no // longer mint a leaf, so the cluster mesh stops forming. - natsPubsub.SetClusterCA(cryptokeys.NoopSigningKeycache{}, nil) + natsPubsub.SetCACache(cryptokeys.NoopSigningKeycache{}) api.Logger.Info(api.ctx, "nats cluster mTLS disabled") api.replicaManager.SetCallback("nats", nil) } From 8f49739f6c8a08658afd339e3255b571920348ce Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 9 Jul 2026 01:09:47 +0000 Subject: [PATCH 08/16] refactor(coderd/x/nats): check cached cluster leaf before minting Reuse the cached leaf while it is still within its validity window before consulting the signing cache, dropping the now-redundant leafSeq field. Keep verify and currentLeaf as pure functions returning wrapped errors, and log at the tls.Config callback sites where the embedded NATS server would otherwise swallow them. --- coderd/x/nats/tls.go | 120 +++++++++++++++-------------- coderd/x/nats/tls_internal_test.go | 1 - 2 files changed, 62 insertions(+), 59 deletions(-) diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go index a91dd8137bb..bd288ec4434 100644 --- a/coderd/x/nats/tls.go +++ b/coderd/x/nats/tls.go @@ -48,10 +48,9 @@ type clusterTLS struct { // embedded as the leaf IP SAN. ca cryptokeys.SigningKeycache ip net.IP - // leaf is the cached leaf certificate. leafSeq is the active CA sequence it - // was minted under; a change means the CA rotated and the leaf is stale. - leaf *tls.Certificate - leafSeq string + // leaf is the cached leaf certificate, reused until it expires (its NotAfter + // equals the signing CA's) or setCACache clears it on a cache swap. + leaf *tls.Certificate // verifyPools caches the root pool used to verify a peer leaf, keyed by the // CA sequence stamped in the leaf. A CA cert is immutable for a given // sequence, so the pool is built once and reused across handshakes. Expired @@ -98,7 +97,6 @@ func (t *clusterTLS) setCACache(ca cryptokeys.SigningKeycache) { defer t.mu.Unlock() t.ca = ca t.leaf = nil - t.leafSeq = "" // Verify pools follow the CA source: drop them so stale roots are not // reused after a swap to a noop or different CA. t.verifyPools = nil @@ -133,43 +131,37 @@ func (t *clusterTLS) caCache() cryptokeys.SigningKeycache { // base VerifyConnection: chain + membership against the known peer set. func (t *clusterTLS) tlsConfig() *tls.Config { return &tls.Config{ - MinVersion: tls.VersionTLS13, - GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return t.currentLeafLogged() }, - GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return t.currentLeafLogged() }, - ClientAuth: tls.RequireAnyClientCert, - //nolint:gosec // Not insecure: verifyConnection performs full chain - // verification against the live CA cache. Go's static RootCAs cannot - // track a rotating CA, so default verification is replaced, not removed. + MinVersion: tls.VersionTLS13, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + leaf, err := t.currentLeaf() + if err != nil { + t.logger.Warn(t.ctx, "get nats cluster leaf for GetCertificate", slog.Error(err)) + } + return leaf, err + }, + GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + leaf, err := t.currentLeaf() + if err != nil { + t.logger.Warn(t.ctx, "get nats cluster leaf for GetClientCertificate", slog.Error(err)) + } + return leaf, err + }, + ClientAuth: tls.RequireAnyClientCert, + //nolint:gosec // Not insecure: verify performs full chain verification + // against the live CA cache. Go's static RootCAs cannot track a rotating + // CA, so default verification is replaced, not removed. InsecureSkipVerify: true, - VerifyConnection: func(cs tls.ConnectionState) error { return t.verifyLogged(cs, nil) }, + VerifyConnection: func(cs tls.ConnectionState) error { + err := t.verify(cs, nil) + if err != nil { + t.logger.Warn(t.ctx, "verify nats cluster peer for VerifyConnection", slog.Error(err)) + } + return err + }, GetConfigForClient: t.configForClient, } } -// verifyLogged runs verify and debug-logs a rejection. The embedded NATS server -// runs with NoLog set, so it swallows its own TLS handshake errors; logging here -// gives deployments a way to see why a cluster peer was rejected. -func (t *clusterTLS) verifyLogged(cs tls.ConnectionState, sourceIP net.IP) error { - err := t.verify(cs, sourceIP) - if err != nil { - t.logger.Debug(t.ctx, "rejected nats cluster peer certificate", slog.Error(err)) - } - return err -} - -// currentLeafLogged mints (or returns the cached) leaf and debug-logs a -// failure. Like verifyLogged, this exists because the embedded NATS server runs -// with NoLog: a currentLeaf error (CA cache error, wrong key type, mint failure) -// is otherwise swallowed by the TLS stack, so a broken CA cache produces zero -// routes with no diagnostic. Used by the GetCertificate callbacks. -func (t *clusterTLS) currentLeafLogged() (*tls.Certificate, error) { - leaf, err := t.currentLeaf() - if err != nil { - t.logger.Debug(t.ctx, "failed to mint nats cluster leaf", slog.Error(err)) - } - return leaf, err -} - // configForClient builds the per-connection config used when accepting a route. // It captures the dialing peer's source IP from the underlying connection so // VerifyConnection can require the peer leaf's IP SAN to match it. NATS calls @@ -185,31 +177,52 @@ func (t *clusterTLS) configForClient(chi *tls.ClientHelloInfo) (*tls.Config, err } } cfg := &tls.Config{ - MinVersion: tls.VersionTLS13, - GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return t.currentLeafLogged() }, - ClientAuth: tls.RequireAnyClientCert, + MinVersion: tls.VersionTLS13, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + leaf, err := t.currentLeaf() + if err != nil { + t.logger.Warn(t.ctx, "get nats cluster leaf for GetCertificate", slog.Error(err)) + } + return leaf, err + }, + ClientAuth: tls.RequireAnyClientCert, //nolint:gosec // See tlsConfig: verification is performed in VerifyConnection. InsecureSkipVerify: true, - VerifyConnection: func(cs tls.ConnectionState) error { return t.verifyLogged(cs, sourceIP) }, + VerifyConnection: func(cs tls.ConnectionState) error { + err := t.verify(cs, sourceIP) + if err != nil { + t.logger.Warn(t.ctx, "verify nats cluster peer for VerifyConnection", slog.Error(err)) + } + return err + }, } return cfg, nil } // currentLeaf returns the cached leaf, re-minting it when it is missing or -// signed by a CA that is no longer the active, still-valid one (a rotation). -// A leaf carries no independent lifetime: it is valid exactly as long as its -// signing CA (see mintLeaf), so re-minting is driven purely by CA rotation. +// expired. A leaf carries no independent lifetime: its NotAfter equals its +// signing CA's (see mintLeaf), so re-minting is driven purely by CA rotation. // // The whole method holds t.mu so the CA cache, IP, and cached leaf are read as // a consistent set: a concurrent setCACache cannot swap the CA out from under -// the IP we mint with. The lock is therefore held across the SigningKey lookup -// and the (rare) mint. Mints happen only at startup and on CA rotation, so the -// keygen+sign cost on the lock is acceptable; the SigningKey lookup is normally -// an in-memory cache hit. +// the IP we mint with. The lock is held across the SigningKey lookup and the +// (rare) mint; both are cheap (an in-memory cache hit and, only on a miss, a +// keygen+sign). func (t *clusterTLS) currentLeaf() (*tls.Certificate, error) { t.mu.Lock() defer t.mu.Unlock() + // Reuse the cached leaf while it is still within its validity window, + // before consulting the signing cache. A leaf's NotAfter equals its signing + // CA's, and the previous CA stays trusted by peers through the rotation + // overlap, so a still-valid cached leaf always chains to a CA peers accept. + // A new CA is picked up when the leaf expires (forcing a re-mint) or when + // setCACache swaps the cache and clears the leaf. + now := t.clock.Now() + if t.leaf != nil && now.Before(t.leaf.Leaf.NotAfter) { + return t.leaf, nil + } + id, key, err := t.ca.SigningKey(t.ctx) if err != nil { return nil, xerrors.Errorf("get signing CA: %w", err) @@ -219,20 +232,11 @@ func (t *clusterTLS) currentLeaf() (*tls.Certificate, error) { return nil, xerrors.Errorf("unexpected signing key type %T", key) } - now := t.clock.Now() - // Reuse the cached leaf while it was signed by the still-active, still-valid - // CA. A CA rotation (sequence change) or the active CA expiring forces a - // re-mint; there is no separate leaf lifetime to track. - if t.leaf != nil && t.leafSeq == id && now.Before(ca.Cert.NotAfter) { - return t.leaf, nil - } - leaf, err := mintLeaf(ca, t.ip, now) if err != nil { - return nil, err + return nil, xerrors.Errorf("mint leaf: %w", err) } t.leaf = leaf - t.leafSeq = id t.logger.Debug(t.ctx, "minted nats cluster leaf", slog.F("ca_sequence", id)) return leaf, nil } diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go index c0e3b487943..2e1645568bc 100644 --- a/coderd/x/nats/tls_internal_test.go +++ b/coderd/x/nats/tls_internal_test.go @@ -104,7 +104,6 @@ func setLeafSAN(ps *Pubsub, ip net.IP) { defer ps.clusterTLS.mu.Unlock() ps.clusterTLS.ip = ip ps.clusterTLS.leaf = nil - ps.clusterTLS.leafSeq = "" } func numRoutes(t *testing.T, ps *Pubsub) int { From 980f3cdae7dd1e3fe6ab94ec42cae09355b6d7f4 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 9 Jul 2026 07:01:24 +0000 Subject: [PATCH 09/16] test(coderd/x/nats): drive cluster TLS route tests through peer fetchers The startup peer refresh runs once with the boot-time noop fetcher and can race a manual setPeerAddresses call, wiping the route and known-peer set; with fail-closed membership that made the route-forming tests flaky, so drive their peers through fetchers as production does. --- coderd/x/nats/tls_internal_test.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go index 2e1645568bc..d540dcb9d10 100644 --- a/coderd/x/nats/tls_internal_test.go +++ b/coderd/x/nats/tls_internal_test.go @@ -136,9 +136,13 @@ func TestPubsub_ClusterTLS(t *testing.T) { addrC := clusterRouteAddress(t, c) // Full symmetric mesh: every node must know a peer to accept a route // from it (accept-side membership), so each is given the other two. - require.NoError(t, a.setPeerAddresses([]string{addrB, addrC})) - require.NoError(t, b.setPeerAddresses([]string{addrA, addrC})) - require.NoError(t, c.setPeerAddresses([]string{addrA, addrB})) + // Drive peers through fetchers, as production does: a fetcher re-applies + // the same peers on every refresh, so the startup refresh (which runs + // with the boot-time noop fetcher) cannot race a manual call and wipe + // the route/known-peer set. + a.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrB, addrC}}) + b.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrC}}) + c.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrB}}) event := "tls-mesh" got := make(chan []byte, 8) @@ -191,9 +195,11 @@ func TestPubsub_ClusterTLS(t *testing.T) { a := newTLSPubsub(t, &fakeCACache{active: ca1, byID: bundle}, net.IPv4(127, 0, 0, 1)) b := newTLSPubsub(t, &fakeCACache{active: ca2, byID: bundle}, net.IPv4(127, 0, 0, 1)) - // Symmetric peers so each side accepts a route from the other. - require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) - require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)})) + // Symmetric peers so each side accepts a route from the other, driven + // through fetchers (see Mesh) so the startup noop refresh cannot race a + // manual call and wipe the route/known-peer set. + a.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, b)}}) + b.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, a)}}) require.Eventually(t, func() bool { return numRoutes(t, a) > 0 && numRoutes(t, b) > 0 @@ -527,9 +533,11 @@ func TestPubsub_ClusterTLS_RealCA(t *testing.T) { addrA := clusterRouteAddress(t, a) addrB := clusterRouteAddress(t, b) addrC := clusterRouteAddress(t, c) - require.NoError(t, a.setPeerAddresses([]string{addrB, addrC})) - require.NoError(t, b.setPeerAddresses([]string{addrA, addrC})) - require.NoError(t, c.setPeerAddresses([]string{addrA, addrB})) + // Drive peers through fetchers, as production does, so the startup noop + // refresh cannot race a manual call and wipe the route/known-peer set. + a.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrB, addrC}}) + b.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrC}}) + c.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrB}}) received := make(chan string, 4) cancelSub, err := c.Subscribe("tls-realca", func(_ context.Context, msg []byte) { From 7e00f75baa3df231a87dd7c2c2592f64dd3ed6a8 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 9 Jul 2026 16:25:28 +0000 Subject: [PATCH 10/16] refactor(coderd/x/nats): rename Options.clusterTLSClock to clock It is the only clock field on Options, so the shorter name is unambiguous. --- coderd/x/nats/pubsub.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coderd/x/nats/pubsub.go b/coderd/x/nats/pubsub.go index 88dff3afbd3..5f575d0df4e 100644 --- a/coderd/x/nats/pubsub.go +++ b/coderd/x/nats/pubsub.go @@ -135,8 +135,8 @@ type Options struct { // ClusterHost, so ClusterHost must be an IP for mTLS to activate. ClusterCA cryptokeys.SigningKeycache - // clusterTLSClock overrides the cluster TLS clock, for tests. - clusterTLSClock quartz.Clock + // clock overrides the cluster TLS clock, for tests. + clock quartz.Clock // PeerFetcher provides the current set of peer route addresses. // RefreshPeers uses it to update the configured cluster routes. @@ -338,7 +338,7 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub, var ct *clusterTLS if !opts.disableCluster && opts.ClusterCA != nil { selfIP := net.ParseIP(opts.ClusterHost) - ct = newClusterTLS(ctx, logger, opts.clusterTLSClock, opts.ClusterCA, selfIP) + ct = newClusterTLS(ctx, logger, opts.clock, opts.ClusterCA, selfIP) sopts.Cluster.TLSConfig = ct.tlsConfig() sopts.Cluster.TLSTimeout = clusterTLSTimeout.Seconds() } From 60f988b2b5b1be6f5e70b5d38f5872cc41bac423 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 9 Jul 2026 16:51:36 +0000 Subject: [PATCH 11/16] refactor(enterprise/coderd): set nats CA cache inline like the peer fetcher Drop configureNATSClusterTLS so SetCACache sits beside SetPeerFetcher in the HA enable block, moving only the mTLS-status logging into a small helper. --- enterprise/coderd/coderd.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index ceab87864c6..82435253447 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -883,17 +883,16 @@ func (api *API) Close() error { return api.AGPL.Close() } -// configureNATSClusterTLS swaps the real nats_ca CA cache into the NATS pubsub, -// enabling cluster mTLS. The leaf IP SAN peers verify is this replica's cluster -// host, fixed when the pubsub was constructed, so without an IP cluster host no -// leaf can be minted and routes stay plaintext (token auth only). The CA is -// read lazily by the TLS callbacks on each handshake, so nothing reads it here. -func (api *API) configureNATSClusterTLS(natsPubsub *nats.Pubsub) { +// logNATSClusterMTLS logs whether cluster mTLS is active. The leaf IP SAN peers +// verify is this replica's cluster host, fixed when the pubsub was constructed, +// so if it is not an IP no leaf can be minted and routes stay plaintext (token +// auth only). +func (api *API) logNATSClusterMTLS() { if net.ParseIP(api.Options.ClusterHost) == nil { api.Logger.Warn(api.ctx, "nats cluster mTLS inactive: cluster host is not an IP; cluster routes use token auth only", slog.F("cluster_host", api.Options.ClusterHost)) + return } - natsPubsub.SetCACache(api.AGPL.NATSCACache) api.Logger.Info(api.ctx, "nats cluster mTLS enabled") } @@ -1024,11 +1023,12 @@ func (api *API) updateEntitlements(ctx context.Context) error { } if natsPubsub, ok := api.Pubsub.(*nats.Pubsub); ok { - // Swap the real nats_ca CA cache in before peers are known - // so the first route handshake can negotiate mTLS. - api.configureNATSClusterTLS(natsPubsub) + // Swap the real nats_ca CA cache and the replica peer fetcher + // in so the first route handshake can negotiate mTLS. + natsPubsub.SetCACache(api.AGPL.NATSCACache) natsPubsub.SetPeerFetcher(api.replicaManager) api.replicaManager.SetCallback("nats", natsPubsub.RefreshPeers) + api.logNATSClusterMTLS() } api.replicaManager.SetCallback("derp", func() { From 382e51860643df13a109bcbeb406e24cb8cf57f0 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 9 Jul 2026 17:01:09 +0000 Subject: [PATCH 12/16] refactor(coderd/x/nats): only relax cluster TLS handshake timeout in tests Production keeps the NATS default (2s); the longer 10s timeout that avoids flaky handshakes under load and in CI is now set through a test-only Options field. --- coderd/x/nats/pubsub.go | 11 ++++++++++- coderd/x/nats/pubsub_internal_test.go | 14 ++++++++++---- coderd/x/nats/tls.go | 3 --- coderd/x/nats/tls_internal_test.go | 9 +++++---- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/coderd/x/nats/pubsub.go b/coderd/x/nats/pubsub.go index 5f575d0df4e..767de025cad 100644 --- a/coderd/x/nats/pubsub.go +++ b/coderd/x/nats/pubsub.go @@ -138,6 +138,11 @@ type Options struct { // clock overrides the cluster TLS clock, for tests. clock quartz.Clock + // clusterTLSTimeout overrides the cluster route TLS handshake timeout, for + // tests. Zero leaves the NATS default (2s). Tests use a longer timeout + // because handshakes are flaky under load and in CI. + clusterTLSTimeout time.Duration + // PeerFetcher provides the current set of peer route addresses. // RefreshPeers uses it to update the configured cluster routes. PeerFetcher PeerFetcher @@ -340,7 +345,11 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub, selfIP := net.ParseIP(opts.ClusterHost) ct = newClusterTLS(ctx, logger, opts.clock, opts.ClusterCA, selfIP) sopts.Cluster.TLSConfig = ct.tlsConfig() - sopts.Cluster.TLSTimeout = clusterTLSTimeout.Seconds() + // Leave TLSTimeout unset (NATS defaults to 2s) unless a test overrides + // it; the default has not shown a need to change in production. + if opts.clusterTLSTimeout > 0 { + sopts.Cluster.TLSTimeout = opts.clusterTLSTimeout.Seconds() + } } ns, err := startEmbeddedServer(sopts) diff --git a/coderd/x/nats/pubsub_internal_test.go b/coderd/x/nats/pubsub_internal_test.go index 4dcdf1abad3..cfdb07e6ff2 100644 --- a/coderd/x/nats/pubsub_internal_test.go +++ b/coderd/x/nats/pubsub_internal_test.go @@ -538,13 +538,19 @@ func defaultTestOptions() Options { return Options{disableCluster: true} } +// testClusterTLSTimeout relaxes the cluster route TLS handshake timeout in +// tests. NATS defaults to a tight 2s, which is flaky under load and in CI; +// production keeps the default until it is shown to need changing. +const testClusterTLSTimeout = 10 * time.Second + func clusterTestOptions(t *testing.T) Options { t.Helper() return Options{ - ClusterHost: "127.0.0.1", - ClusterPort: natsserver.RANDOM_PORT, - disableCluster: false, - ClusterAuthToken: fmt.Sprintf("shared-token-%d", time.Now().UnixNano()), + ClusterHost: "127.0.0.1", + ClusterPort: natsserver.RANDOM_PORT, + disableCluster: false, + ClusterAuthToken: fmt.Sprintf("shared-token-%d", time.Now().UnixNano()), + clusterTLSTimeout: testClusterTLSTimeout, } } diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go index bd288ec4434..7af6da7f602 100644 --- a/coderd/x/nats/tls.go +++ b/coderd/x/nats/tls.go @@ -23,9 +23,6 @@ import ( ) const ( - // clusterTLSTimeout is the route TLS handshake timeout. NATS defaults to a - // tight 2s, which is flaky under load and in CI. - clusterTLSTimeout = 10 * time.Second // leafSerialBits is the entropy of a leaf certificate serial number. leafSerialBits = 128 // clockSkewToleranceTLS backdates a leaf's NotBefore so a peer with a diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go index d540dcb9d10..4ca4c7c9098 100644 --- a/coderd/x/nats/tls_internal_test.go +++ b/coderd/x/nats/tls_internal_test.go @@ -85,10 +85,11 @@ func newTLSPubsub(t *testing.T, ca cryptokeys.SigningKeycache, ip net.IP) *Pubsu logger := slogtest.Make(t, nil) ctx := testutil.Context(t, testutil.WaitLong) ps, err := New(ctx, logger, Options{ - ClusterHost: ip.String(), - ClusterPort: natsserver.RANDOM_PORT, - disableCluster: false, - ClusterCA: ca, + ClusterHost: ip.String(), + ClusterPort: natsserver.RANDOM_PORT, + disableCluster: false, + ClusterCA: ca, + clusterTLSTimeout: testClusterTLSTimeout, }) require.NoError(t, err) t.Cleanup(func() { _ = ps.Close() }) From 30617ccc61432a3c9206e49d144f4061f830f2d0 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 9 Jul 2026 18:23:45 +0000 Subject: [PATCH 13/16] refactor(coderd/x/nats): drop accept-side replica-membership check Cluster route admission now relies on CA-chain verification and the leaf IP-SAN to source-IP binding; the extra check that the source is a current replicas-table member (and its peerIP plumbing) is removed. --- coderd/x/nats/cluster.go | 37 -------------------- coderd/x/nats/pubsub.go | 9 ----- coderd/x/nats/tls.go | 21 ------------ coderd/x/nats/tls_internal_test.go | 54 ------------------------------ 4 files changed, 121 deletions(-) diff --git a/coderd/x/nats/cluster.go b/coderd/x/nats/cluster.go index 3126ab031cf..0b6895e29b8 100644 --- a/coderd/x/nats/cluster.go +++ b/coderd/x/nats/cluster.go @@ -65,39 +65,6 @@ func (p *Pubsub) SetCACache(ca cryptokeys.SigningKeycache) { p.RefreshPeers() } -// knownPeerIPs returns the IPs of the currently configured cluster routes, so -// the set a route may be accepted from is exactly the set this replica dials. -// It reads the snapshot published by setPeerAddresses without taking clusterMu, -// so the accept-side handshake path (which runs while setPeerAddresses may hold -// clusterMu across a server reload) never blocks on route reconfiguration. -func (p *Pubsub) knownPeerIPs() []net.IP { - if ips := p.peerIPs.Load(); ips != nil { - return *ips - } - return nil -} - -// routeIPs derives the accept-side peer IP set from configured routes, skipping -// non-IP hosts. It reuses the same route set setPeerAddresses applies, so the -// accepted-from set stays in lockstep with the dialed set (both from the peer -// fetcher, ultimately the replicas table). -func routeIPs(routes []*url.URL) []net.IP { - ips := make([]net.IP, 0, len(routes)) - for _, route := range routes { - if route == nil { - continue - } - host, _, err := net.SplitHostPort(route.Host) - if err != nil { - continue - } - if ip := net.ParseIP(host); ip != nil { - ips = append(ips, ip) - } - } - return ips -} - // RefreshPeers signals the peer refresh worker to fetch and apply the latest // peer route addresses. Multiple pending refreshes are coalesced. func (p *Pubsub) RefreshPeers() { @@ -166,10 +133,6 @@ func (p *Pubsub) setPeerAddresses(addresses []string) error { } p.serverOpts = newOpts.Clone() p.currentRoutes = cloneRouteURLs(routes) - // Publish the accept-side peer IP set in lockstep with the routes so the - // handshake path reads it lock-free (see knownPeerIPs). - ips := routeIPs(routes) - p.peerIPs.Store(&ips) return nil } diff --git a/coderd/x/nats/pubsub.go b/coderd/x/nats/pubsub.go index 767de025cad..14438f2ed59 100644 --- a/coderd/x/nats/pubsub.go +++ b/coderd/x/nats/pubsub.go @@ -8,7 +8,6 @@ import ( "net" "net/url" "sync" - "sync/atomic" "time" natsserver "github.com/nats-io/nats-server/v2/server" @@ -204,12 +203,7 @@ type Pubsub struct { clustered bool serverOpts *natsserver.Options currentRoutes []*url.URL - // peerIPs holds the IPs of the currently configured cluster routes, - // published by setPeerAddresses (under clusterMu) and read lock-free on the - // accept-side handshake path so verification never contends on clusterMu. - peerIPs atomic.Pointer[[]net.IP] // clusterTLS is non-nil when the cluster route listener runs mutual TLS. - // Its valid-peer-IP set is kept in sync with currentRoutes. clusterTLS *clusterTLS peerFetcher PeerFetcher @@ -382,9 +376,6 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub, p.serverOpts = sopts.Clone() p.currentRoutes = cloneRouteURLs(sopts.Routes) p.clusterTLS = ct - if ct != nil { - ct.peerIPs = p.knownPeerIPs - } handlers := p.buildConnHandlers() publishPool, err := newConnPool(ns, opts, handlers, opts.PublishConns, "coder-pubsub-pub") diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go index 7af6da7f602..5b6bd882fbb 100644 --- a/coderd/x/nats/tls.go +++ b/coderd/x/nats/tls.go @@ -53,13 +53,6 @@ type clusterTLS struct { // sequence, so the pool is built once and reused across handshakes. Expired // entries are pruned on insert to bound the map across rotations. verifyPools map[string]cachedVerifyPool - - // peerIPs returns the current set of replica cluster IPs a route may be - // accepted from. It is the same set this replica dials (the NATS peer - // fetcher, ultimately the replicas table), queried live per handshake so it - // tracks replicas joining and leaving without a cached copy. Handshakes are - // rare (cluster routes are long-lived), so a live query is cheap. - peerIPs func() []net.IP } // cachedVerifyPool is a verify root pool plus the NotAfter of the CA cert it @@ -353,20 +346,6 @@ func (t *clusterTLS) verify(cs tls.ConnectionState, sourceIP net.IP) error { if len(sourceIP) != 0 && !slices.ContainsFunc(leaf.IPAddresses, sourceIP.Equal) { return xerrors.Errorf("peer leaf IP SANs %v do not match source IP %s", leaf.IPAddresses, sourceIP) } - - // On the accept side, the source must also be a currently-known replica, so - // a valid leaf presented from an address outside the cluster is rejected. - // The replica set is the source of truth: an empty or unavailable set - // rejects, rather than falling open. - if len(sourceIP) != 0 { - var known []net.IP - if t.peerIPs != nil { - known = t.peerIPs() - } - if !slices.ContainsFunc(known, sourceIP.Equal) { - return xerrors.Errorf("source IP %s is not a known replica", sourceIP) - } - } return nil } diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go index 4ca4c7c9098..68aa4c22236 100644 --- a/coderd/x/nats/tls_internal_test.go +++ b/coderd/x/nats/tls_internal_test.go @@ -316,9 +316,6 @@ func TestClusterTLS_verify(t *testing.T) { leafIP := net.IPv4(10, 0, 0, 5) ct := newClusterTLS(ctx, slogtest.Make(t, nil), nil, cache, net.IPv4(10, 0, 0, 1)) - // leafIP is a known replica so accept-side membership passes; the source - // binding and CA checks are what these subtests exercise. - ct.peerIPs = func() []net.IP { return []net.IP{leafIP} } // A leaf bound to leafIP, signed by the trusted CA. leafCert, err := mintLeaf(ca, leafIP, time.Now()) @@ -360,57 +357,6 @@ func TestClusterTLS_verify(t *testing.T) { }) } -// TestClusterTLS_verifyReplicaMembership asserts that, on the accept side, the -// connection source IP must belong to the current peer (replica) set. A valid -// leaf with a matching SAN is still rejected when its source is not a known -// replica, and an empty set rejects rather than falling open. -func TestClusterTLS_verifyReplicaMembership(t *testing.T) { - t.Parallel() - - // newVerifier builds a fresh clusterTLS plus a valid leaf/connection state - // bound to leafIP, signed by a trusted CA, with peerIPs returning peers. - leafIP := net.IPv4(10, 0, 0, 5) - newVerifier := func(t *testing.T, peers ...net.IP) (*clusterTLS, tls.ConnectionState) { - t.Helper() - ctx := testutil.Context(t, testutil.WaitShort) - ca := generateTestCA(t, 1) - cache := &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} - ct := newClusterTLS(ctx, slogtest.Make(t, nil), nil, cache, leafIP) - ct.peerIPs = func() []net.IP { return peers } - leafCert, err := mintLeaf(ca, leafIP, time.Now()) - require.NoError(t, err) - leaf, err := x509.ParseCertificate(leafCert.Certificate[0]) - require.NoError(t, err) - return ct, tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}} - } - - t.Run("NotAKnownReplica", func(t *testing.T) { - t.Parallel() - // Peer set does NOT include leafIP: even with a valid chain and a SAN - // matching the source, a source IP that is not a known replica is - // rejected. - ct, cs := newVerifier(t, net.IPv4(10, 0, 0, 9)) - err := ct.verify(cs, leafIP) - require.ErrorContains(t, err, "not a known replica") - }) - - t.Run("KnownReplica", func(t *testing.T) { - t.Parallel() - // leafIP is in the peer set and matches the SAN + source: accepted. - ct, cs := newVerifier(t, leafIP) - require.NoError(t, ct.verify(cs, leafIP)) - }) - - t.Run("EmptySetRejects", func(t *testing.T) { - t.Parallel() - // The replica set is the source of truth: with no known peers, a route - // is rejected rather than accepted. - ct, cs := newVerifier(t) - err := ct.verify(cs, leafIP) - require.ErrorContains(t, err, "not a known replica") - }) -} - // TestClusterTLS_verifyPool asserts the verify-pool cache reuses a pool for a // given CA sequence and prunes entries whose CA cert has expired, so the map // does not grow unbounded across rotations. From 636b238e5cb49b39ee1e1fcbee060a4390f027b1 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 9 Jul 2026 19:11:10 +0000 Subject: [PATCH 14/16] refactor(coderd/x/nats): log cluster mTLS state where the CA cache is set Emit the mTLS enabled/disabled/inactive log from setCACache, the point where the state transition actually happens, instead of from the enterprise HA wiring. --- coderd/x/nats/cluster.go | 2 +- coderd/x/nats/tls.go | 22 +++++++++++++++++++++- enterprise/coderd/coderd.go | 16 ---------------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/coderd/x/nats/cluster.go b/coderd/x/nats/cluster.go index 0b6895e29b8..a2b2ca36125 100644 --- a/coderd/x/nats/cluster.go +++ b/coderd/x/nats/cluster.go @@ -56,7 +56,7 @@ func (p *Pubsub) SetPeerFetcher(fetcher PeerFetcher) { // (Options.ClusterCA set, which installs the TLS callbacks). Passing a noop // cache reverts to no mTLS: new route handshakes can no longer mint a leaf and // will not form. The leaf IP SAN is fixed at construction from ClusterHost, so -// it is not passed here. +// it is not passed here. It logs the resulting mTLS state. func (p *Pubsub) SetCACache(ca cryptokeys.SigningKeycache) { if p.clusterTLS == nil { return diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go index 5b6bd882fbb..b6d0f6acc7e 100644 --- a/coderd/x/nats/tls.go +++ b/coderd/x/nats/tls.go @@ -84,12 +84,32 @@ func newClusterTLS(ctx context.Context, logger slog.Logger, clock quartz.Clock, // re-mints under the new CA. func (t *clusterTLS) setCACache(ca cryptokeys.SigningKeycache) { t.mu.Lock() - defer t.mu.Unlock() t.ca = ca t.leaf = nil // Verify pools follow the CA source: drop them so stale roots are not // reused after a swap to a noop or different CA. t.verifyPools = nil + ip := t.ip + t.mu.Unlock() + + // Log the resulting mTLS state. A noop cache disables mTLS (no leaf can be + // minted); a real cache with a valid self IP enables it; a real cache + // without an IP cluster host leaves routes plaintext (token auth only). + switch { + case isNoopSigningCache(ca): + t.logger.Info(t.ctx, "nats cluster mTLS disabled") + case len(ip) == 0: + t.logger.Warn(t.ctx, "nats cluster mTLS inactive: cluster host is not an IP; cluster routes use token auth only") + default: + t.logger.Info(t.ctx, "nats cluster mTLS enabled") + } +} + +// isNoopSigningCache reports whether ca is the no-op cache used to disable +// cluster mTLS. +func isNoopSigningCache(ca cryptokeys.SigningKeycache) bool { + _, ok := ca.(cryptokeys.NoopSigningKeycache) + return ok } // caCache returns the current CA cache under lock so callers do not hold the diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 82435253447..66957cfbbcb 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "math" - "net" "net/http" "net/url" "strconv" @@ -883,19 +882,6 @@ func (api *API) Close() error { return api.AGPL.Close() } -// logNATSClusterMTLS logs whether cluster mTLS is active. The leaf IP SAN peers -// verify is this replica's cluster host, fixed when the pubsub was constructed, -// so if it is not an IP no leaf can be minted and routes stay plaintext (token -// auth only). -func (api *API) logNATSClusterMTLS() { - if net.ParseIP(api.Options.ClusterHost) == nil { - api.Logger.Warn(api.ctx, "nats cluster mTLS inactive: cluster host is not an IP; cluster routes use token auth only", - slog.F("cluster_host", api.Options.ClusterHost)) - return - } - api.Logger.Info(api.ctx, "nats cluster mTLS enabled") -} - func (api *API) updateEntitlements(ctx context.Context) error { return api.Entitlements.Update(ctx, func(ctx context.Context) (codersdk.Entitlements, error) { replicas := api.replicaManager.AllPrimary() @@ -1028,7 +1014,6 @@ func (api *API) updateEntitlements(ctx context.Context) error { natsPubsub.SetCACache(api.AGPL.NATSCACache) natsPubsub.SetPeerFetcher(api.replicaManager) api.replicaManager.SetCallback("nats", natsPubsub.RefreshPeers) - api.logNATSClusterMTLS() } api.replicaManager.SetCallback("derp", func() { @@ -1062,7 +1047,6 @@ func (api *API) updateEntitlements(ctx context.Context) error { // Revert to the noop CA cache: new route handshakes can no // longer mint a leaf, so the cluster mesh stops forming. natsPubsub.SetCACache(cryptokeys.NoopSigningKeycache{}) - api.Logger.Info(api.ctx, "nats cluster mTLS disabled") api.replicaManager.SetCallback("nats", nil) } } From 41215e264007fae283a2db490a2d46292bbfbfd1 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 9 Jul 2026 19:34:35 +0000 Subject: [PATCH 15/16] fix(coderd/x/nats): reject cluster route handshakes with no source IP The accept side now fails closed when it cannot determine the peer's source IP, instead of leaving it nil and silently skipping the leaf SAN to source-IP binding in verify. --- coderd/x/nats/tls.go | 37 ++++++++++++++++++++++++------ coderd/x/nats/tls_internal_test.go | 17 ++++++++++++++ 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go index b6d0f6acc7e..3ed2b92c2cd 100644 --- a/coderd/x/nats/tls.go +++ b/coderd/x/nats/tls.go @@ -178,14 +178,15 @@ func (t *clusterTLS) tlsConfig() *tls.Config { // this on each inbound handshake, so a fresh config is allocated per accepted // connection; that is fine at cluster-route cardinality (a handful of peers). func (t *clusterTLS) configForClient(chi *tls.ClientHelloInfo) (*tls.Config, error) { - var sourceIP net.IP - if chi.Conn != nil { - if remote := chi.Conn.RemoteAddr(); remote != nil { - if host, _, err := net.SplitHostPort(remote.String()); err == nil { - sourceIP = net.ParseIP(host) - } - } + // The accept side must bind the peer leaf to the address it connected from, + // so a source IP is required. Fail closed if it cannot be determined rather + // than silently skipping the binding in verify. + sourceIP, err := clientSourceIP(chi) + if err != nil { + t.logger.Warn(t.ctx, "reject nats cluster route: no source IP", slog.Error(err)) + return nil, err } + cfg := &tls.Config{ MinVersion: tls.VersionTLS13, GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -209,6 +210,28 @@ func (t *clusterTLS) configForClient(chi *tls.ClientHelloInfo) (*tls.Config, err return cfg, nil } +// clientSourceIP extracts the dialing peer's source IP from the accepted +// connection. The accept side requires it, so every failure is an error rather +// than a nil that would bypass source binding in verify. +func clientSourceIP(chi *tls.ClientHelloInfo) (net.IP, error) { + if chi.Conn == nil { + return nil, xerrors.New("no underlying connection") + } + remote := chi.Conn.RemoteAddr() + if remote == nil { + return nil, xerrors.New("no remote address") + } + host, _, err := net.SplitHostPort(remote.String()) + if err != nil { + return nil, xerrors.Errorf("split remote address %q: %w", remote.String(), err) + } + ip := net.ParseIP(host) + if ip == nil { + return nil, xerrors.Errorf("remote host %q is not an IP", host) + } + return ip, nil +} + // currentLeaf returns the cached leaf, re-minting it when it is missing or // expired. A leaf carries no independent lifetime: its NotAfter equals its // signing CA's (see mintLeaf), so re-minting is driven purely by CA rotation. diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go index 68aa4c22236..1614c5c844d 100644 --- a/coderd/x/nats/tls_internal_test.go +++ b/coderd/x/nats/tls_internal_test.go @@ -304,6 +304,23 @@ func TestPubsub_ClusterTLS_CacheSwap(t *testing.T) { }) } +// TestClusterTLS_configForClient_RequiresSourceIP asserts the accept side fails +// closed when it cannot determine the peer's source IP, rather than skipping the +// source-binding check. +func TestClusterTLS_configForClient_RequiresSourceIP(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ca := generateTestCA(t, 1) + cache := &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + ct := newClusterTLS(ctx, slogtest.Make(t, nil), nil, cache, net.IPv4(127, 0, 0, 1)) + + // No underlying connection: the source IP cannot be determined, so the + // accept-side config is refused and the handshake aborts. + _, err := ct.configForClient(&tls.ClientHelloInfo{}) + require.Error(t, err) +} + // TestClusterTLS_verify unit-tests the verifier directly, isolating chain // verification and source-IP binding that the mesh tests exercise only // indirectly. From af38308e51ca83a19e11ca2d8bde4532a9635b76 Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Thu, 9 Jul 2026 21:41:52 +0000 Subject: [PATCH 16/16] refactor(cli): resolve cluster host with DERP fallback in AGPL server Move the Cluster.Host to DERP-relay-host fallback into cli/server.go behind a new coderd.Options.ClusterHost so the NATS pubsub, leaf SAN, and replicas table all use the same resolved value instead of the enterprise-only resolution diverging from AGPL. --- cli/server.go | 19 +++++++++++++++---- coderd/coderd.go | 7 +++++++ enterprise/cli/server.go | 21 +-------------------- enterprise/coderd/coderd.go | 1 - 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/cli/server.go b/cli/server.go index 323913821b8..9093e09ccb7 100644 --- a/cli/server.go +++ b/cli/server.go @@ -730,6 +730,15 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. return xerrors.Errorf("parse real ip config: %w", err) } + // Resolve this replica's cluster host: the explicit Cluster.Host, + // else the DERP relay host for older HA deployments that predate the + // setting. Used as the NATS cluster route host and, when an IP, the + // cluster mTLS leaf IP SAN. + clusterHost := vals.Cluster.Host.String() + if clusterHost == "" { + clusterHost = vals.DERP.Server.RelayURL.Value().Hostname() + } + options := &coderd.Options{ AccessURL: vals.AccessURL.Value(), AppHostname: appHostname, @@ -737,6 +746,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. Logger: logger.Named("coderd"), Database: nil, BaseDERPMap: derpMap, + ClusterHost: clusterHost, Pubsub: nil, CacheDir: cacheDir, GoogleTokenValidator: googleTokenValidator, @@ -847,10 +857,11 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. token := fmt.Sprintf("%x", sha256.Sum256([]byte(dbURL))) natsps, err := nats.New(ctx, logger.Named("nats_pubsub"), nats.Options{ ClusterAuthToken: token, - // ClusterHost is this replica's routable cluster address. It - // is the NATS route listener host and, when it is an IP, the - // leaf certificate's IP SAN for cluster mTLS. - ClusterHost: options.DeploymentValues.Cluster.Host.String(), + // ClusterHost is this replica's routable cluster address + // (Cluster.Host, or the DERP relay host fallback resolved + // above). It is the NATS route listener host and, when it is + // an IP, the leaf certificate's IP SAN for cluster mTLS. + ClusterHost: options.ClusterHost, // Install the cluster TLS callbacks with a noop CA cache so a // single node (or pre-license deployment) boots without a CA // dependency and forms no routes. Enterprise HA swaps in the diff --git a/coderd/coderd.go b/coderd/coderd.go index 9a4674ff1e2..7eb36905249 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -205,6 +205,13 @@ type Options struct { TLSCertificates []tls.Certificate TailnetCoordinator tailnet.Coordinator DERPServer *derp.Server + // ClusterHost is this replica's routable cluster address (IP or hostname), + // resolved from DeploymentValues.Cluster.Host, falling back to the DERP + // relay host for older HA deployments that predate the setting. It is used + // as the NATS cluster route host and, when it is an IP, the cluster mTLS + // leaf IP SAN. It is consumed by the NATS pubsub (AGPL) and, under + // enterprise HA, by replicasync. + ClusterHost string // BaseDERPMap is used as the base DERP map for all clients and agents. // Proxies are added to this list. BaseDERPMap *tailcfg.DERPMap diff --git a/enterprise/cli/server.go b/enterprise/cli/server.go index 22b3ddbf18b..ec3bc52d393 100644 --- a/enterprise/cli/server.go +++ b/enterprise/cli/server.go @@ -8,7 +8,6 @@ import ( "encoding/base64" "errors" "io" - "net/url" "time" "golang.org/x/xerrors" @@ -32,28 +31,11 @@ import ( func (r *RootCmd) Server(_ func()) *serpent.Command { cmd := r.RootCmd.Server(func(ctx context.Context, options *agplcoderd.Options) (*agplcoderd.API, io.Closer, error) { - var ( - derpURL *url.URL - err error - ) - if options.DeploymentValues.DERP.Server.RelayURL.String() != "" { - derpURL, err = url.Parse(options.DeploymentValues.DERP.Server.RelayURL.String()) - if err != nil { - return nil, nil, xerrors.Errorf("derp-server-relay-address must be a valid HTTP URL: %w", err) - } - } - clusterHost := options.DeploymentValues.Cluster.Host.String() - if clusterHost == "" && derpURL != nil { - // Use the DERP host if the operator didn't specify an explicit cluster host, since this is an older setting - // and more likely to be configured by longtime HA customers. - clusterHost = derpURL.Hostname() - } - // Always generate a mesh key, even if the built-in DERP server is // disabled. This mesh key is still used by workspace proxies running // HA. var meshKey string - err = options.Database.InTx(func(tx database.Store) error { + err := options.Database.InTx(func(tx database.Store) error { // This will block until the lock is acquired, and will be // automatically released when the transaction ends. err := tx.AcquireLock(ctx, database.LockIDEnterpriseDeploymentSetup) @@ -107,7 +89,6 @@ func (r *RootCmd) Server(_ func()) *serpent.Command { SCIMAPIKey: []byte(options.DeploymentValues.SCIMAPIKey.Value()), UseLegacySCIM: options.DeploymentValues.UseLegacySCIM.Value(), RBAC: true, - ClusterHost: clusterHost, DERPServerRelayAddress: options.DeploymentValues.DERP.Server.RelayURL.String(), DERPServerRegionID: int(options.DeploymentValues.DERP.Server.RegionID.Value()), ProxyHealthInterval: options.DeploymentValues.ProxyHealthStatusInterval.Value(), diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 66957cfbbcb..28cb1475ef4 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -802,7 +802,6 @@ type Options struct { // Used for high availability. ReplicaSyncUpdateInterval time.Duration ReplicaErrorGracePeriod time.Duration - ClusterHost string // IP or hostname to reach this specific replica DERPServerRelayAddress string DERPServerRegionID int