diff --git a/cli/server.go b/cli/server.go index 21f26c1dc03b9..9093e09ccb7d9 100644 --- a/cli/server.go +++ b/cli/server.go @@ -66,6 +66,7 @@ import ( "github.com/coder/coder/v2/coderd/aibridged" "github.com/coder/coder/v2/coderd/authlink" "github.com/coder/coder/v2/coderd/autobuild" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/awsiamrds" "github.com/coder/coder/v2/coderd/database/dbauthz" @@ -729,6 +730,15 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. return xerrors.Errorf("parse real ip config: %w", err) } + // Resolve this replica's cluster host: the explicit Cluster.Host, + // else the DERP relay host for older HA deployments that predate the + // setting. Used as the NATS cluster route host and, when an IP, the + // cluster mTLS leaf IP SAN. + clusterHost := vals.Cluster.Host.String() + if clusterHost == "" { + clusterHost = vals.DERP.Server.RelayURL.Value().Hostname() + } + options := &coderd.Options{ AccessURL: vals.AccessURL.Value(), AppHostname: appHostname, @@ -736,6 +746,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. Logger: logger.Named("coderd"), Database: nil, BaseDERPMap: derpMap, + ClusterHost: clusterHost, Pubsub: nil, CacheDir: cacheDir, GoogleTokenValidator: googleTokenValidator, @@ -846,6 +857,24 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. token := fmt.Sprintf("%x", sha256.Sum256([]byte(dbURL))) natsps, err := nats.New(ctx, logger.Named("nats_pubsub"), nats.Options{ ClusterAuthToken: token, + // ClusterHost is this replica's routable cluster address + // (Cluster.Host, or the DERP relay host fallback resolved + // above). It is the NATS route listener host and, when it is + // an IP, the leaf certificate's IP SAN for cluster mTLS. + ClusterHost: options.ClusterHost, + // Install the cluster TLS callbacks with a noop CA cache so a + // single node (or pre-license deployment) boots without a CA + // dependency and forms no routes. Enterprise HA swaps in the + // real nats_ca cache via Pubsub.SetCACache once clustering is + // licensed. + // + // TODO: the real CA cache cannot be built here because + // options.Database is not yet fully instantiated (it is + // wrapped with metrics/dbauthz downstream). This split boot + // (noop here, real cache swapped in by enterprise) wants a + // refactor so the CA cache can be constructed once alongside + // the database. + ClusterCA: cryptokeys.NoopSigningKeycache{}, }) if err != nil { return xerrors.Errorf("create nats pubsub: %w", err) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index b423dca3c7d63..06dfc35b0ca59 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -18937,13 +18937,15 @@ const docTemplate = `{ "workspace_apps_api_key", "workspace_apps_token", "oidc_convert", - "tailnet_resume" + "tailnet_resume", + "nats_ca" ], "x-enum-varnames": [ "CryptoKeyFeatureWorkspaceAppsAPIKey", "CryptoKeyFeatureWorkspaceAppsToken", "CryptoKeyFeatureOIDCConvert", - "CryptoKeyFeatureTailnetResume" + "CryptoKeyFeatureTailnetResume", + "CryptoKeyFeatureNATSCA" ] }, "codersdk.CustomNotificationContent": { diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 811633d02b0c2..1c4ed8d404a89 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -17137,13 +17137,15 @@ "workspace_apps_api_key", "workspace_apps_token", "oidc_convert", - "tailnet_resume" + "tailnet_resume", + "nats_ca" ], "x-enum-varnames": [ "CryptoKeyFeatureWorkspaceAppsAPIKey", "CryptoKeyFeatureWorkspaceAppsToken", "CryptoKeyFeatureOIDCConvert", - "CryptoKeyFeatureTailnetResume" + "CryptoKeyFeatureTailnetResume", + "CryptoKeyFeatureNATSCA" ] }, "codersdk.CustomNotificationContent": { diff --git a/coderd/coderd.go b/coderd/coderd.go index f2d5e6f5df773..7eb36905249eb 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -205,6 +205,13 @@ type Options struct { TLSCertificates []tls.Certificate TailnetCoordinator tailnet.Coordinator DERPServer *derp.Server + // ClusterHost is this replica's routable cluster address (IP or hostname), + // resolved from DeploymentValues.Cluster.Host, falling back to the DERP + // relay host for older HA deployments that predate the setting. It is used + // as the NATS cluster route host and, when it is an IP, the cluster mTLS + // leaf IP SAN. It is consumed by the NATS pubsub (AGPL) and, under + // enterprise HA, by replicasync. + ClusterHost string // BaseDERPMap is used as the base DERP map for all clients and agents. // Proxies are added to this list. BaseDERPMap *tailcfg.DERPMap @@ -305,7 +312,12 @@ type Options struct { AppSigningKeyCache cryptokeys.SigningKeycache AppEncryptionKeyCache cryptokeys.EncryptionKeycache OIDCConvertKeyCache cryptokeys.SigningKeycache - Clock quartz.Clock + // NATSCACache serves the NATS cluster mTLS CA via the generic signing key + // cache for the nats_ca feature. SigningKey returns the active CA + // (a *NATSCA); VerifyingKey returns a specific CA by sequence. The key + // rotator is the sole creator of nats_ca rows, so this cache is read-only. + NATSCACache cryptokeys.SigningKeycache + Clock quartz.Clock // WebPushDispatcher is a way to send notifications over Web Push. WebPushDispatcher webpush.Dispatcher @@ -607,10 +619,34 @@ func New(options *Options) *API { updatesProvider := NewUpdatesProvider(options.Logger.Named("workspace_updates"), options.Pubsub, options.Database, options.Authorizer) + // The NATS cluster CA is only minted and served when NATS pubsub is in use. + // It is experiment-gated, so it is opted into rotation and backed by a real + // signing cache only when the experiment is enabled; otherwise the rotator + // leaves it alone and the cache is a noop, which still answers requests (the + // pubsub treats a missing CA as "mTLS off"). This avoids minting CA private + // keys on deployments that never run NATS clustering. + rotatedFeatures := cryptokeys.DefaultRotatedFeatures() + if experiments.Enabled(codersdk.ExperimentNATSPubsub) { + rotatedFeatures = append(rotatedFeatures, database.CryptoKeyFeatureNATSCA) + } + // Start a background process that rotates keys. We intentionally start this after the caches // are created to force initial requests for a key to populate the caches. This helps catch // bugs that may only occur when a key isn't precached in tests and the latency cost is minimal. - cryptokeys.StartRotator(ctx, options.Logger, options.Database) + cryptokeys.StartRotator(ctx, options.Logger, options.Database, cryptokeys.WithFeatures(rotatedFeatures)) + + // The NATS CA cache is read-only and depends on the rotator having minted + // the nats_ca CA, so it must be constructed after StartRotator. + if options.NATSCACache == nil { + if experiments.Enabled(codersdk.ExperimentNATSPubsub) { + options.NATSCACache, err = cryptokeys.NewSigningCache(ctx, options.Logger.Named("nats_ca_cache"), &cryptokeys.DBFetcher{DB: options.Database}, codersdk.CryptoKeyFeatureNATSCA) + if err != nil { + options.Logger.Fatal(ctx, "failed to instantiate NATS CA cache", slog.Error(err)) + } + } else { + options.NATSCACache = cryptokeys.NoopSigningKeycache{} + } + } // Ensure all system role permissions are current. //nolint:gocritic // Startup reconciliation reads/writes system roles. There is @@ -2398,6 +2434,9 @@ func (api *API) Close() error { _ = api.OIDCConvertKeyCache.Close() _ = api.AppSigningKeyCache.Close() _ = api.AppEncryptionKeyCache.Close() + if api.NATSCACache != nil { + _ = api.NATSCACache.Close() + } _ = api.UpdatesProvider.Close() api.workspaceAgentConnWatcher.Close() diff --git a/coderd/cryptokeys/ca.go b/coderd/cryptokeys/ca.go new file mode 100644 index 0000000000000..7103a08c9e1fe --- /dev/null +++ b/coderd/cryptokeys/ca.go @@ -0,0 +1,149 @@ +package cryptokeys + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "time" + + "golang.org/x/xerrors" +) + +const ( + caCertPEMBlockType = "CERTIFICATE" + caKeyPEMBlockType = "EC PRIVATE KEY" + + // clockSkewTolerance backdates the CA certificate's NotBefore and extends + // its NotAfter so that replicas with mildly skewed clocks still accept it. + clockSkewTolerance = time.Hour +) + +// NATSCA is the decoded form of a single nats_ca crypto key row, produced by +// the generic crypto key cache (see idSecret). The CA signs the ephemeral leaf +// certificates that replicas use for NATS cluster mTLS. +// +// The active CA is served by a SigningKeycache.SigningKey call for the nats_ca +// feature; a specific historical CA (for verifying a peer leaf minted under an +// earlier CA during a rotation overlap) is served by VerifyingKey with that +// row's sequence. +type NATSCA struct { + // Sequence is the crypto_keys sequence of the row this CA came from. + Sequence int32 + // Cert is the CA certificate used to sign or verify leaf certificates. + Cert *x509.Certificate + // Key is the CA private key, used to sign leaves. + Key crypto.Signer +} + +// generateCASecret generates a new self-signed CA certificate and private key +// for signing NATS cluster leaf certificates, PEM-encoded into a single +// bundle for storage in the crypto_keys secret column. +// +// anchorTime is the key row's starts_at (which may be in the future for a +// rotated-in key). keyDuration is the rotator's key duration: the row stays the +// active signer for that long. The certificate stays valid for NATSCAOverlap +// past that window so that, once the next CA becomes the active signer, this CA +// is still valid while replicas' key caches refresh onto the new one. Leaves +// are separately clamped to expire before this NotAfter (see coderd/x/nats +// mintLeaf), so the overlap only needs to cover the cache-refresh transition. +func generateCASecret(anchorTime time.Time, keyDuration time.Duration) (string, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return "", xerrors.Errorf("generate key: %w", err) + } + + // 128-bit random serial per CA/Browser Forum conventions. + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return "", xerrors.Errorf("generate serial: %w", err) + } + + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: "coder-nats-ca", + }, + NotBefore: anchorTime.Add(-clockSkewTolerance), + NotAfter: anchorTime.Add(keyDuration + NATSCAOverlap), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLenZero: true, + } + + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + if err != nil { + return "", xerrors.Errorf("create certificate: %w", err) + } + + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return "", xerrors.Errorf("marshal private key: %w", err) + } + + var secret []byte + secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: caCertPEMBlockType, Bytes: der})...) + secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: caKeyPEMBlockType, Bytes: keyDER})...) + return string(secret), nil +} + +// parseCASecret parses a PEM bundle produced by generateCASecret back into +// the CA certificate and private key. +func parseCASecret(secret string) (*x509.Certificate, crypto.Signer, error) { + var ( + cert *x509.Certificate + key *ecdsa.PrivateKey + ) + rest := []byte(secret) + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + switch block.Type { + case caCertPEMBlockType: + if cert != nil { + return nil, nil, xerrors.New("multiple certificates in CA secret") + } + var err error + cert, err = x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, nil, xerrors.Errorf("parse certificate: %w", err) + } + case caKeyPEMBlockType: + if key != nil { + return nil, nil, xerrors.New("multiple private keys in CA secret") + } + var err error + key, err = x509.ParseECPrivateKey(block.Bytes) + if err != nil { + return nil, nil, xerrors.Errorf("parse private key: %w", err) + } + default: + return nil, nil, xerrors.Errorf("unexpected PEM block type: %q", block.Type) + } + } + if cert == nil { + return nil, nil, xerrors.New("no certificate in CA secret") + } + if key == nil { + return nil, nil, xerrors.New("no private key in CA secret") + } + if !key.PublicKey.Equal(cert.PublicKey) { + return nil, nil, xerrors.New("private key does not match certificate") + } + // Reject a structurally valid bundle whose certificate cannot act as a + // signing CA. Without this, a corrupted secret could yield a non-CA cert + // that silently becomes the active signer; leaves signed under it would + // then fail x509 verification on every replica. + if !cert.IsCA || !cert.BasicConstraintsValid || cert.KeyUsage&x509.KeyUsageCertSign == 0 { + return nil, nil, xerrors.New("certificate is not a valid signing CA") + } + return cert, key, nil +} diff --git a/coderd/cryptokeys/ca_internal_test.go b/coderd/cryptokeys/ca_internal_test.go new file mode 100644 index 0000000000000..419f52f1cdd78 --- /dev/null +++ b/coderd/cryptokeys/ca_internal_test.go @@ -0,0 +1,251 @@ +package cryptokeys + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/database/dbtime" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/testutil" + "github.com/coder/quartz" +) + +func TestCASecretRoundTrip(t *testing.T) { + t.Parallel() + + // The certificate's NotAfter must track the supplied keyDuration, not a + // hardcoded default, so a CA stays valid for as long as it can be the + // active signer plus the longest leaf it signs. + for _, keyDuration := range []time.Duration{DefaultKeyDuration, DefaultKeyDuration * 3, time.Hour} { + now := time.Now().UTC().Truncate(time.Second) + secret, err := generateCASecret(now, keyDuration) + require.NoError(t, err) + + cert, signer, err := parseCASecret(secret) + require.NoError(t, err) + + require.True(t, cert.IsCA) + require.True(t, cert.BasicConstraintsValid) + require.True(t, cert.MaxPathLenZero) + require.Equal(t, x509.KeyUsageCertSign, cert.KeyUsage) + require.Equal(t, now.Add(-clockSkewTolerance), cert.NotBefore) + require.Equal(t, now.Add(keyDuration+NATSCAOverlap), cert.NotAfter) + require.Equal(t, cert.PublicKey, signer.Public()) + + // The cert must outlive its active-signer window so leaves signed at + // the end of that window still chain to a valid CA. + require.True(t, cert.NotAfter.After(now.Add(keyDuration)), + "cert must remain valid past the end of its active-signer window") + + // The cert must be able to verify itself as a trust root. + pool := x509.NewCertPool() + pool.AddCert(cert) + _, err = cert.Verify(x509.VerifyOptions{Roots: pool}) + require.NoError(t, err) + } +} + +func TestParseCASecretErrors(t *testing.T) { + t.Parallel() + + now := time.Now() + secretA, err := generateCASecret(now, DefaultKeyDuration) + require.NoError(t, err) + secretB, err := generateCASecret(now, DefaultKeyDuration) + require.NoError(t, err) + + certA, keyA := splitCAPEM(t, secretA) + _, keyB := splitCAPEM(t, secretB) + + nonCACert, nonCAKey := generateNonCAPEM(t, now) + + cases := []struct { + name string + secret string + errText string + }{ + {"Empty", "", "no certificate"}, + {"NotPEM", "not pem at all", "no certificate"}, + {"CertOnly", string(certA), "no private key"}, + {"KeyCertMismatch", string(certA) + string(keyB), "does not match certificate"}, + {"MultipleCertificates", string(certA) + string(certA) + string(keyA), "multiple certificates"}, + {"MultiplePrivateKeys", string(certA) + string(keyA) + string(keyA), "multiple private keys"}, + {"UnexpectedBlockType", string(pemBlock("RSA PRIVATE KEY", []byte("x"))), "unexpected PEM block type"}, + {"BadCertificateBytes", string(pemBlock(caCertPEMBlockType, []byte("garbage"))), "parse certificate"}, + {"BadPrivateKeyBytes", string(certA) + string(pemBlock(caKeyPEMBlockType, []byte("garbage"))), "parse private key"}, + {"NotASigningCA", string(nonCACert) + string(nonCAKey), "not a valid signing CA"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, _, err := parseCASecret(tc.secret) + require.ErrorContains(t, err, tc.errText) + }) + } +} + +// splitCAPEM splits a CA secret bundle into its certificate and private key +// PEM blocks so tests can recombine them into malformed bundles. +func splitCAPEM(t *testing.T, secret string) (certPEM, keyPEM []byte) { + t.Helper() + rest := []byte(secret) + for { + block, r := pem.Decode(rest) + if block == nil { + break + } + rest = r + switch block.Type { + case caCertPEMBlockType: + certPEM = pem.EncodeToMemory(block) + case caKeyPEMBlockType: + keyPEM = pem.EncodeToMemory(block) + } + } + require.NotNil(t, certPEM) + require.NotNil(t, keyPEM) + return certPEM, keyPEM +} + +func pemBlock(blockType string, der []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: der}) +} + +// generateNonCAPEM produces a structurally valid cert+key bundle whose +// certificate is not a CA (no IsCA, no KeyUsageCertSign). The key matches the +// cert, so it passes every parseCASecret check except the signing-CA check. +func generateNonCAPEM(t *testing.T, now time.Time) (certPEM, keyPEM []byte) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "not-a-ca"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + require.NoError(t, err) + keyDER, err := x509.MarshalECPrivateKey(key) + require.NoError(t, err) + return pemBlock(caCertPEMBlockType, der), pemBlock(caKeyPEMBlockType, keyDER) +} + +// TestNATSCASigningCache exercises the nats_ca feature through the generic +// signing key cache: the PEM secret decodes into a *NATSCA, SigningKey serves +// the active CA, VerifyingKey serves a specific CA by sequence, and a rotation +// is picked up on the next refresh. +func TestNATSCASigningCache(t *testing.T) { + t.Parallel() + + t.Run("ActiveAndVerifyingByID", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + now := time.Now().UTC() + + current := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: 1, + StartsAt: now.Add(-time.Hour), + }) + + cache, err := NewSigningCache(ctx, testutil.Logger(t), &DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA) + require.NoError(t, err) + defer cache.Close() + + id, key, err := cache.SigningKey(ctx) + require.NoError(t, err) + + ca, ok := key.(*NATSCA) + require.True(t, ok, "signing key should decode to *NATSCA, got %T", key) + require.Equal(t, current.Sequence, ca.Sequence) + require.NotNil(t, ca.Cert) + require.NotNil(t, ca.Key) + + currentCert, _, err := parseCASecret(current.Secret.String) + require.NoError(t, err) + require.Equal(t, currentCert.Raw, ca.Cert.Raw) + + // VerifyingKey looks the CA up by the sequence embedded in id, which is + // how a peer leaf minted under this CA is verified. + verifying, err := cache.VerifyingKey(ctx, id) + require.NoError(t, err) + vca, ok := verifying.(*NATSCA) + require.True(t, ok, "verifying key should decode to *NATSCA, got %T", verifying) + require.Equal(t, currentCert.Raw, vca.Cert.Raw) + }) + + t.Run("RefreshesOnRotation", func(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitShort) + clock := quartz.NewMock(t) + now := dbtime.Now() + clock.Set(now) + + first := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: 1, + StartsAt: now.Add(-time.Hour), + }) + + cache, err := NewSigningCache(ctx, testutil.Logger(t), &DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA, WithCacheClock(clock)) + require.NoError(t, err) + defer cache.Close() + + _, key, err := cache.SigningKey(ctx) + require.NoError(t, err) + require.Equal(t, first.Sequence, key.(*NATSCA).Sequence) + + // Simulate a rotation by inserting a higher-sequence active CA. The old + // CA stays valid for verification by its sequence. + second := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: 2, + StartsAt: now.Add(-time.Minute), + }) + + // Fire the background refresher; the active CA advances to the new row. + clock.Advance(refreshInterval).MustWait(ctx) + + _, key, err = cache.SigningKey(ctx) + require.NoError(t, err) + require.Equal(t, second.Sequence, key.(*NATSCA).Sequence) + + oldVerifying, err := cache.VerifyingKey(ctx, "1") + require.NoError(t, err) + require.Equal(t, first.Sequence, oldVerifying.(*NATSCA).Sequence) + }) +} + +func TestNoopSigningKeycache(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + var cache SigningKeycache = NoopSigningKeycache{} + + _, _, err := cache.SigningKey(ctx) + require.ErrorIs(t, err, ErrKeyNotFound) + + _, err = cache.VerifyingKey(ctx, "1") + require.ErrorIs(t, err, ErrKeyNotFound) + + require.NoError(t, cache.Close()) +} diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index de40324df1a06..75269d98395e7 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -55,6 +55,26 @@ type SigningKeycache interface { io.Closer } +// NoopSigningKeycache is a SigningKeycache that holds no keys: SigningKey and +// VerifyingKey always report ErrKeyNotFound. It lets a subsystem that only +// needs real keys once an optional feature is enabled (for example NATS +// cluster mTLS, which only signs leaves under enterprise HA) be constructed +// without a database dependency, then be swapped for a real cache when the +// feature turns on. +type NoopSigningKeycache struct{} + +var _ SigningKeycache = NoopSigningKeycache{} + +func (NoopSigningKeycache) SigningKey(context.Context) (string, interface{}, error) { + return "", nil, ErrKeyNotFound +} + +func (NoopSigningKeycache) VerifyingKey(context.Context, string) (interface{}, error) { + return nil, ErrKeyNotFound +} + +func (NoopSigningKeycache) Close() error { return nil } + const ( // latestSequence is a special sequence number that represents the latest key. latestSequence = -1 @@ -213,23 +233,42 @@ func isEncryptionKeyFeature(feature codersdk.CryptoKeyFeature) bool { func isSigningKeyFeature(feature codersdk.CryptoKeyFeature) bool { switch feature { - case codersdk.CryptoKeyFeatureTailnetResume, codersdk.CryptoKeyFeatureOIDCConvert, codersdk.CryptoKeyFeatureWorkspaceAppsToken: + case codersdk.CryptoKeyFeatureTailnetResume, codersdk.CryptoKeyFeatureOIDCConvert, codersdk.CryptoKeyFeatureWorkspaceAppsToken, codersdk.CryptoKeyFeatureNATSCA: return true default: return false } } -func idSecret(k codersdk.CryptoKey) (string, []byte, error) { +// idSecret materializes a stored crypto key into the in-memory key object the +// feature uses, returning it as an interface{} alongside the key's id (its +// sequence as a decimal string). Most features hex-decode the secret into raw +// bytes, but nats_ca stores a PEM cert+key bundle and decodes into a *NATSCA. +// +// TODO: this hard-coded switch on feature is the simplest way to support a +// second secret encoding, but it couples this generic cache to nats_ca +// specifics. Explore abstracting the decode step (for example a per-feature +// decoder injected at construction) so new key types can be added without +// editing this function. +func idSecret(k codersdk.CryptoKey) (string, interface{}, error) { + id := strconv.FormatInt(int64(k.Sequence), 10) + + if k.Feature == codersdk.CryptoKeyFeatureNATSCA { + cert, signer, err := parseCASecret(k.Secret) + if err != nil { + return "", nil, xerrors.Errorf("decode nats_ca key: %w", err) + } + return id, &NATSCA{Sequence: k.Sequence, Cert: cert, Key: signer}, nil + } + key, err := hex.DecodeString(k.Secret) if err != nil { return "", nil, xerrors.Errorf("decode key: %w", err) } - - return strconv.FormatInt(int64(k.Sequence), 10), key, nil + return id, key, nil } -func (c *cache) cryptoKey(ctx context.Context, sequence int32) (string, []byte, error) { +func (c *cache) cryptoKey(ctx context.Context, sequence int32) (string, interface{}, error) { c.logger.Debug(ctx, "request for key", slog.F("sequence", sequence)) c.mu.Lock() defer c.mu.Unlock() @@ -284,7 +323,7 @@ func (c *cache) key(sequence int32) (codersdk.CryptoKey, bool) { return key, ok } -func checkKey(key codersdk.CryptoKey, sequence int32, now time.Time) (string, []byte, error) { +func checkKey(key codersdk.CryptoKey, sequence int32, now time.Time) (string, interface{}, error) { if sequence == latestSequence { if !key.CanSign(now) { return "", nil, ErrKeyInvalid diff --git a/coderd/cryptokeys/rotate.go b/coderd/cryptokeys/rotate.go index 4f30f842597b0..775131185bf03 100644 --- a/coderd/cryptokeys/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -21,6 +21,15 @@ const ( WorkspaceAppsTokenDuration = time.Minute OIDCConvertTokenDuration = time.Minute * 5 TailnetResumeTokenDuration = time.Hour * 24 + // NATSCAOverlap is how long a NATS cluster CA certificate stays valid past + // the end of its active-signing window (startsAt + keyDuration). The next CA + // becomes the active signer at the window's end, but replicas keep minting + // leaves with the old CA until their key cache refreshes onto the new one. + // This overlap keeps the old CA valid through that transition, so it must + // exceed the cache refresh interval (plus a small leaf clamp buffer). Leaf + // lifetime imposes nothing here: leaves are clamped to just before their + // signing CA's NotAfter (see coderd/x/nats mintLeaf). + NATSCAOverlap = time.Minute * 30 // defaultRotationInterval is the default interval at which keys are checked for rotation. defaultRotationInterval = time.Minute * 10 @@ -70,6 +79,15 @@ func WithKeyDuration(keyDuration time.Duration) RotatorOption { } } +// WithFeatures sets the crypto key features the rotator manages, replacing the +// default set. Use this to opt experiment- or deployment-gated features (such +// as the NATS cluster CA) into rotation only when their owner is active. +func WithFeatures(features []database.CryptoKeyFeature) RotatorOption { + return func(r *rotator) { + r.features = slices.Clone(features) + } +} + // StartRotator starts a background process that rotates keys in the database. // It ensures there's at least one valid key per feature prior to returning. // Canceling the provided context will stop the background process. @@ -126,10 +144,7 @@ func (k *rotator) rotateKeys(ctx context.Context) error { return xerrors.Errorf("get keys: %w", err) } - featureKeys, err := keysByFeature(cryptokeys, k.features) - if err != nil { - return xerrors.Errorf("keys by feature: %w", err) - } + featureKeys := keysByFeature(cryptokeys, k.features) now := dbtime.Time(k.clock.Now().UTC()) for feature, keys := range featureKeys { @@ -189,7 +204,7 @@ func (k *rotator) rotateKeys(ctx context.Context) error { } func (k *rotator) insertNewKey(ctx context.Context, tx database.Store, feature database.CryptoKeyFeature, startsAt time.Time) (database.CryptoKey, error) { - secret, err := generateNewSecret(feature) + secret, err := generateNewSecret(feature, startsAt, k.keyDuration) if err != nil { return database.CryptoKey{}, xerrors.Errorf("generate new secret: %w", err) } @@ -246,7 +261,11 @@ func (k *rotator) rotateKey(ctx context.Context, tx database.Store, key database return []database.CryptoKey{updatedKey, newKey}, nil } -func generateNewSecret(feature database.CryptoKeyFeature) (string, error) { +// generateNewSecret generates the secret for a new key of the given feature. +// keyDuration is the rotator's key duration; it is only used by features whose +// secret encodes its own validity window (currently only the NATS CA, whose +// certificate must outlive the key row's active-signer period). +func generateNewSecret(feature database.CryptoKeyFeature, startsAt time.Time, keyDuration time.Duration) (string, error) { switch feature { case database.CryptoKeyFeatureWorkspaceAppsAPIKey: return generateKey(32) @@ -256,6 +275,8 @@ func generateNewSecret(feature database.CryptoKeyFeature) (string, error) { return generateKey(64) case database.CryptoKeyFeatureTailnetResume: return generateKey(64) + case database.CryptoKeyFeatureNATSCA: + return generateCASecret(startsAt, keyDuration) } return "", xerrors.Errorf("unknown feature: %s", feature) } @@ -279,6 +300,11 @@ func tokenDuration(feature database.CryptoKeyFeature) time.Duration { return OIDCConvertTokenDuration case database.CryptoKeyFeatureTailnetResume: return TailnetResumeTokenDuration + case database.CryptoKeyFeatureNATSCA: + // The old CA row only needs to outlive its own certificate, which stays + // valid for NATSCAOverlap past the active-signing window. Keeping the + // row (and thus its trust-root status) beyond cert expiry is pointless. + return NATSCAOverlap default: return 0 } @@ -297,19 +323,25 @@ func shouldRotateKey(key database.CryptoKey, keyDuration time.Duration, now time return !now.Add(time.Hour).UTC().Before(expirationTime) } -func keysByFeature(keys []database.CryptoKey, features []database.CryptoKeyFeature) (map[database.CryptoKeyFeature][]database.CryptoKey, error) { +// keysByFeature groups keys by feature, restricted to the managed feature set. +// GetCryptoKeys returns rows for every feature, but the rotator only manages a +// subset (features can be gated, e.g. nats_ca behind an experiment). Keys for +// features outside the managed set belong to features this rotator is not +// responsible for and are skipped, so their presence (for example nats_ca rows +// left over from a prior experiment-on run) does not abort rotation of the +// managed features. +func keysByFeature(keys []database.CryptoKey, features []database.CryptoKeyFeature) map[database.CryptoKeyFeature][]database.CryptoKey { m := map[database.CryptoKeyFeature][]database.CryptoKey{} for _, feature := range features { m[feature] = []database.CryptoKey{} } for _, key := range keys { if _, ok := m[key.Feature]; !ok { - return nil, xerrors.Errorf("unknown feature: %s", key.Feature) + continue } - m[key.Feature] = append(m[key.Feature], key) } - return m, nil + return m } // minStartsAt ensures the minimum starts_at time we use for a new diff --git a/coderd/cryptokeys/rotate_internal_test.go b/coderd/cryptokeys/rotate_internal_test.go index 4d43f24e1876d..89216cf708947 100644 --- a/coderd/cryptokeys/rotate_internal_test.go +++ b/coderd/cryptokeys/rotate_internal_test.go @@ -104,6 +104,112 @@ func Test_rotateKeys(t *testing.T) { require.Equal(t, newKey, keys[0]) }) + t.Run("RotatesNATSCA", func(t *testing.T) { + t.Parallel() + + var ( + db, _ = dbtestutil.NewDB(t) + clock = quartz.NewMock(t) + keyDuration = time.Hour * 24 * 7 + logger = testutil.Logger(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + kr := &rotator{ + db: db, + keyDuration: keyDuration, + clock: clock, + logger: logger, + features: []database.CryptoKeyFeature{ + database.CryptoKeyFeatureNATSCA, + }, + } + + now := dbnow(clock) + + oldKey := dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + StartsAt: now, + Sequence: 4, + }) + + // Advance the window to just inside rotation time. + _ = clock.Advance(keyDuration - time.Minute*59) + err := kr.rotateKeys(ctx) + require.NoError(t, err) + + // The old CA row is retained roughly as long as its certificate is + // valid: NATSCAOverlap past the active-signing window, plus the + // rotator's standard 1h propagation buffer. + expectedDeletesAt := oldKey.ExpiresAt(keyDuration).Add(NATSCAOverlap + time.Hour) + oldKey, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ + Feature: oldKey.Feature, + Sequence: oldKey.Sequence, + }) + require.NoError(t, err) + require.Equal(t, expectedDeletesAt, oldKey.DeletesAt.Time.UTC()) + + newKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: oldKey.Sequence + 1, + }) + require.NoError(t, err) + requireKey(t, newKey, database.CryptoKeyFeatureNATSCA, oldKey.ExpiresAt(keyDuration), nullTime, oldKey.Sequence+1) + }) + + t.Run("IgnoresUnmanagedFeatureKeys", func(t *testing.T) { + t.Parallel() + + // Regression: a rotator managing a subset of features (e.g. after the + // nats_ca experiment is toggled off) must still rotate its managed + // features even when the DB holds keys for features it does not manage, + // such as nats_ca rows left over from a prior experiment-on run. + // Previously such rows aborted every rotation. + var ( + db, _ = dbtestutil.NewDB(t) + clock = quartz.NewMock(t) + keyDuration = time.Hour * 24 * 7 + logger = testutil.Logger(t) + ctx = testutil.Context(t, testutil.WaitShort) + ) + + kr := &rotator{ + db: db, + keyDuration: keyDuration, + clock: clock, + logger: logger, + // Manages only tailnet resume; nats_ca is intentionally not managed, + // mirroring the experiment being off. + features: []database.CryptoKeyFeature{ + database.CryptoKeyFeatureTailnetResume, + }, + } + + now := dbnow(clock) + + // A leftover nats_ca row the rotator does not manage. + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + StartsAt: now, + Sequence: 1, + }) + + // No managed key exists yet, so rotation must insert one for the managed + // feature and must not error on the unmanaged nats_ca row. + err := kr.rotateKeys(ctx) + require.NoError(t, err) + + newKey, err := db.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureTailnetResume) + require.NoError(t, err) + require.Equal(t, database.CryptoKeyFeatureTailnetResume, newKey.Feature) + + // The unmanaged nats_ca row is untouched (no rotation, no delete). + natsKeys, err := db.GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureNATSCA) + require.NoError(t, err) + require.Len(t, natsKeys, 1) + require.False(t, natsKeys[0].DeletesAt.Valid) + }) + t.Run("DoesNotRotateValidKeys", func(t *testing.T) { t.Parallel() @@ -409,8 +515,7 @@ func Test_rotateKeys(t *testing.T) { require.NoError(t, err) require.Len(t, keys, 5) - kbf, err := keysByFeature(keys, defaultRotatedFeatures) - require.NoError(t, err) + kbf := keysByFeature(keys, defaultRotatedFeatures) // No actions on OIDC convert. require.Len(t, kbf[database.CryptoKeyFeatureOIDCConvert], 1) @@ -586,6 +691,14 @@ func requireKey(t *testing.T, key database.CryptoKey, feature database.CryptoKey require.Equal(t, deletesAt.Time.UTC(), key.DeletesAt.Time.UTC()) require.Equal(t, sequence, key.Sequence) + // The NATS CA secret is a PEM bundle rather than hex-encoded bytes. + if key.Feature == database.CryptoKeyFeatureNATSCA { + cert, _, err := parseCASecret(key.Secret.String) + require.NoError(t, err) + require.True(t, cert.IsCA) + return + } + secret, err := hex.DecodeString(key.Secret.String) require.NoError(t, err) diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index 97d3094ea06c1..db7d8e6f0cb42 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -2,13 +2,19 @@ package dbgen import ( "context" + "crypto/ecdsa" + "crypto/elliptic" "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" "database/sql" "encoding/hex" "encoding/json" + "encoding/pem" "errors" "fmt" "maps" + "math/big" "net" "strings" "testing" @@ -2219,10 +2225,47 @@ func newCryptoKeySecret(feature database.CryptoKeyFeature) (string, error) { return generateCryptoKey(64) case database.CryptoKeyFeatureTailnetResume: return generateCryptoKey(64) + case database.CryptoKeyFeatureNATSCA: + return generateCACryptoKeySecret() } return "", xerrors.Errorf("unknown feature: %s", feature) } +// generateCACryptoKeySecret generates a self-signed CA certificate and private +// key as a PEM bundle, matching the secret format that coderd/cryptokeys +// produces for the nats_ca feature. It intentionally duplicates +// cryptokeys.generateCASecret rather than calling it: coderd/cryptokeys's +// internal tests import dbgen, so importing cryptokeys here would create a +// test-build import cycle. +func generateCACryptoKeySecret() (string, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return "", xerrors.Errorf("generate key: %w", err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "dbgen-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLenZero: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + if err != nil { + return "", xerrors.Errorf("create certificate: %w", err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return "", xerrors.Errorf("marshal private key: %w", err) + } + var secret []byte + secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...) + secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})...) + return string(secret), nil +} + func generateCryptoKey(length int) (string, error) { b := make([]byte, length) _, err := rand.Read(b) diff --git a/coderd/database/modelmethods.go b/coderd/database/modelmethods.go index cec3e1d291b6e..67a6c7812d953 100644 --- a/coderd/database/modelmethods.go +++ b/coderd/database/modelmethods.go @@ -2,7 +2,6 @@ package database import ( "database/sql" - "encoding/hex" "fmt" "slices" "sort" @@ -869,10 +868,6 @@ func (k CryptoKey) ExpiresAt(keyDuration time.Duration) time.Time { return k.StartsAt.Add(keyDuration).UTC() } -func (k CryptoKey) DecodeString() ([]byte, error) { - return hex.DecodeString(k.Secret.String) -} - func (k CryptoKey) CanSign(now time.Time) bool { isAfterStart := !k.StartsAt.IsZero() && !now.Before(k.StartsAt) return isAfterStart && k.CanVerify(now) diff --git a/coderd/x/nats/cluster.go b/coderd/x/nats/cluster.go index d70de4dc3a2dc..a2b2ca3612510 100644 --- a/coderd/x/nats/cluster.go +++ b/coderd/x/nats/cluster.go @@ -11,6 +11,7 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog/v3" + "github.com/coder/coder/v2/coderd/cryptokeys" ) const defaultClusterTokenUsername = "coder" @@ -49,6 +50,21 @@ func (p *Pubsub) SetPeerFetcher(fetcher PeerFetcher) { p.RefreshPeers() } +// SetCACache swaps the cluster mTLS CA cache, then triggers a peer refresh so +// any route blocked by the previous (for example noop) cache is retried. It is +// a no-op unless the pubsub was started with cluster TLS enabled +// (Options.ClusterCA set, which installs the TLS callbacks). Passing a noop +// cache reverts to no mTLS: new route handshakes can no longer mint a leaf and +// will not form. The leaf IP SAN is fixed at construction from ClusterHost, so +// it is not passed here. It logs the resulting mTLS state. +func (p *Pubsub) SetCACache(ca cryptokeys.SigningKeycache) { + if p.clusterTLS == nil { + return + } + p.clusterTLS.setCACache(ca) + p.RefreshPeers() +} + // RefreshPeers signals the peer refresh worker to fetch and apply the latest // peer route addresses. Multiple pending refreshes are coalesced. func (p *Pubsub) RefreshPeers() { diff --git a/coderd/x/nats/pubsub.go b/coderd/x/nats/pubsub.go index 57ced2aeb3933..14438f2ed5923 100644 --- a/coderd/x/nats/pubsub.go +++ b/coderd/x/nats/pubsub.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "hash/fnv" + "net" "net/url" "sync" "time" @@ -15,7 +16,9 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog/v3" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database/pubsub" + "github.com/coder/quartz" ) // DefaultServerMaxPendingBytes caps how many bytes the embedded NATS server will @@ -121,6 +124,24 @@ type Options struct { // clustered embedded NATS servers. Empty disables route auth. ClusterAuthToken string + // ClusterCA enables mutual TLS on the cluster route listener. When set + // (and cluster mode is enabled), each replica mints an ephemeral leaf + // certificate from the active nats_ca CA and verifies peers against the + // CA fetched from this cache on each handshake. Nil keeps routes + // plaintext (token auth only). cryptokeys.SigningKeycache satisfies this. + // + // The leaf's IP SAN (and the accept-side source binding) is this replica's + // ClusterHost, so ClusterHost must be an IP for mTLS to activate. + ClusterCA cryptokeys.SigningKeycache + + // clock overrides the cluster TLS clock, for tests. + clock quartz.Clock + + // clusterTLSTimeout overrides the cluster route TLS handshake timeout, for + // tests. Zero leaves the NATS default (2s). Tests use a longer timeout + // because handshakes are flaky under load and in CI. + clusterTLSTimeout time.Duration + // PeerFetcher provides the current set of peer route addresses. // RefreshPeers uses it to update the configured cluster routes. PeerFetcher PeerFetcher @@ -182,6 +203,8 @@ type Pubsub struct { clustered bool serverOpts *natsserver.Options currentRoutes []*url.URL + // clusterTLS is non-nil when the cluster route listener runs mutual TLS. + clusterTLS *clusterTLS peerFetcher PeerFetcher peerRefresh chan struct{} @@ -304,6 +327,25 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub, return nil, err } + // When ClusterCA is set, install the cluster TLS callbacks at boot so the + // route listener can negotiate mTLS. The callbacks read the CA cache on + // each handshake, so the default noop cache keeps routes inert (no leaf can + // be minted) until SetCACache swaps in a real cache. The leaf IP SAN is this + // replica's ClusterHost, fixed here at construction; leaf minting enforces + // that it is an IP. ClusterCA == nil keeps routes plaintext (token auth + // only). + var ct *clusterTLS + if !opts.disableCluster && opts.ClusterCA != nil { + selfIP := net.ParseIP(opts.ClusterHost) + ct = newClusterTLS(ctx, logger, opts.clock, opts.ClusterCA, selfIP) + sopts.Cluster.TLSConfig = ct.tlsConfig() + // Leave TLSTimeout unset (NATS defaults to 2s) unless a test overrides + // it; the default has not shown a need to change in production. + if opts.clusterTLSTimeout > 0 { + sopts.Cluster.TLSTimeout = opts.clusterTLSTimeout.Seconds() + } + } + ns, err := startEmbeddedServer(sopts) if err != nil { return nil, err @@ -333,6 +375,7 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub, p.clustered = !opts.disableCluster p.serverOpts = sopts.Clone() p.currentRoutes = cloneRouteURLs(sopts.Routes) + p.clusterTLS = ct handlers := p.buildConnHandlers() publishPool, err := newConnPool(ns, opts, handlers, opts.PublishConns, "coder-pubsub-pub") diff --git a/coderd/x/nats/pubsub_internal_test.go b/coderd/x/nats/pubsub_internal_test.go index 4dcdf1abad302..cfdb07e6ff2c8 100644 --- a/coderd/x/nats/pubsub_internal_test.go +++ b/coderd/x/nats/pubsub_internal_test.go @@ -538,13 +538,19 @@ func defaultTestOptions() Options { return Options{disableCluster: true} } +// testClusterTLSTimeout relaxes the cluster route TLS handshake timeout in +// tests. NATS defaults to a tight 2s, which is flaky under load and in CI; +// production keeps the default until it is shown to need changing. +const testClusterTLSTimeout = 10 * time.Second + func clusterTestOptions(t *testing.T) Options { t.Helper() return Options{ - ClusterHost: "127.0.0.1", - ClusterPort: natsserver.RANDOM_PORT, - disableCluster: false, - ClusterAuthToken: fmt.Sprintf("shared-token-%d", time.Now().UnixNano()), + ClusterHost: "127.0.0.1", + ClusterPort: natsserver.RANDOM_PORT, + disableCluster: false, + ClusterAuthToken: fmt.Sprintf("shared-token-%d", time.Now().UnixNano()), + clusterTLSTimeout: testClusterTLSTimeout, } } diff --git a/coderd/x/nats/tls.go b/coderd/x/nats/tls.go new file mode 100644 index 0000000000000..3ed2b92c2cd14 --- /dev/null +++ b/coderd/x/nats/tls.go @@ -0,0 +1,429 @@ +package nats + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "slices" + "strconv" + "sync" + "time" + + "golang.org/x/xerrors" + + "cdr.dev/slog/v3" + "github.com/coder/coder/v2/coderd/cryptokeys" + "github.com/coder/quartz" +) + +const ( + // leafSerialBits is the entropy of a leaf certificate serial number. + leafSerialBits = 128 + // clockSkewToleranceTLS backdates a leaf's NotBefore so a peer with a + // mildly skewed clock still accepts a freshly minted leaf. + clockSkewToleranceTLS = time.Hour +) + +// clusterTLS builds the cluster route *tls.Config. Certificate selection and +// peer verification are tls.Config callbacks that consult the CA cache on each +// use, so a CA rotation is tracked without restarting or reloading the server. +type clusterTLS struct { + ctx context.Context + logger slog.Logger + clock quartz.Clock + + mu sync.Mutex + // ca is swapped by setCACache: the default noop cache mints no leaf (so no + // route forms) until the real cache is installed once cluster mTLS is + // enabled. ip is this replica's cluster host, fixed at construction and + // embedded as the leaf IP SAN. + ca cryptokeys.SigningKeycache + ip net.IP + // leaf is the cached leaf certificate, reused until it expires (its NotAfter + // equals the signing CA's) or setCACache clears it on a cache swap. + leaf *tls.Certificate + // verifyPools caches the root pool used to verify a peer leaf, keyed by the + // CA sequence stamped in the leaf. A CA cert is immutable for a given + // sequence, so the pool is built once and reused across handshakes. Expired + // entries are pruned on insert to bound the map across rotations. + verifyPools map[string]cachedVerifyPool +} + +// cachedVerifyPool is a verify root pool plus the NotAfter of the CA cert it +// holds; the entry is dropped once the clock passes notAfter. +type cachedVerifyPool struct { + pool *x509.CertPool + notAfter time.Time +} + +func newClusterTLS(ctx context.Context, logger slog.Logger, clock quartz.Clock, ca cryptokeys.SigningKeycache, ip net.IP) *clusterTLS { + if clock == nil { + clock = quartz.NewReal() + } + return &clusterTLS{ + ctx: ctx, + logger: logger.Named("cluster_tls"), + clock: clock, + ca: ca, + ip: ip, + } +} + +// setCACache swaps the CA cache. Because the tls.Config callbacks read it on +// each handshake, the swap takes effect without a server restart or route +// reload: installing the real cache lets routes negotiate mTLS, and reverting +// to a noop cache makes leaf minting fail so no new route can form. The leaf IP +// SAN is fixed at construction (this replica's cluster host does not change), so +// it is not touched here. A swap clears the cached leaf so the next handshake +// re-mints under the new CA. +func (t *clusterTLS) setCACache(ca cryptokeys.SigningKeycache) { + t.mu.Lock() + t.ca = ca + t.leaf = nil + // Verify pools follow the CA source: drop them so stale roots are not + // reused after a swap to a noop or different CA. + t.verifyPools = nil + ip := t.ip + t.mu.Unlock() + + // Log the resulting mTLS state. A noop cache disables mTLS (no leaf can be + // minted); a real cache with a valid self IP enables it; a real cache + // without an IP cluster host leaves routes plaintext (token auth only). + switch { + case isNoopSigningCache(ca): + t.logger.Info(t.ctx, "nats cluster mTLS disabled") + case len(ip) == 0: + t.logger.Warn(t.ctx, "nats cluster mTLS inactive: cluster host is not an IP; cluster routes use token auth only") + default: + t.logger.Info(t.ctx, "nats cluster mTLS enabled") + } +} + +// isNoopSigningCache reports whether ca is the no-op cache used to disable +// cluster mTLS. +func isNoopSigningCache(ca cryptokeys.SigningKeycache) bool { + _, ok := ca.(cryptokeys.NoopSigningKeycache) + return ok +} + +// caCache returns the current CA cache under lock so callers do not hold the +// lock across cache I/O. +func (t *clusterTLS) caCache() cryptokeys.SigningKeycache { + t.mu.Lock() + defer t.mu.Unlock() + return t.ca +} + +// tlsConfig returns the *tls.Config for the embedded server's cluster route +// listener. The same config is used by NATS for both accepting inbound routes +// (TLS server) and soliciting outbound routes (TLS client), so it sets both +// GetCertificate and GetClientCertificate. +// +// Verification is done in VerifyConnection against the CA fetched fresh from +// the cache, not against a static RootCAs/ClientCAs pool that cannot follow a +// rotating CA. InsecureSkipVerify disables Go's default static-root check on +// the dialing side ONLY so verifyConnection can run instead; it does not make +// the connection unauthenticated. Every connection is still mutually verified +// (ClientAuth requires a peer certificate) against live CA material. +// +// GetConfigForClient runs only when accepting a route (TLS server side), where +// the dialing peer's source IP is available on the underlying connection. It +// returns a per-connection config whose VerifyConnection additionally requires +// the peer leaf's IP SAN to match that source IP, binding the certificate to +// the network origin. The dialing side has no equivalent hook (Go does not +// expose the connection in client-certificate callbacks), so it relies on the +// base VerifyConnection: chain + membership against the known peer set. +func (t *clusterTLS) tlsConfig() *tls.Config { + return &tls.Config{ + MinVersion: tls.VersionTLS13, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + leaf, err := t.currentLeaf() + if err != nil { + t.logger.Warn(t.ctx, "get nats cluster leaf for GetCertificate", slog.Error(err)) + } + return leaf, err + }, + GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + leaf, err := t.currentLeaf() + if err != nil { + t.logger.Warn(t.ctx, "get nats cluster leaf for GetClientCertificate", slog.Error(err)) + } + return leaf, err + }, + ClientAuth: tls.RequireAnyClientCert, + //nolint:gosec // Not insecure: verify performs full chain verification + // against the live CA cache. Go's static RootCAs cannot track a rotating + // CA, so default verification is replaced, not removed. + InsecureSkipVerify: true, + VerifyConnection: func(cs tls.ConnectionState) error { + err := t.verify(cs, nil) + if err != nil { + t.logger.Warn(t.ctx, "verify nats cluster peer for VerifyConnection", slog.Error(err)) + } + return err + }, + GetConfigForClient: t.configForClient, + } +} + +// configForClient builds the per-connection config used when accepting a route. +// It captures the dialing peer's source IP from the underlying connection so +// VerifyConnection can require the peer leaf's IP SAN to match it. NATS calls +// this on each inbound handshake, so a fresh config is allocated per accepted +// connection; that is fine at cluster-route cardinality (a handful of peers). +func (t *clusterTLS) configForClient(chi *tls.ClientHelloInfo) (*tls.Config, error) { + // The accept side must bind the peer leaf to the address it connected from, + // so a source IP is required. Fail closed if it cannot be determined rather + // than silently skipping the binding in verify. + sourceIP, err := clientSourceIP(chi) + if err != nil { + t.logger.Warn(t.ctx, "reject nats cluster route: no source IP", slog.Error(err)) + return nil, err + } + + cfg := &tls.Config{ + MinVersion: tls.VersionTLS13, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + leaf, err := t.currentLeaf() + if err != nil { + t.logger.Warn(t.ctx, "get nats cluster leaf for GetCertificate", slog.Error(err)) + } + return leaf, err + }, + ClientAuth: tls.RequireAnyClientCert, + //nolint:gosec // See tlsConfig: verification is performed in VerifyConnection. + InsecureSkipVerify: true, + VerifyConnection: func(cs tls.ConnectionState) error { + err := t.verify(cs, sourceIP) + if err != nil { + t.logger.Warn(t.ctx, "verify nats cluster peer for VerifyConnection", slog.Error(err)) + } + return err + }, + } + return cfg, nil +} + +// clientSourceIP extracts the dialing peer's source IP from the accepted +// connection. The accept side requires it, so every failure is an error rather +// than a nil that would bypass source binding in verify. +func clientSourceIP(chi *tls.ClientHelloInfo) (net.IP, error) { + if chi.Conn == nil { + return nil, xerrors.New("no underlying connection") + } + remote := chi.Conn.RemoteAddr() + if remote == nil { + return nil, xerrors.New("no remote address") + } + host, _, err := net.SplitHostPort(remote.String()) + if err != nil { + return nil, xerrors.Errorf("split remote address %q: %w", remote.String(), err) + } + ip := net.ParseIP(host) + if ip == nil { + return nil, xerrors.Errorf("remote host %q is not an IP", host) + } + return ip, nil +} + +// currentLeaf returns the cached leaf, re-minting it when it is missing or +// expired. A leaf carries no independent lifetime: its NotAfter equals its +// signing CA's (see mintLeaf), so re-minting is driven purely by CA rotation. +// +// The whole method holds t.mu so the CA cache, IP, and cached leaf are read as +// a consistent set: a concurrent setCACache cannot swap the CA out from under +// the IP we mint with. The lock is held across the SigningKey lookup and the +// (rare) mint; both are cheap (an in-memory cache hit and, only on a miss, a +// keygen+sign). +func (t *clusterTLS) currentLeaf() (*tls.Certificate, error) { + t.mu.Lock() + defer t.mu.Unlock() + + // Reuse the cached leaf while it is still within its validity window, + // before consulting the signing cache. A leaf's NotAfter equals its signing + // CA's, and the previous CA stays trusted by peers through the rotation + // overlap, so a still-valid cached leaf always chains to a CA peers accept. + // A new CA is picked up when the leaf expires (forcing a re-mint) or when + // setCACache swaps the cache and clears the leaf. + now := t.clock.Now() + if t.leaf != nil && now.Before(t.leaf.Leaf.NotAfter) { + return t.leaf, nil + } + + id, key, err := t.ca.SigningKey(t.ctx) + if err != nil { + return nil, xerrors.Errorf("get signing CA: %w", err) + } + ca, ok := key.(*cryptokeys.NATSCA) + if !ok { + return nil, xerrors.Errorf("unexpected signing key type %T", key) + } + + leaf, err := mintLeaf(ca, t.ip, now) + if err != nil { + return nil, xerrors.Errorf("mint leaf: %w", err) + } + t.leaf = leaf + t.logger.Debug(t.ctx, "minted nats cluster leaf", slog.F("ca_sequence", id)) + return leaf, nil +} + +// mintLeaf creates an ephemeral leaf certificate signed by the active CA. The +// signing CA's sequence is stamped into the leaf's Subject SerialNumber so a +// verifying peer can look up exactly that CA (see verifyConnection), and the +// replica's relay IP is embedded as an IP SAN so a dialing peer can confirm it +// reached the host it intended. The leaf is usable as both a TLS server and +// client certificate because each replica both accepts and dials cluster +// routes. +func mintLeaf(ca *cryptokeys.NATSCA, ip net.IP, now time.Time) (*tls.Certificate, error) { + if len(ip) == 0 { + return nil, xerrors.New("leaf IP SAN is required") + } + + leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, xerrors.Errorf("generate leaf key: %w", err) + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), leafSerialBits)) + if err != nil { + return nil, xerrors.Errorf("generate serial: %w", err) + } + + // A leaf is only ever used to authenticate a handshake, so it need only be + // valid as long as the CA that signed it. Tie the leaf's NotAfter to the + // CA's so a leaf never outlives its CA and carries no independent lifetime. + // An expired active CA means a fully-dead rotator; fail loud rather than + // mint a dead leaf. + if !ca.Cert.NotAfter.After(now) { + return nil, xerrors.Errorf("signing CA (seq %d) is expired: NotAfter %s", + ca.Sequence, ca.Cert.NotAfter) + } + notAfter := ca.Cert.NotAfter + + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: "coder-nats-cluster-leaf", + // SerialNumber carries the sequence of the CA that signed this + // leaf, letting a verifier fetch exactly that CA from its cache. + SerialNumber: strconv.FormatInt(int64(ca.Sequence), 10), + }, + IPAddresses: []net.IP{ip}, + NotBefore: now.Add(-clockSkewToleranceTLS), + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + + leafDER, err := x509.CreateCertificate(rand.Reader, template, ca.Cert, &leafKey.PublicKey, ca.Key) + if err != nil { + return nil, xerrors.Errorf("create leaf certificate: %w", err) + } + leaf, err := x509.ParseCertificate(leafDER) + if err != nil { + return nil, xerrors.Errorf("parse leaf certificate: %w", err) + } + + return &tls.Certificate{ + Certificate: [][]byte{leafDER}, + PrivateKey: leafKey, + Leaf: leaf, + }, nil +} + +// verify verifies a peer's leaf certificate. It reads the signing CA sequence +// the peer stamped into its leaf, fetches that exact CA from the cache, and +// confirms the leaf chains to it. Using the stamped sequence is not a trust +// decision: the leaf must still chain to OUR trusted copy of that CA, and a CA +// that has been retired is no longer returned by the cache, so leaves from a +// deleted CA are rejected. +// +// It then enforces source binding: when sourceIP is set (the accept side, where +// the dialing peer's connection address is available), the leaf must carry that +// source IP as an IP SAN, binding the certificate to the network origin. Go's +// default hostname verification, which InsecureSkipVerify disables, cannot do +// this because Go does not populate cs.ServerName for IP-based routes. On the +// dial side sourceIP is nil (Go does not expose the connection in the +// client-certificate callbacks), so only the chain is verified there. +func (t *clusterTLS) verify(cs tls.ConnectionState, sourceIP net.IP) error { + if len(cs.PeerCertificates) == 0 { + return xerrors.New("no peer certificate presented") + } + leaf := cs.PeerCertificates[0] + + seq := leaf.Subject.SerialNumber + if seq == "" { + return xerrors.New("peer leaf missing signing CA sequence") + } + + key, err := t.caCache().VerifyingKey(t.ctx, seq) + if err != nil { + return xerrors.Errorf("get CA for sequence %q: %w", seq, err) + } + ca, ok := key.(*cryptokeys.NATSCA) + if !ok { + return xerrors.Errorf("unexpected verifying key type %T", key) + } + + // Leaves carry both ServerAuth and ClientAuth, since each replica is both a + // route server and client. Requiring those specific usages rejects a leaf + // with some unexpected EKU rather than accepting any usage. + if _, err := leaf.Verify(x509.VerifyOptions{ + Roots: t.verifyPool(seq, ca.Cert), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + CurrentTime: t.clock.Now(), + }); err != nil { + return xerrors.Errorf("verify peer leaf against CA sequence %q: %w", seq, err) + } + + // On the accept side, confirm the leaf's IP SAN matches the address the + // peer actually connected from. + if len(sourceIP) != 0 && !slices.ContainsFunc(leaf.IPAddresses, sourceIP.Equal) { + return xerrors.Errorf("peer leaf IP SANs %v do not match source IP %s", leaf.IPAddresses, sourceIP) + } + return nil +} + +// verifyPool returns the root pool used to verify a peer leaf minted under the +// given CA sequence, building it once and caching it for reuse. It is called +// from verify on every route handshake; cluster routes are long-lived, so a +// handshake is a rare event, and the common case here is a cache hit (a single +// map lookup). +// +// A miss occurs only the first time a sequence is seen (startup, and once per +// CA rotation), which is the only moment the map can grow, so pruning of expired +// entries is attached to the miss path rather than run on every handshake. An +// entry is dropped once the clock passes the CA cert's NotAfter: no valid leaf +// can chain to an expired CA, and the CA outlives every leaf it signed, so this +// is always safe and bounds the map across rotations. +func (t *clusterTLS) verifyPool(seq string, cert *x509.Certificate) *x509.CertPool { + t.mu.Lock() + defer t.mu.Unlock() + + if cp, ok := t.verifyPools[seq]; ok { + return cp.pool + } + + now := t.clock.Now() + for s, cp := range t.verifyPools { + if now.After(cp.notAfter) { + delete(t.verifyPools, s) + } + } + + pool := x509.NewCertPool() + pool.AddCert(cert) + if t.verifyPools == nil { + t.verifyPools = map[string]cachedVerifyPool{} + } + t.verifyPools[seq] = cachedVerifyPool{pool: pool, notAfter: cert.NotAfter} + return pool +} diff --git a/coderd/x/nats/tls_internal_test.go b/coderd/x/nats/tls_internal_test.go new file mode 100644 index 0000000000000..1614c5c844d54 --- /dev/null +++ b/coderd/x/nats/tls_internal_test.go @@ -0,0 +1,530 @@ +package nats + +import ( + "context" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "strconv" + "testing" + "time" + + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/stretchr/testify/require" + + "cdr.dev/slog/v3/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/cryptokeys" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/testutil" + "github.com/coder/quartz" +) + +// fakeCACache is an in-memory cryptokeys.SigningKeycache for tests. active is returned +// by SigningKey (the CA this replica mints leaves under); byID is consulted by +// VerifyingKey (the CAs this replica trusts when verifying peers). +type fakeCACache struct { + active *cryptokeys.NATSCA + byID map[string]*cryptokeys.NATSCA +} + +func (f *fakeCACache) SigningKey(context.Context) (string, interface{}, error) { + if f.active == nil { + return "", nil, cryptokeys.ErrKeyNotFound + } + return strconv.FormatInt(int64(f.active.Sequence), 10), f.active, nil +} + +func (f *fakeCACache) VerifyingKey(_ context.Context, id string) (interface{}, error) { + ca, ok := f.byID[id] + if !ok { + return nil, cryptokeys.ErrKeyNotFound + } + return ca, nil +} + +func (*fakeCACache) Close() error { return nil } + +func generateTestCA(t *testing.T, sequence int32) *cryptokeys.NATSCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + now := time.Now() + template := &x509.Certificate{ + SerialNumber: big.NewInt(int64(sequence)), + Subject: pkix.Name{CommonName: "coder-nats-ca-test"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(72 * time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + require.NoError(t, err) + cert, err := x509.ParseCertificate(der) + require.NoError(t, err) + + return &cryptokeys.NATSCA{Sequence: sequence, Cert: cert, Key: crypto.Signer(key)} +} + +// newTLSPubsub builds a clustered pubsub whose route listener requires mTLS, +// using the supplied CA cache. ip is this node's cluster host: the route +// listener bind host and the leaf IP SAN. Peers dial each other on that host +// (clusterRouteAddress), so ip must be 127.0.0.1 for routes to form. +func newTLSPubsub(t *testing.T, ca cryptokeys.SigningKeycache, ip net.IP) *Pubsub { + t.Helper() + logger := slogtest.Make(t, nil) + ctx := testutil.Context(t, testutil.WaitLong) + ps, err := New(ctx, logger, Options{ + ClusterHost: ip.String(), + ClusterPort: natsserver.RANDOM_PORT, + disableCluster: false, + ClusterCA: ca, + clusterTLSTimeout: testClusterTLSTimeout, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = ps.Close() }) + return ps +} + +// setLeafSAN overrides a node's leaf IP SAN after construction, for tests that +// need the minted SAN to differ from the loopback address the node binds and +// connects on (which are otherwise both the node's ClusterHost). Clearing the +// cached leaf forces the next handshake to re-mint under the new SAN. +func setLeafSAN(ps *Pubsub, ip net.IP) { + ps.clusterTLS.mu.Lock() + defer ps.clusterTLS.mu.Unlock() + ps.clusterTLS.ip = ip + ps.clusterTLS.leaf = nil +} + +func numRoutes(t *testing.T, ps *Pubsub) int { + t.Helper() + routes, err := ps.Server.Routez(&natsserver.RoutezOptions{}) + require.NoError(t, err) + return routes.NumRoutes +} + +// TestPubsub_ClusterTLS validates that the embedded NATS server honors the +// tls.Config callbacks on cluster routes: leaves minted from the CA cache form +// a verified mesh, peers under unrelated CAs are rejected, and peers on either +// side of a CA rotation still verify each other. +func TestPubsub_ClusterTLS(t *testing.T) { + t.Parallel() + + t.Run("Mesh", func(t *testing.T) { + t.Parallel() + + ca := generateTestCA(t, 1) + cache := func() *fakeCACache { + return &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + } + a := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + c := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + + addrA := clusterRouteAddress(t, a) + addrB := clusterRouteAddress(t, b) + addrC := clusterRouteAddress(t, c) + // Full symmetric mesh: every node must know a peer to accept a route + // from it (accept-side membership), so each is given the other two. + // Drive peers through fetchers, as production does: a fetcher re-applies + // the same peers on every refresh, so the startup refresh (which runs + // with the boot-time noop fetcher) cannot race a manual call and wipe + // the route/known-peer set. + a.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrB, addrC}}) + b.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrC}}) + c.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrB}}) + + event := "tls-mesh" + got := make(chan []byte, 8) + cancel, err := c.Subscribe(event, func(_ context.Context, msg []byte) { got <- msg }) + require.NoError(t, err) + defer cancel() + + // Retry publishes until the route subscription has propagated. + require.Eventually(t, func() bool { + if err := b.Publish(event, []byte("hello")); err != nil { + return false + } + if err := b.Flush(); err != nil { + return false + } + select { + case msg := <-got: + return string(msg) == "hello" + case <-time.After(testutil.IntervalMedium): + return false + } + }, testutil.WaitLong, testutil.IntervalFast) + }) + + t.Run("WrongCARejected", func(t *testing.T) { + t.Parallel() + + caX := generateTestCA(t, 1) + caY := generateTestCA(t, 1) + a := newTLSPubsub(t, &fakeCACache{active: caX, byID: map[string]*cryptokeys.NATSCA{"1": caX}}, net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, &fakeCACache{active: caY, byID: map[string]*cryptokeys.NATSCA{"1": caY}}, net.IPv4(127, 0, 0, 1)) + + require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) + + // Each side only trusts its own CA, so the route handshake never + // completes and no route is established. + require.Never(t, func() bool { + return numRoutes(t, a) > 0 || numRoutes(t, b) > 0 + }, testutil.WaitShort, testutil.IntervalFast) + }) + + t.Run("RotationOverlap", func(t *testing.T) { + t.Parallel() + + ca1 := generateTestCA(t, 1) + ca2 := generateTestCA(t, 2) + bundle := map[string]*cryptokeys.NATSCA{"1": ca1, "2": ca2} + // a still mints under the old CA; b has already rotated to the new CA. + // Both trust both CAs, so the mesh forms across the rotation overlap. + a := newTLSPubsub(t, &fakeCACache{active: ca1, byID: bundle}, net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, &fakeCACache{active: ca2, byID: bundle}, net.IPv4(127, 0, 0, 1)) + + // Symmetric peers so each side accepts a route from the other, driven + // through fetchers (see Mesh) so the startup noop refresh cannot race a + // manual call and wipe the route/known-peer set. + a.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, b)}}) + b.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, a)}}) + + require.Eventually(t, func() bool { + return numRoutes(t, a) > 0 && numRoutes(t, b) > 0 + }, testutil.WaitLong, testutil.IntervalFast) + }) + + t.Run("SANMismatch", func(t *testing.T) { + t.Parallel() + + ca := generateTestCA(t, 1) + cache := func() *fakeCACache { + return &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + } + // Both nodes bind and connect on loopback and know each other as peers, + // so the CA and source-membership checks pass. But both mint their leaf + // with a SAN that does not match the loopback address they connect from, + // so every handshake is rejected on the SAN binding alone and no route + // forms. This isolates the SAN check: a valid CA-signed leaf presented + // from a known replica is still rejected when the cert is not bound to + // the address it connects from (e.g. a stolen or mis-minted leaf). + a := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1)) + setLeafSAN(a, net.IPv4(10, 99, 99, 99)) + setLeafSAN(b, net.IPv4(10, 99, 99, 99)) + + require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) + require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)})) + + require.Never(t, func() bool { + return numRoutes(t, a) > 0 || numRoutes(t, b) > 0 + }, testutil.WaitShort, testutil.IntervalFast) + }) + + t.Run("MixedTLSAndPlaintext", func(t *testing.T) { + t.Parallel() + + ca := generateTestCA(t, 1) + // a requires mTLS on its route listener; b is a plaintext node + // (newTestPubsub leaves ClusterCA nil). Routes must not form in either + // direction: a rollout has to enable TLS on every replica at once. + a := newTLSPubsub(t, &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}, net.IPv4(127, 0, 0, 1)) + b := newTestPubsub(t, clusterTestOptions(t)) + + require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) + require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)})) + + require.Never(t, func() bool { + return numRoutes(t, a) > 0 || numRoutes(t, b) > 0 + }, testutil.WaitShort, testutil.IntervalFast) + }) +} + +// TestPubsub_ClusterTLS_CacheSwap covers the Part C optional-mTLS model: a node +// that boots with the noop CA cache forms no route, and swapping in a real cache +// via SetCACache lets routes form over mTLS with no server restart. +func TestPubsub_ClusterTLS_CacheSwap(t *testing.T) { + t.Parallel() + + t.Run("NoopFormsNoRoute", func(t *testing.T) { + t.Parallel() + + ca := generateTestCA(t, 1) + // a boots with the noop cache (production default); b has a real cache. + // a cannot mint a leaf, so its route handshakes fail and no route forms. + a := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}, net.IPv4(127, 0, 0, 1)) + + require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)})) + require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)})) + + require.Never(t, func() bool { + return numRoutes(t, a) > 0 || numRoutes(t, b) > 0 + }, testutil.WaitShort, testutil.IntervalFast) + }) + + t.Run("SwapToRealFormsRoute", func(t *testing.T) { + t.Parallel() + + ca := generateTestCA(t, 1) + realCache := func() *fakeCACache { + return &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + } + // Both boot with the noop cache, then both get the real cache swapped in + // (mirroring the enterprise HA enable path) without a server restart. + a := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, net.IPv4(127, 0, 0, 1)) + b := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, net.IPv4(127, 0, 0, 1)) + + // Drive peers through fetchers, as production does, rather than calling + // setPeerAddresses directly: SetCACache and SetPeerFetcher both trigger + // a peer refresh that reads the current fetcher, so routes converge on + // the fetcher's addresses without racing a manual call. + a.SetCACache(realCache()) + b.SetCACache(realCache()) + + a.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, b)}}) + b.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, a)}}) + + require.Eventually(t, func() bool { + return numRoutes(t, a) > 0 && numRoutes(t, b) > 0 + }, testutil.WaitLong, testutil.IntervalFast) + }) +} + +// TestClusterTLS_configForClient_RequiresSourceIP asserts the accept side fails +// closed when it cannot determine the peer's source IP, rather than skipping the +// source-binding check. +func TestClusterTLS_configForClient_RequiresSourceIP(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ca := generateTestCA(t, 1) + cache := &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + ct := newClusterTLS(ctx, slogtest.Make(t, nil), nil, cache, net.IPv4(127, 0, 0, 1)) + + // No underlying connection: the source IP cannot be determined, so the + // accept-side config is refused and the handshake aborts. + _, err := ct.configForClient(&tls.ClientHelloInfo{}) + require.Error(t, err) +} + +// TestClusterTLS_verify unit-tests the verifier directly, isolating chain +// verification and source-IP binding that the mesh tests exercise only +// indirectly. +func TestClusterTLS_verify(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + ca := generateTestCA(t, 1) + cache := &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}} + + leafIP := net.IPv4(10, 0, 0, 5) + ct := newClusterTLS(ctx, slogtest.Make(t, nil), nil, cache, net.IPv4(10, 0, 0, 1)) + + // A leaf bound to leafIP, signed by the trusted CA. + leafCert, err := mintLeaf(ca, leafIP, time.Now()) + require.NoError(t, err) + leaf, err := x509.ParseCertificate(leafCert.Certificate[0]) + require.NoError(t, err) + cs := tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}} + + t.Run("DialSideChainOnly", func(t *testing.T) { + t.Parallel() + // No source IP (dial side): only the chain is verified. + require.NoError(t, ct.verify(cs, nil)) + }) + + t.Run("AcceptSideSourceMatches", func(t *testing.T) { + t.Parallel() + // Source IP equals the leaf SAN: accepted. + require.NoError(t, ct.verify(cs, leafIP)) + }) + + t.Run("AcceptSideSourceMismatch", func(t *testing.T) { + t.Parallel() + // The leaf is bound to leafIP, so a connection from a different source + // is rejected even though the chain is valid. + err := ct.verify(cs, net.IPv4(10, 0, 0, 1)) + require.ErrorContains(t, err, "do not match source IP") + }) + + t.Run("UntrustedCARejected", func(t *testing.T) { + t.Parallel() + otherCA := generateTestCA(t, 9) + strangerCert, err := mintLeaf(otherCA, leafIP, time.Now()) + require.NoError(t, err) + stranger, err := x509.ParseCertificate(strangerCert.Certificate[0]) + require.NoError(t, err) + // The stamped sequence (9) is not in the cache, so the CA lookup fails. + err = ct.verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{stranger}}, nil) + require.Error(t, err) + }) +} + +// TestClusterTLS_verifyPool asserts the verify-pool cache reuses a pool for a +// given CA sequence and prunes entries whose CA cert has expired, so the map +// does not grow unbounded across rotations. +func TestClusterTLS_verifyPool(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitShort) + clock := quartz.NewMock(t) + clock.Set(time.Now()) + + ca1 := generateTestCA(t, 1) + ca2 := generateTestCA(t, 2) + cache := &fakeCACache{byID: map[string]*cryptokeys.NATSCA{"1": ca1, "2": ca2}} + ct := newClusterTLS(ctx, slogtest.Make(t, nil), clock, cache, net.IPv4(10, 0, 0, 1)) + + // First build for seq 1 caches the pool; a second call returns the same one. + p1 := ct.verifyPool("1", ca1.Cert) + require.Same(t, p1, ct.verifyPool("1", ca1.Cert)) + require.Len(t, ct.verifyPools, 1) + + // Advance past ca1's NotAfter. Building a pool for a new sequence prunes the + // now-expired seq 1 entry, leaving only seq 2. + clock.Set(ca1.Cert.NotAfter.Add(time.Minute)) + ct.verifyPool("2", ca2.Cert) + require.Len(t, ct.verifyPools, 1) + _, ok := ct.verifyPools["1"] + require.False(t, ok, "expired seq 1 pool should be pruned") + _, ok = ct.verifyPools["2"] + require.True(t, ok) +} + +// generateTestCAWithValidity is like generateTestCA but lets a test control the +// CA certificate's NotAfter, so leaf-clamp behavior near CA expiry is testable. +func generateTestCAWithValidity(t *testing.T, sequence int32, notAfter time.Time) *cryptokeys.NATSCA { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := &x509.Certificate{ + SerialNumber: big.NewInt(int64(sequence)), + Subject: pkix.Name{CommonName: "coder-nats-ca-test"}, + NotBefore: notAfter.Add(-90 * 24 * time.Hour), + NotAfter: notAfter, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + require.NoError(t, err) + cert, err := x509.ParseCertificate(der) + require.NoError(t, err) + + return &cryptokeys.NATSCA{Sequence: sequence, Cert: cert, Key: crypto.Signer(key)} +} + +// TestMintLeaf asserts a leaf's NotAfter is exactly its signing CA's NotAfter +// (a leaf carries no independent lifetime), and that minting against an +// already-expired CA fails rather than emitting a dead leaf. +func TestMintLeaf(t *testing.T) { + t.Parallel() + + ip := net.IPv4(127, 0, 0, 1) + + t.Run("MatchesCAValidity", func(t *testing.T) { + t.Parallel() + now := time.Now() + // generateTestCA mints a 72h CA; the leaf's NotAfter tracks it exactly. + ca := generateTestCA(t, 1) + leaf, err := mintLeaf(ca, ip, now) + require.NoError(t, err) + require.WithinDuration(t, ca.Cert.NotAfter, leaf.Leaf.NotAfter, time.Second) + require.WithinDuration(t, now.Add(-clockSkewToleranceTLS), leaf.Leaf.NotBefore, time.Second) + }) + + t.Run("ErrorsWhenCAExpired", func(t *testing.T) { + t.Parallel() + now := time.Now() + // CA's NotAfter is already in the past: no usable leaf can be minted. + ca := generateTestCAWithValidity(t, 1, now.Add(-time.Minute)) + _, err := mintLeaf(ca, ip, now) + require.Error(t, err) + require.ErrorContains(t, err, "expired") + }) +} + +// TestPubsub_ClusterTLS_RealCA stands up a three-node TLS mesh whose trust root +// is a real CA served by the cryptokeys signing cache against a real DB, then +// verifies a cross-route publish/subscribe round-trip. This exercises the +// integration seam between the cryptokeys CA cache and the x/nats cluster TLS +// callbacks, including the real PEM/x509 round-trip that the synthetic +// generateTestCA helper does not cover. Nodes form a direct full mesh to avoid +// depending on multi-hop route gossip. +func TestPubsub_ClusterTLS_RealCA(t *testing.T) { + t.Parallel() + + db, _ := dbtestutil.NewDB(t) + ctx := testutil.Context(t, testutil.WaitLong) + + // Seed an active nats_ca crypto key, mirroring the row the key rotator + // mints in production. The signing cache decodes the PEM secret into a + // *cryptokeys.NATSCA the same way production reads it. + dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureNATSCA, + Sequence: 1, + StartsAt: time.Now().UTC().Add(-time.Hour), + }) + + newNode := func() *Pubsub { + // A real signing cache per node, as each replica builds in coderd.New. + cache, err := cryptokeys.NewSigningCache(ctx, slogtest.Make(t, nil), &cryptokeys.DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA) + require.NoError(t, err) + t.Cleanup(func() { _ = cache.Close() }) + // Nodes mesh on loopback, so the leaf IP SAN must be 127.0.0.1. + return newTLSPubsub(t, cache, net.IPv4(127, 0, 0, 1)) + } + + a := newNode() + b := newNode() + c := newNode() + + addrA := clusterRouteAddress(t, a) + addrB := clusterRouteAddress(t, b) + addrC := clusterRouteAddress(t, c) + // Drive peers through fetchers, as production does, so the startup noop + // refresh cannot race a manual call and wipe the route/known-peer set. + a.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrB, addrC}}) + b.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrC}}) + c.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrB}}) + + received := make(chan string, 4) + cancelSub, err := c.Subscribe("tls-realca", func(_ context.Context, msg []byte) { + select { + case received <- string(msg): + default: + } + }) + require.NoError(t, err) + defer cancelSub() + + // Routes and subscription interest propagate asynchronously after the + // servers report ready, so retry rather than gate on a one-shot check. + require.Eventually(t, func() bool { + if err := b.Publish("tls-realca", []byte("hello")); err != nil { + return false + } + select { + case msg := <-received: + require.Equal(t, "hello", msg) + return true + case <-time.After(testutil.IntervalMedium): + return false + } + }, testutil.WaitLong, testutil.IntervalFast) +} diff --git a/codersdk/deployment.go b/codersdk/deployment.go index 2edd295dbc669..608878b8c2d8e 100644 --- a/codersdk/deployment.go +++ b/codersdk/deployment.go @@ -5512,6 +5512,12 @@ const ( CryptoKeyFeatureWorkspaceAppsToken CryptoKeyFeature = "workspace_apps_token" CryptoKeyFeatureOIDCConvert CryptoKeyFeature = "oidc_convert" CryptoKeyFeatureTailnetResume CryptoKeyFeature = "tailnet_resume" + // CryptoKeyFeatureNATSCA is the CA that signs NATS cluster mTLS leaf + // certificates. Its secret is a PEM cert+key bundle (not a hex secret like + // the other features) and contains a private key, so it must never be + // served over the API. It is deliberately excluded from + // whitelistedCryptoKeyFeatures in enterprise/coderd/workspaceproxy.go. + CryptoKeyFeatureNATSCA CryptoKeyFeature = "nats_ca" ) type CryptoKey struct { diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 4d9124782bd38..3aa7488e41c7d 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -5202,9 +5202,9 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o #### Enumerated Values -| Value(s) | -|------------------------------------------------------------------------------------| -| `oidc_convert`, `tailnet_resume`, `workspace_apps_api_key`, `workspace_apps_token` | +| Value(s) | +|-----------------------------------------------------------------------------------------------| +| `nats_ca`, `oidc_convert`, `tailnet_resume`, `workspace_apps_api_key`, `workspace_apps_token` | ## codersdk.CustomNotificationContent diff --git a/enterprise/cli/server.go b/enterprise/cli/server.go index 22b3ddbf18b8e..ec3bc52d39351 100644 --- a/enterprise/cli/server.go +++ b/enterprise/cli/server.go @@ -8,7 +8,6 @@ import ( "encoding/base64" "errors" "io" - "net/url" "time" "golang.org/x/xerrors" @@ -32,28 +31,11 @@ import ( func (r *RootCmd) Server(_ func()) *serpent.Command { cmd := r.RootCmd.Server(func(ctx context.Context, options *agplcoderd.Options) (*agplcoderd.API, io.Closer, error) { - var ( - derpURL *url.URL - err error - ) - if options.DeploymentValues.DERP.Server.RelayURL.String() != "" { - derpURL, err = url.Parse(options.DeploymentValues.DERP.Server.RelayURL.String()) - if err != nil { - return nil, nil, xerrors.Errorf("derp-server-relay-address must be a valid HTTP URL: %w", err) - } - } - clusterHost := options.DeploymentValues.Cluster.Host.String() - if clusterHost == "" && derpURL != nil { - // Use the DERP host if the operator didn't specify an explicit cluster host, since this is an older setting - // and more likely to be configured by longtime HA customers. - clusterHost = derpURL.Hostname() - } - // Always generate a mesh key, even if the built-in DERP server is // disabled. This mesh key is still used by workspace proxies running // HA. var meshKey string - err = options.Database.InTx(func(tx database.Store) error { + err := options.Database.InTx(func(tx database.Store) error { // This will block until the lock is acquired, and will be // automatically released when the transaction ends. err := tx.AcquireLock(ctx, database.LockIDEnterpriseDeploymentSetup) @@ -107,7 +89,6 @@ func (r *RootCmd) Server(_ func()) *serpent.Command { SCIMAPIKey: []byte(options.DeploymentValues.SCIMAPIKey.Value()), UseLegacySCIM: options.DeploymentValues.UseLegacySCIM.Value(), RBAC: true, - ClusterHost: clusterHost, DERPServerRelayAddress: options.DeploymentValues.DERP.Server.RelayURL.String(), DERPServerRegionID: int(options.DeploymentValues.DERP.Server.RegionID.Value()), ProxyHealthInterval: options.DeploymentValues.ProxyHealthStatusInterval.Value(), diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 9f3860f3ca4b7..28cb1475ef472 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -29,6 +29,7 @@ import ( agplaudit "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/boundaryusage" agplconnectionlog "github.com/coder/coder/v2/coderd/connectionlog" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" agpldbauthz "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/database/dbtime" @@ -801,7 +802,6 @@ type Options struct { // Used for high availability. ReplicaSyncUpdateInterval time.Duration ReplicaErrorGracePeriod time.Duration - ClusterHost string // IP or hostname to reach this specific replica DERPServerRelayAddress string DERPServerRegionID int @@ -1008,6 +1008,9 @@ func (api *API) updateEntitlements(ctx context.Context) error { } if natsPubsub, ok := api.Pubsub.(*nats.Pubsub); ok { + // Swap the real nats_ca CA cache and the replica peer fetcher + // in so the first route handshake can negotiate mTLS. + natsPubsub.SetCACache(api.AGPL.NATSCACache) natsPubsub.SetPeerFetcher(api.replicaManager) api.replicaManager.SetCallback("nats", natsPubsub.RefreshPeers) } @@ -1040,6 +1043,9 @@ func (api *API) updateEntitlements(ctx context.Context) error { if natsPubsub, ok := api.Pubsub.(*nats.Pubsub); ok { natsPubsub.SetPeerFetcher(nats.NopPeerFetcher{}) + // Revert to the noop CA cache: new route handshakes can no + // longer mint a leaf, so the cluster mesh stops forming. + natsPubsub.SetCACache(cryptokeys.NoopSigningKeycache{}) api.replicaManager.SetCallback("nats", nil) } } diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index 41956485521b8..50bb7cf68c519 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -1092,6 +1092,12 @@ func TestGetCryptoKeys(t *testing.T) { require.Error(t, err) require.ErrorAs(t, err, &sdkErr) require.Equal(t, http.StatusBadRequest, sdkErr.StatusCode()) + // The NATS cluster CA bundle contains a private key and must never be + // served to workspace proxies. + _, err = proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeature(database.CryptoKeyFeatureNATSCA)) + require.Error(t, err) + require.ErrorAs(t, err, &sdkErr) + require.Equal(t, http.StatusBadRequest, sdkErr.StatusCode()) _, err = proxy.SDKClient.CryptoKeys(ctx, "invalid") require.Error(t, err) require.ErrorAs(t, err, &sdkErr) diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 84a8a8c3ae4f6..3af72190b414e 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -4044,12 +4044,14 @@ export interface CryptoKey { // From codersdk/deployment.go export type CryptoKeyFeature = + | "nats_ca" | "oidc_convert" | "tailnet_resume" | "workspace_apps_api_key" | "workspace_apps_token"; export const CryptoKeyFeatures: CryptoKeyFeature[] = [ + "nats_ca", "oidc_convert", "tailnet_resume", "workspace_apps_api_key",