feat: Add leader election

mikesmithgh · Mike Smith · commit 95b3337d87b4 · 2022-03-07T16:21:54.000-05:00
Signed-off-by: Mike Smith &lt;10135646+mjsmith1028@users.noreply.github.com&gt;
diff --git a/deploy/helm/metacontroller/templates/statefulset.yaml b/deploy/helm/metacontroller/templates/statefulset.yaml
@@ -6,7 +6,7 @@ metadata:
   labels:
     {{- include "metacontroller.labels" . | nindent 4 }}
 spec:
-  replicas: 1
+  replicas: {{ .Values.replicas | default 1 }}
   serviceName: ''
   selector:
     matchLabels:
diff --git a/docs/src/guide/configuration.md b/docs/src/guide/configuration.md
@@ -26,5 +26,9 @@ in `manifests/metacontroller.yaml`):
 | `--events-qps` | Rate of events flowing per object (default - 1 event per 5 minutes, e.g. `--events-qps=0.0033`) |
 | `--events-burst` | Number of events allowed to send per object (default 25, e.g. `--events-burst=25`) |
 | `--pprof-address` | Enable pprof and bind to endpoint /debug/pprof, set to 0 to disable pprof serving (default 0, e.g. `--pprof-address=:6060`) |
+| `--leader-election` | Determines whether or not to use leader election when starting metacontroller (default `false`, e.g., `--leader-election`) |
+| `--leader-election-resource-lock` | Determines which resource lock to use for leader election (default `leases`, e.g., `--leader-election-resource-lock=leases`). Valid resource locks are `endpoints`, `configmaps`, `leases`, `endpointsleases`, or `configmapsleases`. See the client-go documentation [leaderelection/resourcelock](https://pkg.go.dev/k8s.io/client-go/tools/leaderelection/resourcelock#pkg-constants) for additional information. |
+| `--leader-election-namespace` | Determines the namespace in which the leader election resource will be created. If metacontroller is running in-cluster, the default leader election namespace is the same namespace as metacontroller. If metacontroller is running out-of-cluster, the default leader election namespace is undefined. If you are running metacontroller out-of-cluster with leader election enabled, you must specify the leader election namespace. (e.g., `--leader-election-namespace=metacontroller`) |
+| `--leader-election-id` | Determines the name of the resource that leader election will use for holding the leader lock. For example, if the leader election id is `metacontroller` and the leader election resource lock is `leases`, then a resource of kind `leases` with metadata.name `metacontroller` will hold the leader lock. (default metacontroller, e.g., `--leader-election-id=metacontroller`) |
 
 Logging flags are being set by `controller-runtime`, more on the meaning of them can be found [here](https://sdk.operatorframework.io/docs/building-operators/golang/references/logging/#overview)
diff --git a/examples/leader-election/README.md b/examples/leader-election/README.md
@@ -0,0 +1,27 @@
+## Leader Election
+
+Metacontroller leverages [controller-runtime's leader election](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/leaderelection). 
+This is used to ensure that multiple replicas of metacontroller can run with only one active pod, for active-passive high availability.
+
+
+### Enable leader election
+Once enabled, metacontroller will attempt to acquire a leader on startup.
+- Add the metacontroller command-line argument leader-election
+  ```
+  args:
+  - --leader-election
+  ```
+- Increase `replicas` to desired count in [values.yaml](../../deploy/helm/values.yaml)
+  ```
+  replicas: 2  
+  ```     
+- See [configuration.md](../../docs/src/guide/configuration.md) for additional configuration arguments.
+
+### Disable leader election
+Once disabled, metacontroller will not attempt to acquire a leader on startup.
+- Omit the metacontroller command-line argument leader-election or set to false.
+  ```
+  args:
+  - --leader-election=false
+  ```
+
diff --git a/examples/leader-election/manifest/args.yaml b/examples/leader-election/manifest/args.yaml
@@ -0,0 +1,16 @@
+# Override args for development mode.
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: metacontroller
+  namespace: metacontroller
+spec:
+  template:
+    spec:
+      containers:
+      - name: metacontroller
+        args:
+        - --zap-devel
+        - --zap-log-level=5
+        - --discovery-interval=5s
+        - --leader-election
diff --git a/examples/leader-election/manifest/kustomization.yaml b/examples/leader-election/manifest/kustomization.yaml
@@ -0,0 +1,5 @@
+bases:
+- ../../../manifests/dev
+patches:
+- args.yaml
+- replicas.yaml
diff --git a/examples/leader-election/manifest/replicas.yaml b/examples/leader-election/manifest/replicas.yaml
@@ -0,0 +1,8 @@
+# Override args for development mode.
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: metacontroller
+  namespace: metacontroller
+spec:
+  replicas: 2
diff --git a/examples/leader-election/test.sh b/examples/leader-election/test.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+cleanup() {
+  exit_code=$?
+  set +e
+  echo "Rollback metacontroller..."
+  kubectl scale statefulset --replicas="$previous_replicas" -n metacontroller metacontroller
+  kubectl rollout undo statefulset metacontroller -n metacontroller
+  kubectl rollout status --watch --timeout=180s statefulset/metacontroller -n metacontroller
+  exit $exit_code
+}
+trap cleanup EXIT
+
+set -ex
+
+success_msg='successfully acquired lease'
+attempt_msg='attempting to acquire leader lease'
+
+previous_replicas=$(kubectl get statefulset metacontroller -n metacontroller -o=jsonpath='{.spec.replicas}')
+kubectl apply -k ./manifest
+kubectl rollout status --watch --timeout=180s statefulset/metacontroller -n metacontroller
+
+# both pods must be ready before checking logs
+kubectl wait --timeout=180s --for=condition=ready pod metacontroller-0 -n metacontroller
+kubectl wait --timeout=180s --for=condition=ready pod metacontroller-1 -n metacontroller
+
+maximum_attempts=36
+# wait for one pod to acquire the leader lease
+until [[ "$(kubectl logs metacontroller-0 -n metacontroller | grep "$success_msg" | wc -l)" -eq 1 ||
+         "$(kubectl logs metacontroller-1 -n metacontroller | grep "$success_msg" | wc -l)" -eq 1 ]]; do
+  sleep 5
+  # timeout at 180s if no leader lease acquired
+  ((maximum_attempts--)) # this will exit with an error when equal to zero
+done
+
+# determine which pods have attempted or successfully acquired the leader lease
+pod0_attempt=$(kubectl logs metacontroller-0 -n metacontroller | grep "$attempt_msg" | wc -l | xargs echo -n)
+pod0_success=$(kubectl logs metacontroller-0 -n metacontroller | grep "$success_msg" | wc -l | xargs echo -n)
+pod1_attempt=$(kubectl logs metacontroller-1 -n metacontroller | grep "$attempt_msg" | wc -l | xargs echo -n)
+pod1_success=$(kubectl logs metacontroller-1 -n metacontroller | grep "$success_msg" | wc -l | xargs echo -n)
+
+echo
+echo "Leader election results:"
+echo "metacontroller-0 leader election attempt count: $pod0_attempt, acquired count: $pod0_success"
+echo "metacontroller-1 leader election attempt count: $pod1_attempt, acquired count: $pod1_success"
+echo
+
+# only one pod should successfully acquire the leader lease
+if [[ $((pod0_success + pod1_success)) -eq 0 ]]; then
+  exit 1
+fi
diff --git a/examples/pprof-profiling/README.md b/examples/pprof-profiling/README.md
diff --git a/examples/prometheus-monitoring/README.md b/examples/prometheus-monitoring/README.md
diff --git a/main.go b/main.go
@@ -24,6 +24,9 @@ import (
 	"sync"
 	"time"
 
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"sigs.k8s.io/controller-runtime/pkg/leaderelection"
+
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
@@ -37,16 +40,20 @@ import (
 )
 
 var (
-	discoveryInterval = flag.Duration("discovery-interval", 30*time.Second, "How often to refresh discovery cache to pick up newly-installed resources")
-	informerRelist    = flag.Duration("cache-flush-interval", 30*time.Minute, "How often to flush local caches and relist objects from the API server")
-	metricsAddr       = flag.String("metrics-address", ":9999", "The address to bind metrics endpoint - /metrics")
-	clientGoQPS       = flag.Float64("client-go-qps", 5, "Number of queries per second client-go is allowed to make (default 5)")
-	clientGoBurst     = flag.Int("client-go-burst", 10, "Allowed burst queries for client-go (default 10)")
-	workers           = flag.Int("workers", 5, "Number of sync workers to run (default 5)")
-	eventsQPS         = flag.Float64("events-qps", 1./300., "Rate of events flowing per object (default - 1 event per 5 minutes)")
-	eventsBurst       = flag.Int("events-burst", 25, "Number of events allowed to send per object (default 25)")
-	pprofAddr         = flag.String("pprof-address", "0", "Enable pprof and bind to endpoint - /debug/pprof, set to 0 to disable pprof serving")
-	version           = "No version provided"
+	discoveryInterval          = flag.Duration("discovery-interval", 30*time.Second, "How often to refresh discovery cache to pick up newly-installed resources")
+	informerRelist             = flag.Duration("cache-flush-interval", 30*time.Minute, "How often to flush local caches and relist objects from the API server")
+	metricsAddr                = flag.String("metrics-address", ":9999", "The address to bind metrics endpoint - /metrics")
+	clientGoQPS                = flag.Float64("client-go-qps", 5, "Number of queries per second client-go is allowed to make (default 5)")
+	clientGoBurst              = flag.Int("client-go-burst", 10, "Allowed burst queries for client-go (default 10)")
+	workers                    = flag.Int("workers", 5, "Number of sync workers to run (default 5)")
+	eventsQPS                  = flag.Float64("events-qps", 1./300., "Rate of events flowing per object (default - 1 event per 5 minutes)")
+	eventsBurst                = flag.Int("events-burst", 25, "Number of events allowed to send per object (default 25)")
+	pprofAddr                  = flag.String("pprof-address", "0", "Enable pprof and bind to endpoint - /debug/pprof, set to 0 to disable pprof serving")
+	leaderElection             = flag.Bool("leader-election", false, "Determines whether or not to use leader election when starting metacontroller")
+	leaderElectionResourceLock = flag.String("leader-election-resource-lock", resourcelock.LeasesResourceLock, "Determines which resource lock to use for leader election")
+	leaderElectionNamespace    = flag.String("leader-election-namespace", "", "Determines the namespace in which the leader election resource will be created")
+	leaderElectionID           = flag.String("leader-election-id", "metacontroller", "Determines the name of the resource that leader election will use for holding the leader lock")
+	version                    = "No version provided"
 )
 
 func main() {
@@ -65,6 +72,10 @@ func main() {
 		"events-qps", *eventsQPS,
 		"events-burst", *eventsBurst,
 		"pprofAddr", *pprofAddr,
+		"leader-election", *leaderElection,
+		"leader-election-resource-lock", *leaderElectionResourceLock,
+		"leader-election-namespace", *leaderElectionNamespace,
+		"leader-election-id", *leaderElectionID,
 		"version", version)
 
 	pprofStopChan := profile.EnablePprof(*pprofAddr)
@@ -87,6 +98,12 @@ func main() {
 			QPS:       float32(*eventsQPS),
 		},
 		MetricsEndpoint: *metricsAddr,
+		LeaderElectionOptions: leaderelection.Options{
+			LeaderElection:             *leaderElection,
+			LeaderElectionResourceLock: *leaderElectionResourceLock,
+			LeaderElectionNamespace:    *leaderElectionNamespace,
+			LeaderElectionID:           *leaderElectionID,
+		},
 	}
 
 	// Create a new manager with a stop function
diff --git a/pkg/options/configuration.go b/pkg/options/configuration.go
@@ -19,15 +19,18 @@ package options
 import (
 	"time"
 
+	"sigs.k8s.io/controller-runtime/pkg/leaderelection"
+
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 )
 
 type Configuration struct {
-	RestConfig        *rest.Config
-	DiscoveryInterval time.Duration
-	InformerRelist    time.Duration
-	Workers           int
-	CorrelatorOptions record.CorrelatorOptions
-	MetricsEndpoint   string
+	RestConfig            *rest.Config
+	DiscoveryInterval     time.Duration
+	InformerRelist        time.Duration
+	Workers               int
+	CorrelatorOptions     record.CorrelatorOptions
+	MetricsEndpoint       string
+	LeaderElectionOptions leaderelection.Options
 }
diff --git a/pkg/server/server.go b/pkg/server/server.go
@@ -67,8 +67,12 @@ func New(configuration options.Configuration) (controllerruntime.Manager, error)
 	mgr, err := controllerruntime.NewManager(configuration.RestConfig, manager.Options{
 		// Disables serving built-in metrics.
 		// We already start a standalone metrics server in parallel to the manager.
-		MetricsBindAddress: configuration.MetricsEndpoint,
-		EventBroadcaster:   controllerContext.Broadcaster,
+		MetricsBindAddress:         configuration.MetricsEndpoint,
+		EventBroadcaster:           controllerContext.Broadcaster,
+		LeaderElection:             configuration.LeaderElectionOptions.LeaderElection,
+		LeaderElectionResourceLock: configuration.LeaderElectionOptions.LeaderElectionResourceLock,
+		LeaderElectionNamespace:    configuration.LeaderElectionOptions.LeaderElectionNamespace,
+		LeaderElectionID:           configuration.LeaderElectionOptions.LeaderElectionID,
 	})
 	if err != nil {
 		return nil, err
