feat: Add leader election

mikesmithgh · Mike Smith · commit 481f3789dd19 · 2022-03-07T14:41:21.000-05:00
Signed-off-by: Mike Smith &lt;10135646+mjsmith1028@users.noreply.github.com&gt;
diff --git a/deploy/helm/metacontroller/templates/statefulset.yaml b/deploy/helm/metacontroller/templates/statefulset.yaml
@@ -6,7 +6,7 @@ metadata:
   labels:
     {{- include "metacontroller.labels" . | nindent 4 }}
 spec:
-  replicas: 1
+  replicas: {{ .Values.replicas | default 1 }}
   serviceName: ''
   selector:
     matchLabels:
diff --git a/docs/src/guide/configuration.md b/docs/src/guide/configuration.md
@@ -26,5 +26,9 @@ in `manifests/metacontroller.yaml`):
 | `--events-qps` | Rate of events flowing per object (default - 1 event per 5 minutes, e.g. `--events-qps=0.0033`) |
 | `--events-burst` | Number of events allowed to send per object (default 25, e.g. `--events-burst=25`) |
 | `--pprof-address` | Enable pprof and bind to endpoint /debug/pprof, set to 0 to disable pprof serving (default 0, e.g. `--pprof-address=:6060`) |
+| `--leader-election` | Determines whether or not to use leader election when starting metacontroller (default `false`, e.g., `--leader-election`) |
+| `--leader-election-resource-lock` | Determines which resource lock to use for leader election (default `leases`, e.g., `--leader-election-resource-lock=leases`). Valid resource locks are `endpoints`, `configmaps`, `leases`, `endpointsleases`, or `configmapsleases`. See the client-go documentation [leaderelection/resourcelock](https://pkg.go.dev/k8s.io/client-go/tools/leaderelection/resourcelock#pkg-constants) for additional information. |
+| `--leader-election-namespace` | Determines the namespace in which the leader election resource will be created. If metacontroller is running in-cluster, the default leader election namespace is the same namespace as metacontroller. If metacontroller is running out-of-cluster, the default leader election namespace is undefined. If you are running metacontroller out-of-cluster with leader election enabled, you must specify the leader election namespace. (e.g., `--leader-election-namespace=metacontroller`) |
+| `--leader-election-id` | Determines the name of the resource that leader election will use for holding the leader lock. For example, if the leader election id is metacontroller and the leader election resource lock is leases, then a resource of kind `leases` with metadata.name `metacontroller` will hold the leader lock. (default metacontroller, e.g., `--leader-election-id=metacontroller`) |
 
 Logging flags are being set by `controller-runtime`, more on the meaning of them can be found [here](https://sdk.operatorframework.io/docs/building-operators/golang/references/logging/#overview)
diff --git a/examples/leader-election/README.md b/examples/leader-election/README.md
@@ -0,0 +1,27 @@
+## Leader Election
+
+Metacontroller leverages [controller-runtime's leader election](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/leaderelection). 
+This is used to ensure that multiple replicas of metacontroller can run with only one active pod, for active-passive high availability.
+
+
+### Enable leader election
+Once enabled, metacontroller will attempt to acquire a leader on startup.
+- Add the metacontroller command-line argument leader-election
+  ```
+  args:
+  - --leader-election
+  ```
+- Increase `replicas` to desired count in [values.yaml](../../deploy/helm/values.yaml)
+  ```
+  replicas: 2  
+  ```     
+- See [configuration.md](../../docs/src/guide/configuration.md) for additional configuration arguments.
+
+### Disable leader election
+Once disabled, metacontroller will not attempt to acquire a leader on startup.
+- Omit the metacontroller command-line argument leader-election or set to false.
+  ```
+  args:
+  - --leader-election=false
+  ```
+
diff --git a/examples/leader-election/manifest/args.yaml b/examples/leader-election/manifest/args.yaml
@@ -0,0 +1,16 @@
+# Override args for development mode.
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: metacontroller
+  namespace: metacontroller
+spec:
+  template:
+    spec:
+      containers:
+      - name: metacontroller
+        args:
+        - --zap-devel
+        - --zap-log-level=5
+        - --discovery-interval=5s
+        - --leader-election
diff --git a/examples/leader-election/manifest/kustomization.yaml b/examples/leader-election/manifest/kustomization.yaml
@@ -0,0 +1,5 @@
+bases:
+- ../../../manifests/dev
+patches:
+- args.yaml
+- replicas.yaml
diff --git a/examples/leader-election/manifest/replicas.yaml b/examples/leader-election/manifest/replicas.yaml
@@ -0,0 +1,8 @@
+# Override args for development mode.
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: metacontroller
+  namespace: metacontroller
+spec:
+  replicas: 2
diff --git a/examples/leader-election/test.sh b/examples/leader-election/test.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+cleanup() {
+  exit_code=$?
+  set +e
+  echo "Rollback metacontroller..."
+  kubectl scale statefulset --replicas="$previous_replicas" -n metacontroller metacontroller
+  kubectl rollout undo statefulset metacontroller -n metacontroller
+  kubectl rollout status --watch --timeout=180s statefulset/metacontroller -n metacontroller
+  exit $exit_code
+}
+trap cleanup EXIT
+
+set -ex
+
+success_msg='successfully acquired lease'
+attempt_msg='attempting to acquire leader lease'
+
+previous_replicas=$(kubectl get statefulset metacontroller -n metacontroller -o=jsonpath='{.spec.replicas}')
+kubectl apply -k ./manifest
+kubectl rollout status --watch --timeout=180s statefulset/metacontroller -n metacontroller
+
+# both pods must be ready before checking logs
+kubectl wait --timeout=180s --for=condition=ready pod metacontroller-0 -n metacontroller
+kubectl wait --timeout=180s --for=condition=ready pod metacontroller-1 -n metacontroller
+
+maximum_attempts=36
+# wait for one pod to acquire the leader lease
+until [[ "$(kubectl logs metacontroller-0 -n metacontroller | grep "$success_msg" | wc -l)" -eq 1 ||
+         "$(kubectl logs metacontroller-1 -n metacontroller | grep "$success_msg" | wc -l)" -eq 1 ]]; do
+  sleep 5
+  # timeout at 180s if no leader lease acquired
+  ((maximum_attempts--)) # this will exit with an error when equal to zero
+done
+
+# determine which pods have attempted or successfully acquired the leader lease
+pod0_attempt=$(kubectl logs metacontroller-0 -n metacontroller | grep "$attempt_msg" | wc -l | xargs echo -n)
+pod0_success=$(kubectl logs metacontroller-0 -n metacontroller | grep "$success_msg" | wc -l | xargs echo -n)
+pod1_attempt=$(kubectl logs metacontroller-1 -n metacontroller | grep "$attempt_msg" | wc -l | xargs echo -n)
+pod1_success=$(kubectl logs metacontroller-1 -n metacontroller | grep "$success_msg" | wc -l | xargs echo -n)
+
+echo
+echo "Leader election results:"
+echo "metacontroller-0 leader election attempt count: $pod0_attempt, acquired count: $pod0_success"
+echo "metacontroller-1 leader election attempt count: $pod1_attempt, acquired count: $pod1_success"
+echo
+
+# only one pod should successfully acquire the leader lease
+if [[ $((pod0_attempt + pod1_attempt)) -eq 0 ||
+      $((pod0_success + pod1_success)) -eq 0 ]]; then
+  exit 1
+fi
diff --git a/examples/pprof-profiling/README.md b/examples/pprof-profiling/README.md
diff --git a/examples/prometheus-monitoring/README.md b/examples/prometheus-monitoring/README.md
diff --git a/main.go b/main.go
@@ -24,6 +24,9 @@ import (
 	"sync"
 	"time"
 
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"sigs.k8s.io/controller-runtime/pkg/leaderelection"
+
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
@@ -37,16 +40,20 @@ import (
 )
 
 var (
-	discoveryInterval = flag.Duration("discovery-interval", 30*time.Second, "How often to refresh discovery cache to pick up newly-installed resources")
-	informerRelist    = flag.Duration("cache-flush-interval", 30*time.Minute, "How often to flush local caches and relist objects from the API server")
-	metricsAddr       = flag.String("metrics-address", ":9999", "The address to bind metrics endpoint - /metrics")
-	clientGoQPS       = flag.Float64("client-go-qps", 5, "Number of queries per second client-go is allowed to make (default 5)")
-	clientGoBurst     = flag.Int("client-go-burst", 10, "Allowed burst queries for client-go (default 10)")
-	workers           = flag.Int("workers", 5, "Number of sync workers to run (default 5)")
-	eventsQPS         = flag.Float64("events-qps", 1./300., "Rate of events flowing per object (default - 1 event per 5 minutes)")
-	eventsBurst       = flag.Int("events-burst", 25, "Number of events allowed to send per object (default 25)")
-	pprofAddr         = flag.String("pprof-address", "0", "Enable pprof and bind to endpoint - /debug/pprof, set to 0 to disable pprof serving")
-	version           = "No version provided"
+	discoveryInterval          = flag.Duration("discovery-interval", 30*time.Second, "How often to refresh discovery cache to pick up newly-installed resources")
+	informerRelist             = flag.Duration("cache-flush-interval", 30*time.Minute, "How often to flush local caches and relist objects from the API server")
+	metricsAddr                = flag.String("metrics-address", ":9999", "The address to bind metrics endpoint - /metrics")
+	clientGoQPS                = flag.Float64("client-go-qps", 5, "Number of queries per second client-go is allowed to make (default 5)")
+	clientGoBurst              = flag.Int("client-go-burst", 10, "Allowed burst queries for client-go (default 10)")
+	workers                    = flag.Int("workers", 5, "Number of sync workers to run (default 5)")
+	eventsQPS                  = flag.Float64("events-qps", 1./300., "Rate of events flowing per object (default - 1 event per 5 minutes)")
+	eventsBurst                = flag.Int("events-burst", 25, "Number of events allowed to send per object (default 25)")
+	pprofAddr                  = flag.String("pprof-address", "0", "Enable pprof and bind to endpoint - /debug/pprof, set to 0 to disable pprof serving")
+	leaderElection             = flag.Bool("leader-election", false, "Determines whether or not to use leader election when starting metacontroller")
+	leaderElectionResourceLock = flag.String("leader-election-resource-lock", resourcelock.LeasesResourceLock, "Determines which resource lock to use for leader election")
+	leaderElectionNamespace    = flag.String("leader-election-namespace", "", "Determines the namespace in which the leader election resource will be created")
+	leaderElectionID           = flag.String("leader-election-id", "metacontroller", "Determines the name of the resource that leader election will use for holding the leader lock")
+	version                    = "No version provided"
 )
 
 func main() {
@@ -65,6 +72,10 @@ func main() {
 		"events-qps", *eventsQPS,
 		"events-burst", *eventsBurst,
 		"pprofAddr", *pprofAddr,
+		"leader-election", *leaderElection,
+		"leader-election-resource-lock", *leaderElectionResourceLock,
+		"leader-election-namespace", *leaderElectionNamespace,
+		"leader-election-id", *leaderElectionID,
 		"version", version)
 
 	pprofStopChan := profile.EnablePprof(*pprofAddr)
@@ -87,6 +98,12 @@ func main() {
 			QPS:       float32(*eventsQPS),
 		},
 		MetricsEndpoint: *metricsAddr,
+		LeaderElectionOptions: leaderelection.Options{
+			LeaderElection:             *leaderElection,
+			LeaderElectionResourceLock: *leaderElectionResourceLock,
+			LeaderElectionNamespace:    *leaderElectionNamespace,
+			LeaderElectionID:           *leaderElectionID,
+		},
 	}
 
 	// Create a new manager with a stop function
diff --git a/pkg/options/configuration.go b/pkg/options/configuration.go
@@ -19,15 +19,18 @@ package options
 import (
 	"time"
 
+	"sigs.k8s.io/controller-runtime/pkg/leaderelection"
+
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 )
 
 type Configuration struct {
-	RestConfig        *rest.Config
-	DiscoveryInterval time.Duration
-	InformerRelist    time.Duration
-	Workers           int
-	CorrelatorOptions record.CorrelatorOptions
-	MetricsEndpoint   string
+	RestConfig            *rest.Config
+	DiscoveryInterval     time.Duration
+	InformerRelist        time.Duration
+	Workers               int
+	CorrelatorOptions     record.CorrelatorOptions
+	MetricsEndpoint       string
+	LeaderElectionOptions leaderelection.Options
 }
diff --git a/pkg/server/server.go b/pkg/server/server.go
@@ -67,8 +67,12 @@ func New(configuration options.Configuration) (controllerruntime.Manager, error)
 	mgr, err := controllerruntime.NewManager(configuration.RestConfig, manager.Options{
 		// Disables serving built-in metrics.
 		// We already start a standalone metrics server in parallel to the manager.
-		MetricsBindAddress: configuration.MetricsEndpoint,
-		EventBroadcaster:   controllerContext.Broadcaster,
+		MetricsBindAddress:         configuration.MetricsEndpoint,
+		EventBroadcaster:           controllerContext.Broadcaster,
+		LeaderElection:             configuration.LeaderElectionOptions.LeaderElection,
+		LeaderElectionResourceLock: configuration.LeaderElectionOptions.LeaderElectionResourceLock,
+		LeaderElectionNamespace:    configuration.LeaderElectionOptions.LeaderElectionNamespace,
+		LeaderElectionID:           configuration.LeaderElectionOptions.LeaderElectionID,
 	})
 	if err != nil {
 		return nil, err
