fix(hatchery:k8s): add delay before pending workers cleanup (#6107)

richardlt · web-flow · commit 0030f53cdc04 · 2022-03-10T07:50:24.000+01:00
diff --git a/engine/hatchery/kubernetes/kill_workers.go b/engine/hatchery/kubernetes/kill_workers.go
@@ -40,26 +40,29 @@ func (h *HatcheryKubernetes) killAwolWorkers(ctx context.Context) error {
 			continue
 		}
 
-		var toDelete, found bool
-		for _, w := range workers {
-			if workerName, ok := labels[LABEL_WORKER_NAME]; ok && workerName == w.Name {
-				found = true
+		var toDelete bool
+		for _, container := range pod.Status.ContainerStatuses {
+			terminated := (container.State.Terminated != nil && (container.State.Terminated.Reason == "Completed" || container.State.Terminated.Reason == "Error"))
+			errImagePull := (container.State.Waiting != nil && container.State.Waiting.Reason == "ErrImagePull")
+			if terminated || errImagePull {
+				toDelete = true
+				log.Debug(ctx, "pod %s/%s is terminated or in error", pod.Namespace, pod.Name)
 				break
 			}
 		}
-		if !found {
-			toDelete = true
-		}
 
 		if !toDelete {
-			for _, container := range pod.Status.ContainerStatuses {
-				terminated := (container.State.Terminated != nil && (container.State.Terminated.Reason == "Completed" || container.State.Terminated.Reason == "Error"))
-				errImagePull := (container.State.Waiting != nil && container.State.Waiting.Reason == "ErrImagePull")
-				if terminated || errImagePull {
-					toDelete = true
+			var found bool
+			for _, w := range workers {
+				if workerName, ok := labels[LABEL_WORKER_NAME]; ok && workerName == w.Name {
+					found = true
 					break
 				}
 			}
+			if !found && time.Since(pod.CreationTimestamp.Time) > 3*time.Minute {
+				toDelete = true
+				log.Debug(ctx, "pod %s/%s didn't match a registered worker and was started since %v", pod.Namespace, pod.Name, pod.CreationTimestamp.Time)
+			}
 		}
 
 		if toDelete {
@@ -130,6 +133,7 @@ func (h *HatcheryKubernetes) killAwolWorkers(ctx context.Context) error {
 				globalErr = err
 				log.Error(ctx, "hatchery:kubernetes> killAwolWorkers> Cannot delete pod %s (%s)", pod.Name, err)
 			}
+			log.Debug(ctx, "pod %s/%s killed", pod.Namespace, pod.Name)
 		}
 	}
 	return globalErr
diff --git a/engine/hatchery/kubernetes/kill_workers_test.go b/engine/hatchery/kubernetes/kill_workers_test.go
@@ -103,6 +103,17 @@ func TestHatcheryKubernetes_KillAwolWorkers(t *testing.T) {
 					},
 				},
 			},
+			{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "worker-6",
+					Namespace: "cds-workers",
+					Labels: map[string]string{
+						LABEL_HATCHERY_NAME: "my-hatchery",
+						LABEL_WORKER_NAME:   "worker-6",
+					},
+					CreationTimestamp: metav1.Now(),
+				},
+			},
 		},
 	}
 	gock.New("http://lolcat.kube").Get("/api/v1/namespaces/cds-workers/pods").Reply(http.StatusOK).JSON(podsList)

Original file line number	Diff line number	Diff line change
`@@ -40,26 +40,29 @@ func (h *HatcheryKubernetes) killAwolWorkers(ctx context.Context) error {`
`40`	`40`	`continue`
`41`	`41`	`}`
`42`	`42`
`43`		`- var toDelete, found bool`
`44`		`- for _, w := range workers {`
`45`		`- if workerName, ok := labels[LABEL_WORKER_NAME]; ok && workerName == w.Name {`
`46`		`- found = true`
	`43`	`+ var toDelete bool`
	`44`	`+ for _, container := range pod.Status.ContainerStatuses {`
	`45`	`+ terminated := (container.State.Terminated != nil && (container.State.Terminated.Reason == "Completed" \|\| container.State.Terminated.Reason == "Error"))`
	`46`	`+ errImagePull := (container.State.Waiting != nil && container.State.Waiting.Reason == "ErrImagePull")`
	`47`	`+ if terminated \|\| errImagePull {`
	`48`	`+ toDelete = true`
	`49`	`+ log.Debug(ctx, "pod %s/%s is terminated or in error", pod.Namespace, pod.Name)`
`47`	`50`	`break`
`48`	`51`	`}`
`49`	`52`	`}`
`50`		`- if !found {`
`51`		`- toDelete = true`
`52`		`- }`
`53`	`53`
`54`	`54`	`if !toDelete {`
`55`		`- for _, container := range pod.Status.ContainerStatuses {`
`56`		`- terminated := (container.State.Terminated != nil && (container.State.Terminated.Reason == "Completed" \|\| container.State.Terminated.Reason == "Error"))`
`57`		`- errImagePull := (container.State.Waiting != nil && container.State.Waiting.Reason == "ErrImagePull")`
`58`		`- if terminated \|\| errImagePull {`
`59`		`- toDelete = true`
	`55`	`+ var found bool`
	`56`	`+ for _, w := range workers {`
	`57`	`+ if workerName, ok := labels[LABEL_WORKER_NAME]; ok && workerName == w.Name {`
	`58`	`+ found = true`
`60`	`59`	`break`
`61`	`60`	`}`
`62`	`61`	`}`
	`62`	`+ if !found && time.Since(pod.CreationTimestamp.Time) > 3*time.Minute {`
	`63`	`+ toDelete = true`
	`64`	`+ log.Debug(ctx, "pod %s/%s didn't match a registered worker and was started since %v", pod.Namespace, pod.Name, pod.CreationTimestamp.Time)`
	`65`	`+ }`
`63`	`66`	`}`
`64`	`67`
`65`	`68`	`if toDelete {`
`@@ -130,6 +133,7 @@ func (h *HatcheryKubernetes) killAwolWorkers(ctx context.Context) error {`
`130`	`133`	`globalErr = err`
`131`	`134`	`log.Error(ctx, "hatchery:kubernetes> killAwolWorkers> Cannot delete pod %s (%s)", pod.Name, err)`
`132`	`135`	`}`
	`136`	`+ log.Debug(ctx, "pod %s/%s killed", pod.Namespace, pod.Name)`
`133`	`137`	`}`
`134`	`138`	`}`
`135`	`139`	`return globalErr`