Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5468f12

Browse files
vdemeestertekton-robot
vdemeester
authored and
tekton-robot
committed
Do not fail PipelineRun if pvc creation error is because of exceeded quotas
In case of the PVC creation (from volumeclaimtemplate) is due to a quota error (quota exceeded), do not fail with a permanent error, and instead mark the PipelineRun as pending. Once there is some quota available back, it will be able to start. Signed-off-by: Vincent Demeester <[email protected]>
1 parent a217f73 commit 5468f12

File tree

4 files changed

+60
-40
lines changed

4 files changed

+60
-40
lines changed

‎pkg/reconciler/pipelinerun/affinity_assistant.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ const (
5050
)
5151

5252
var (
53-
ErrPvcCreationFailed = errors.New("PVC creation error")
53+
ErrPvcCreationFailed = volumeclaim.ErrPvcCreationFailed
54+
ErrPvcCreationFailedRetryable = volumeclaim.ErrPvcCreationFailedRetryable
5455
ErrAffinityAssistantCreationFailed = errors.New("Affinity Assistant creation error")
5556
)
5657

@@ -95,7 +96,7 @@ func (c *Reconciler) createOrUpdateAffinityAssistantsAndPVCs(ctx context.Context
9596
// To support PVC auto deletion at pipelinerun deletion time, the OwnerReference of the PVCs should be set to the owning pipelinerun instead of the StatefulSets,
9697
// so we create PVCs from PipelineRuns' VolumeClaimTemplate and pass the PVCs to the Affinity Assistant StatefulSet for volume scheduling.
9798
if err := c.pvcHandler.CreatePVCFromVolumeClaimTemplate(ctx, workspace, *kmeta.NewControllerRef(pr), pr.Namespace); err != nil {
98-
return fmt.Errorf("%w: %v", ErrPvcCreationFailed, err) //nolint:errorlint
99+
return err
99100
}
100101
aaName := GetAffinityAssistantName(workspace.Name, pr.Name)
101102
if err := c.createOrUpdateAffinityAssistant(ctx, aaName, pr, nil, []string{claimTemplate.Name}, unschedulableNodes); err != nil {
@@ -114,7 +115,7 @@ func (c *Reconciler) createOrUpdateAffinityAssistantsAndPVCs(ctx context.Context
114115
case aa.AffinityAssistantDisabled:
115116
for _, workspace := range claimTemplateToWorkspace {
116117
if err := c.pvcHandler.CreatePVCFromVolumeClaimTemplate(ctx, workspace, *kmeta.NewControllerRef(pr), pr.Namespace); err != nil {
117-
return fmt.Errorf("%w: %v", ErrPvcCreationFailed, err) //nolint:errorlint
118+
return err
118119
}
119120
}
120121
}

‎pkg/reconciler/pipelinerun/affinity_assistant_test.go‎

Lines changed: 37 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -577,54 +577,56 @@ func TestCreateOrUpdateAffinityAssistantsAndPVCs_Failure(t *testing.T) {
577577
name: "pvc creation failed - per workspace",
578578
failureType: "pvc",
579579
aaBehavior: aa.AffinityAssistantPerWorkspace,
580-
expectedErr: fmt.Errorf("%w: failed to create PVC pvc-b9eea16dce: error creating persistentvolumeclaims", ErrPvcCreationFailed),
580+
expectedErr: fmt.Errorf("%w for pvc-b9eea16dce: error creating persistentvolumeclaims", ErrPvcCreationFailed),
581581
}, {
582582
name: "pvc creation failed - disabled",
583583
failureType: "pvc",
584584
aaBehavior: aa.AffinityAssistantDisabled,
585-
expectedErr: fmt.Errorf("%w: failed to create PVC pvc-b9eea16dce: error creating persistentvolumeclaims", ErrPvcCreationFailed),
585+
expectedErr: fmt.Errorf("%w for pvc-b9eea16dce: error creating persistentvolumeclaims", ErrPvcCreationFailed),
586586
}}
587587

588588
for _, tc := range testCases {
589-
ctx := t.Context()
590-
kubeClientSet := fakek8s.NewSimpleClientset()
591-
c := Reconciler{
592-
KubeClientSet: kubeClientSet,
593-
pvcHandler: volumeclaim.NewPVCHandler(kubeClientSet, zap.NewExample().Sugar()),
594-
}
589+
t.Run(tc.name, func(t *testing.T) {
590+
ctx := t.Context()
591+
kubeClientSet := fakek8s.NewSimpleClientset()
592+
c := Reconciler{
593+
KubeClientSet: kubeClientSet,
594+
pvcHandler: volumeclaim.NewPVCHandler(kubeClientSet, zap.NewExample().Sugar()),
595+
}
595596

596-
switch tc.failureType {
597-
case "pvc":
598-
c.KubeClientSet.CoreV1().(*fake.FakeCoreV1).PrependReactor("create", "persistentvolumeclaims",
599-
func(action testing2.Action) (handled bool, ret runtime.Object, err error) {
600-
return true, &corev1.PersistentVolumeClaim{}, errors.New("error creating persistentvolumeclaims")
601-
})
602-
case "statefulset":
603-
c.KubeClientSet.CoreV1().(*fake.FakeCoreV1).PrependReactor("create", "statefulsets",
604-
func(action testing2.Action) (handled bool, ret runtime.Object, err error) {
605-
return true, &appsv1.StatefulSet{}, errors.New("error creating statefulsets")
606-
})
607-
}
597+
switch tc.failureType {
598+
case "pvc":
599+
c.KubeClientSet.CoreV1().(*fake.FakeCoreV1).PrependReactor("create", "persistentvolumeclaims",
600+
func(action testing2.Action) (handled bool, ret runtime.Object, err error) {
601+
return true, &corev1.PersistentVolumeClaim{}, errors.New("error creating persistentvolumeclaims")
602+
})
603+
case "statefulset":
604+
c.KubeClientSet.CoreV1().(*fake.FakeCoreV1).PrependReactor("create", "statefulsets",
605+
func(action testing2.Action) (handled bool, ret runtime.Object, err error) {
606+
return true, &appsv1.StatefulSet{}, errors.New("error creating statefulsets")
607+
})
608+
}
608609

609-
err := c.createOrUpdateAffinityAssistantsAndPVCs(ctx, testPRWithVolumeClaimTemplate, tc.aaBehavior)
610+
err := c.createOrUpdateAffinityAssistantsAndPVCs(ctx, testPRWithVolumeClaimTemplate, tc.aaBehavior)
610611

611-
if err == nil {
612-
t.Errorf("expect error from createOrUpdateAffinityAssistantsAndPVCs but got nil")
613-
}
612+
if err == nil {
613+
t.Errorf("expect error from createOrUpdateAffinityAssistantsAndPVCs but got nil")
614+
}
614615

615-
switch tc.failureType {
616-
case "pvc":
617-
if !errors.Is(err, ErrPvcCreationFailed) {
618-
t.Errorf("expected err type mismatching, expecting %v but got: %v", ErrPvcCreationFailed, err)
616+
switch tc.failureType {
617+
case "pvc":
618+
if !errors.Is(err, ErrPvcCreationFailed) {
619+
t.Errorf("expected err type mismatching, expecting %v but got: %v", ErrPvcCreationFailed, err)
620+
}
621+
case "statefulset":
622+
if !errors.Is(err, ErrAffinityAssistantCreationFailed) {
623+
t.Errorf("expected err type mismatching, expecting %v but got: %v", ErrAffinityAssistantCreationFailed, err)
624+
}
619625
}
620-
case "statefulset":
621-
if !errors.Is(err, ErrAffinityAssistantCreationFailed) {
622-
t.Errorf("expected err type mismatching, expecting %v but got: %v", ErrAffinityAssistantCreationFailed, err)
626+
if d := cmp.Diff(tc.expectedErr.Error(), err.Error()); d != "" {
627+
t.Errorf("expected err mismatching: %v", diff.PrintWantGot(d))
623628
}
624-
}
625-
if d := cmp.Diff(tc.expectedErr.Error(), err.Error()); d != "" {
626-
t.Errorf("expected err mismatching: %v", diff.PrintWantGot(d))
627-
}
629+
})
628630
}
629631
}
630632

‎pkg/reconciler/pipelinerun/pipelinerun.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -758,6 +758,10 @@ func (c *Reconciler) reconcile(ctx context.Context, pr *v1.PipelineRun, getPipel
758758
pr.Status.MarkFailed(volumeclaim.ReasonCouldntCreateWorkspacePVC,
759759
"Failed to create PVC for PipelineRun %s/%s correctly: %s",
760760
pr.Namespace, pr.Name, err)
761+
case errors.Is(err, ErrPvcCreationFailedRetryable):
762+
logger.Errorf("Failed to create PVC for PipelineRun %s: %v", pr.Name, err)
763+
pr.Status.MarkRunning(ReasonPending, "Waiting for PVC creation to succeed: %v", err)
764+
return err // not a permanent error, will requeue
761765
case errors.Is(err, ErrAffinityAssistantCreationFailed):
762766
logger.Errorf("Failed to create affinity assistant StatefulSet for PipelineRun %s: %v", pr.Name, err)
763767
pr.Status.MarkFailed(ReasonCouldntCreateOrUpdateAffinityAssistantStatefulSet,

‎pkg/reconciler/volumeclaim/pvchandler.go‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ import (
2121
"crypto/sha256"
2222
"encoding/hex"
2323
"encoding/json"
24+
"errors"
2425
"fmt"
26+
"strings"
2527

2628
v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
2729
"go.uber.org/zap"
@@ -39,6 +41,11 @@ const (
3941
ReasonCouldntCreateWorkspacePVC = "CouldntCreateWorkspacePVC"
4042
)
4143

44+
var (
45+
ErrPvcCreationFailed = errors.New("PVC creation error")
46+
ErrPvcCreationFailedRetryable = errors.New("PVC creation error, retryable")
47+
)
48+
4249
// PvcHandler is used to create PVCs for workspaces
4350
type PvcHandler interface {
4451
CreatePVCFromVolumeClaimTemplate(ctx context.Context, wb v1.WorkspaceBinding, ownerReference metav1.OwnerReference, namespace string) error
@@ -73,14 +80,20 @@ func (c *defaultPVCHandler) CreatePVCFromVolumeClaimTemplate(ctx context.Context
7380
if apierrors.IsAlreadyExists(err) {
7481
c.logger.Infof("Tried to create PersistentVolumeClaim %s in namespace %s, but it already exists",
7582
claim.Name, claim.Namespace)
83+
} else if apierrors.IsForbidden(err) {
84+
if strings.Contains(err.Error(), "exceeded quota") {
85+
// This is a retry-able error
86+
return fmt.Errorf("%w: %v", ErrPvcCreationFailedRetryable, err.Error())
87+
}
88+
return fmt.Errorf("%w for %s: %w", ErrPvcCreationFailed, claim.Name, err)
7689
} else {
77-
return fmt.Errorf("failed to create PVC %s: %w", claim.Name, err)
90+
return fmt.Errorf("%w for %s: %w", ErrPvcCreationFailed, claim.Name, err)
7891
}
7992
} else {
8093
c.logger.Infof("Created PersistentVolumeClaim %s in namespace %s", claim.Name, claim.Namespace)
8194
}
8295
case err != nil:
83-
return fmt.Errorf("failed to retrieve PVC %s: %w", claim.Name, err)
96+
return fmt.Errorf("%w: failed to retrieve PVC %s: %w", ErrPvcCreationFailed, claim.Name, err)
8497
}
8598

8699
return nil

0 commit comments

Comments
 (0)