From 664e09fdbcccedc50916ca533acb700e39e7fa9b Mon Sep 17 00:00:00 2001 From: Phill MV Date: Mon, 21 Oct 2024 11:20:46 -0400 Subject: [PATCH 1/5] wip: added test that fails in the absence of a backoff. --- pkg/cmd/attestation/api/client_test.go | 26 +++++++++++++++++++ .../attestation/api/mock_apiClient_test.go | 25 ++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/pkg/cmd/attestation/api/client_test.go b/pkg/cmd/attestation/api/client_test.go index bfcb40f5afc..5544f747698 100644 --- a/pkg/cmd/attestation/api/client_test.go +++ b/pkg/cmd/attestation/api/client_test.go @@ -204,3 +204,29 @@ func TestGetTrustDomain(t *testing.T) { }) } + +func TestGetAttestationsRetries(t *testing.T) { + fetcher := mockDataGenerator{ + NumAttestations: 5, + } + l := io.NewTestHandler() + + c := &LiveClient{ + api: mockAPIClient{ + OnRESTWithNext: fetcher.FlakyOnRESTSuccessWithNextPageHandler(), + }, + logger: l, + } + + attestations, err := c.GetByRepoAndDigest(testRepo, testDigest, DefaultLimit) + require.NoError(t, err) + + // assert the error path was executed; because this is a paged + // request, it should have errored twice + fetcher.AssertNumberOfCalls(t, "FlakyOnRESTSuccessWithNextPage:error", 2) + + // but we still successfully got the right data + require.Equal(t, 10, len(attestations)) + bundle := (attestations)[0].Bundle + require.Equal(t, bundle.GetMediaType(), "application/vnd.dev.sigstore.bundle.v0.3+json") +} diff --git a/pkg/cmd/attestation/api/mock_apiClient_test.go b/pkg/cmd/attestation/api/mock_apiClient_test.go index d58cdbc7941..e2364f0487d 100644 --- a/pkg/cmd/attestation/api/mock_apiClient_test.go +++ b/pkg/cmd/attestation/api/mock_apiClient_test.go @@ -6,6 +6,10 @@ import ( "fmt" "io" "strings" + + cliAPI "github.com/cli/cli/v2/api" + ghAPI "github.com/cli/go-gh/v2/pkg/api" + "github.com/stretchr/testify/mock" ) type mockAPIClient struct { @@ -22,6 +26,7 @@ func (m mockAPIClient) REST(hostname, method, p string, body io.Reader, data int } type mockDataGenerator struct { + mock.Mock NumAttestations int } @@ -40,6 +45,26 @@ func (m mockDataGenerator) OnRESTSuccessWithNextPage(hostname, method, p string, return m.OnRESTWithNextSuccessHelper(hostname, method, p, body, data, false) } +// Returns a func that just calls OnRESTSuccessWithNextPage but half the time +// it returns a 500 error. +func (m *mockDataGenerator) FlakyOnRESTSuccessWithNextPageHandler() func(hostname, method, p string, body io.Reader, data interface{}) (string, error) { + // set up the flake counter + m.On("FlakyOnRESTSuccessWithNextPage:error").Return() + + count := 0 + return func(hostname, method, p string, body io.Reader, data interface{}) (string, error) { + if count%2 == 0 { + m.MethodCalled("FlakyOnRESTSuccessWithNextPage:error") + + count = count + 1 + return "", cliAPI.HTTPError{HTTPError: &ghAPI.HTTPError{StatusCode: 500}} + } else { + count = count + 1 + return m.OnRESTSuccessWithNextPage(hostname, method, p, body, data) + } + } +} + func (m mockDataGenerator) OnRESTWithNextSuccessHelper(hostname, method, p string, body io.Reader, data interface{}, hasNext bool) (string, error) { atts := make([]*Attestation, m.NumAttestations) for j := 0; j < m.NumAttestations; j++ { From efc1c97cf1f1933ac762567f6834ef1f3925ff6f Mon Sep 17 00:00:00 2001 From: Phill MV Date: Mon, 21 Oct 2024 12:10:18 -0400 Subject: [PATCH 2/5] Added constant backoff retry to getAttestations. --- pkg/cmd/attestation/api/client.go | 40 ++++++++++++++++++++++++-- pkg/cmd/attestation/api/client_test.go | 5 ++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/pkg/cmd/attestation/api/client.go b/pkg/cmd/attestation/api/client.go index 460ae3aad45..fb9caf246d6 100644 --- a/pkg/cmd/attestation/api/client.go +++ b/pkg/cmd/attestation/api/client.go @@ -1,11 +1,14 @@ package api import ( + "errors" "fmt" "io" "net/http" "strings" + "time" + "github.com/cenkalti/backoff/v4" "github.com/cli/cli/v2/api" ioconfig "github.com/cli/cli/v2/pkg/cmd/attestation/io" ) @@ -69,6 +72,9 @@ func (c *LiveClient) GetTrustDomain() (string, error) { return c.getTrustDomain(MetaPath) } +// Allow injecting backoff interval in tests. +var getAttestationRetryInterval = time.Millisecond * 200 + func (c *LiveClient) getAttestations(url, name, digest string, limit int) ([]*Attestation, error) { c.logger.VerbosePrintf("Fetching attestations for artifact digest %s\n\n", digest) @@ -87,14 +93,31 @@ func (c *LiveClient) getAttestations(url, name, digest string, limit int) ([]*At var attestations []*Attestation var resp AttestationsResponse var err error + bo := backoff.NewConstantBackOff(getAttestationRetryInterval) + // if no attestation or less than limit, then keep fetching for url != "" && len(attestations) < limit { - url, err = c.api.RESTWithNext(c.host, http.MethodGet, url, nil, &resp) + err = backoff.Retry(func() error { + newURL, err := c.api.RESTWithNext(c.host, http.MethodGet, url, nil, &resp) + + if err != nil { + if shouldRetry(err) { + return err + } else { + return backoff.Permanent(err) + } + } + + url = newURL + attestations = append(attestations, resp.Attestations...) + + return nil + }, backoff.WithMaxRetries(bo, 3)) + + // bail if RESTWithNext errored out if err != nil { return nil, err } - - attestations = append(attestations, resp.Attestations...) } if len(attestations) == 0 { @@ -108,6 +131,17 @@ func (c *LiveClient) getAttestations(url, name, digest string, limit int) ([]*At return attestations, nil } +func shouldRetry(err error) bool { + var httpError api.HTTPError + if errors.As(err, &httpError) { + if httpError.StatusCode >= 500 && httpError.StatusCode <= 599 { + return true + } + } + + return false +} + func (c *LiveClient) getTrustDomain(url string) (string, error) { var resp MetaResponse diff --git a/pkg/cmd/attestation/api/client_test.go b/pkg/cmd/attestation/api/client_test.go index 5544f747698..4751a3ea8de 100644 --- a/pkg/cmd/attestation/api/client_test.go +++ b/pkg/cmd/attestation/api/client_test.go @@ -206,6 +206,9 @@ func TestGetTrustDomain(t *testing.T) { } func TestGetAttestationsRetries(t *testing.T) { + oldInterval := getAttestationRetryInterval + getAttestationRetryInterval = 0 + fetcher := mockDataGenerator{ NumAttestations: 5, } @@ -229,4 +232,6 @@ func TestGetAttestationsRetries(t *testing.T) { require.Equal(t, 10, len(attestations)) bundle := (attestations)[0].Bundle require.Equal(t, bundle.GetMediaType(), "application/vnd.dev.sigstore.bundle.v0.3+json") + + getAttestationRetryInterval = oldInterval } From fafda48905d0a1cbcef19ffcf0c2e2f1679546e2 Mon Sep 17 00:00:00 2001 From: Phill MV Date: Mon, 21 Oct 2024 12:32:57 -0400 Subject: [PATCH 3/5] added test for verifying we do 3 retries when fetching attestations. --- pkg/cmd/attestation/api/client_test.go | 39 +++++++++++++++++-- .../attestation/api/mock_apiClient_test.go | 10 +++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/pkg/cmd/attestation/api/client_test.go b/pkg/cmd/attestation/api/client_test.go index 4751a3ea8de..e6bca440641 100644 --- a/pkg/cmd/attestation/api/client_test.go +++ b/pkg/cmd/attestation/api/client_test.go @@ -212,13 +212,12 @@ func TestGetAttestationsRetries(t *testing.T) { fetcher := mockDataGenerator{ NumAttestations: 5, } - l := io.NewTestHandler() c := &LiveClient{ api: mockAPIClient{ OnRESTWithNext: fetcher.FlakyOnRESTSuccessWithNextPageHandler(), }, - logger: l, + logger: io.NewTestHandler(), } attestations, err := c.GetByRepoAndDigest(testRepo, testDigest, DefaultLimit) @@ -229,9 +228,43 @@ func TestGetAttestationsRetries(t *testing.T) { fetcher.AssertNumberOfCalls(t, "FlakyOnRESTSuccessWithNextPage:error", 2) // but we still successfully got the right data - require.Equal(t, 10, len(attestations)) + require.Equal(t, len(attestations), 10) bundle := (attestations)[0].Bundle require.Equal(t, bundle.GetMediaType(), "application/vnd.dev.sigstore.bundle.v0.3+json") + // same test as above, but for GetByOwnerAndDigest: + attestations, err = c.GetByOwnerAndDigest(testOwner, testDigest, DefaultLimit) + require.NoError(t, err) + + // because we haven't reset the mock, we have added 2 more failed requests + fetcher.AssertNumberOfCalls(t, "FlakyOnRESTSuccessWithNextPage:error", 4) + + require.Equal(t, len(attestations), 10) + bundle = (attestations)[0].Bundle + require.Equal(t, bundle.GetMediaType(), "application/vnd.dev.sigstore.bundle.v0.3+json") + + getAttestationRetryInterval = oldInterval +} + +// test total retries +func TestGetAttestationsMaxRetries(t *testing.T) { + oldInterval := getAttestationRetryInterval + getAttestationRetryInterval = 0 + + fetcher := mockDataGenerator{ + NumAttestations: 5, + } + + c := &LiveClient{ + api: mockAPIClient{ + OnRESTWithNext: fetcher.OnREST500ErrorHandler(), + }, + logger: io.NewTestHandler(), + } + + _, err := c.GetByRepoAndDigest(testRepo, testDigest, DefaultLimit) + require.Error(t, err) + + fetcher.AssertNumberOfCalls(t, "OnREST500Error", 4) getAttestationRetryInterval = oldInterval } diff --git a/pkg/cmd/attestation/api/mock_apiClient_test.go b/pkg/cmd/attestation/api/mock_apiClient_test.go index e2364f0487d..0add67b7579 100644 --- a/pkg/cmd/attestation/api/mock_apiClient_test.go +++ b/pkg/cmd/attestation/api/mock_apiClient_test.go @@ -65,6 +65,16 @@ func (m *mockDataGenerator) FlakyOnRESTSuccessWithNextPageHandler() func(hostnam } } +// always returns a 500 +func (m *mockDataGenerator) OnREST500ErrorHandler() func(hostname, method, p string, body io.Reader, data interface{}) (string, error) { + m.On("OnREST500Error").Return() + return func(hostname, method, p string, body io.Reader, data interface{}) (string, error) { + m.MethodCalled("OnREST500Error") + + return "", cliAPI.HTTPError{HTTPError: &ghAPI.HTTPError{StatusCode: 500}} + } +} + func (m mockDataGenerator) OnRESTWithNextSuccessHelper(hostname, method, p string, body io.Reader, data interface{}, hasNext bool) (string, error) { atts := make([]*Attestation, m.NumAttestations) for j := 0; j < m.NumAttestations; j++ { From e7446676b601f0d7a721c97d8e8fcc4138af313d Mon Sep 17 00:00:00 2001 From: Phill MV Date: Mon, 21 Oct 2024 12:44:51 -0400 Subject: [PATCH 4/5] Minor tweaks, added backoff to getTrustDomain --- pkg/cmd/attestation/api/client.go | 28 ++++++++++++++++++-------- pkg/cmd/attestation/api/client_test.go | 5 ----- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pkg/cmd/attestation/api/client.go b/pkg/cmd/attestation/api/client.go index fb9caf246d6..f47b5f759da 100644 --- a/pkg/cmd/attestation/api/client.go +++ b/pkg/cmd/attestation/api/client.go @@ -92,19 +92,18 @@ func (c *LiveClient) getAttestations(url, name, digest string, limit int) ([]*At var attestations []*Attestation var resp AttestationsResponse - var err error bo := backoff.NewConstantBackOff(getAttestationRetryInterval) // if no attestation or less than limit, then keep fetching for url != "" && len(attestations) < limit { - err = backoff.Retry(func() error { - newURL, err := c.api.RESTWithNext(c.host, http.MethodGet, url, nil, &resp) + err := backoff.Retry(func() error { + newURL, restErr := c.api.RESTWithNext(c.host, http.MethodGet, url, nil, &resp) - if err != nil { - if shouldRetry(err) { - return err + if restErr != nil { + if shouldRetry(restErr) { + return restErr } else { - return backoff.Permanent(err) + return backoff.Permanent(restErr) } } @@ -145,7 +144,20 @@ func shouldRetry(err error) bool { func (c *LiveClient) getTrustDomain(url string) (string, error) { var resp MetaResponse - err := c.api.REST(c.host, http.MethodGet, url, nil, &resp) + bo := backoff.NewConstantBackOff(getAttestationRetryInterval) + err := backoff.Retry(func() error { + restErr := c.api.REST(c.host, http.MethodGet, url, nil, &resp) + if restErr != nil { + if shouldRetry(restErr) { + return restErr + } else { + return backoff.Permanent(restErr) + } + } + + return nil + }, backoff.WithMaxRetries(bo, 3)) + if err != nil { return "", err } diff --git a/pkg/cmd/attestation/api/client_test.go b/pkg/cmd/attestation/api/client_test.go index e6bca440641..adac0059898 100644 --- a/pkg/cmd/attestation/api/client_test.go +++ b/pkg/cmd/attestation/api/client_test.go @@ -206,7 +206,6 @@ func TestGetTrustDomain(t *testing.T) { } func TestGetAttestationsRetries(t *testing.T) { - oldInterval := getAttestationRetryInterval getAttestationRetryInterval = 0 fetcher := mockDataGenerator{ @@ -242,13 +241,10 @@ func TestGetAttestationsRetries(t *testing.T) { require.Equal(t, len(attestations), 10) bundle = (attestations)[0].Bundle require.Equal(t, bundle.GetMediaType(), "application/vnd.dev.sigstore.bundle.v0.3+json") - - getAttestationRetryInterval = oldInterval } // test total retries func TestGetAttestationsMaxRetries(t *testing.T) { - oldInterval := getAttestationRetryInterval getAttestationRetryInterval = 0 fetcher := mockDataGenerator{ @@ -266,5 +262,4 @@ func TestGetAttestationsMaxRetries(t *testing.T) { require.Error(t, err) fetcher.AssertNumberOfCalls(t, "OnREST500Error", 4) - getAttestationRetryInterval = oldInterval } From de4c05fb61c4aec921feb0e09a395eccfaee9ebc Mon Sep 17 00:00:00 2001 From: Phill MV Date: Mon, 21 Oct 2024 14:32:32 -0400 Subject: [PATCH 5/5] Linting: now that mockDataGenerator has an embedded mock, we ought to have pointer receivers in its funcs. --- pkg/cmd/attestation/api/mock_apiClient_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/cmd/attestation/api/mock_apiClient_test.go b/pkg/cmd/attestation/api/mock_apiClient_test.go index 0add67b7579..e1654bd3f7f 100644 --- a/pkg/cmd/attestation/api/mock_apiClient_test.go +++ b/pkg/cmd/attestation/api/mock_apiClient_test.go @@ -30,11 +30,11 @@ type mockDataGenerator struct { NumAttestations int } -func (m mockDataGenerator) OnRESTSuccess(hostname, method, p string, body io.Reader, data interface{}) (string, error) { +func (m *mockDataGenerator) OnRESTSuccess(hostname, method, p string, body io.Reader, data interface{}) (string, error) { return m.OnRESTWithNextSuccessHelper(hostname, method, p, body, data, false) } -func (m mockDataGenerator) OnRESTSuccessWithNextPage(hostname, method, p string, body io.Reader, data interface{}) (string, error) { +func (m *mockDataGenerator) OnRESTSuccessWithNextPage(hostname, method, p string, body io.Reader, data interface{}) (string, error) { // if path doesn't contain after, it means first time hitting the mock server // so return the first page and return the link header in the response if !strings.Contains(p, "after") { @@ -75,7 +75,7 @@ func (m *mockDataGenerator) OnREST500ErrorHandler() func(hostname, method, p str } } -func (m mockDataGenerator) OnRESTWithNextSuccessHelper(hostname, method, p string, body io.Reader, data interface{}, hasNext bool) (string, error) { +func (m *mockDataGenerator) OnRESTWithNextSuccessHelper(hostname, method, p string, body io.Reader, data interface{}, hasNext bool) (string, error) { atts := make([]*Attestation, m.NumAttestations) for j := 0; j < m.NumAttestations; j++ { att := makeTestAttestation() @@ -105,7 +105,7 @@ func (m mockDataGenerator) OnRESTWithNextSuccessHelper(hostname, method, p strin return "", nil } -func (m mockDataGenerator) OnRESTWithNextNoAttestations(hostname, method, p string, body io.Reader, data interface{}) (string, error) { +func (m *mockDataGenerator) OnRESTWithNextNoAttestations(hostname, method, p string, body io.Reader, data interface{}) (string, error) { resp := AttestationsResponse{ Attestations: make([]*Attestation, 0), } @@ -124,7 +124,7 @@ func (m mockDataGenerator) OnRESTWithNextNoAttestations(hostname, method, p stri return "", nil } -func (m mockDataGenerator) OnRESTWithNextError(hostname, method, p string, body io.Reader, data interface{}) (string, error) { +func (m *mockDataGenerator) OnRESTWithNextError(hostname, method, p string, body io.Reader, data interface{}) (string, error) { return "", errors.New("failed to get attestations") }