From a820457b098ea3843f976f9ca39849fbf9461a98 Mon Sep 17 00:00:00 2001 From: Meredith Lancaster Date: Mon, 28 Oct 2024 11:47:31 -0600 Subject: [PATCH 1/6] clean up skipped online tests Signed-off-by: Meredith Lancaster --- .../verify/verify_integration_test.go | 27 ++++++++ pkg/cmd/attestation/verify/verify_test.go | 69 ------------------- 2 files changed, 27 insertions(+), 69 deletions(-) diff --git a/pkg/cmd/attestation/verify/verify_integration_test.go b/pkg/cmd/attestation/verify/verify_integration_test.go index 4b0f0adfb3e..0b15e823e1d 100644 --- a/pkg/cmd/attestation/verify/verify_integration_test.go +++ b/pkg/cmd/attestation/verify/verify_integration_test.go @@ -83,6 +83,33 @@ func TestVerifyIntegration(t *testing.T) { require.Error(t, err) require.ErrorContains(t, err, "expected SourceRepositoryURI to be https://github.com/fakeowner/fakerepo, got https://github.com/sigstore/sigstore-js") }) + + t.Run("with no matching OIDC issuer", func(t *testing.T) { + opts := publicGoodOpts + opts.OIDCIssuer = "some-other-issuer" + + err := runVerify(&opts) + require.Error(t, err) + require.ErrorContains(t, err, "expected Issuer to be some-other-issuer, got https://token.actions.githubusercontent.com") + }) + + t.Run("with invalid SAN", func(t *testing.T) { + opts := publicGoodOpts + opts.SAN = "fake san" + + err := runVerify(&opts) + require.Error(t, err) + require.ErrorContains(t, err, "verifying with issuer \"sigstore.dev\"") + }) + + t.Run("with invalid SAN regex", func(t *testing.T) { + opts := publicGoodOpts + opts.SANRegex = "^https://github.com/sigstore/not-real/" + + err := runVerify(&opts) + require.Error(t, err) + require.ErrorContains(t, err, "verifying with issuer \"sigstore.dev\"") + }) } func TestVerifyIntegrationCustomIssuer(t *testing.T) { diff --git a/pkg/cmd/attestation/verify/verify_test.go b/pkg/cmd/attestation/verify/verify_test.go index 306ff8b3504..93ad4bdbcbc 100644 --- a/pkg/cmd/attestation/verify/verify_test.go +++ b/pkg/cmd/attestation/verify/verify_test.go @@ -453,75 +453,6 @@ func TestRunVerify(t *testing.T) { require.ErrorContains(t, err, "failed to fetch attestations from wrong-owner") }) - // TODO: this test can only be tested with a live SigstoreVerifier - // add integration tests or HTTP mocked sigstore verifier tests - // to test this case - t.Run("with invalid OIDC issuer", func(t *testing.T) { - t.Skip() - opts := publicGoodOpts - opts.OIDCIssuer = "not-a-real-issuer" - require.Error(t, runVerify(&opts)) - }) - - // TODO: this test can only be tested with a live SigstoreVerifier - // add integration tests or HTTP mocked sigstore verifier tests - // to test this case - t.Run("with SAN enforcement", func(t *testing.T) { - t.Skip() - opts := Options{ - ArtifactPath: artifactPath, - BundlePath: bundlePath, - APIClient: api.NewTestClient(), - DigestAlgorithm: "sha512", - Logger: logger, - OIDCIssuer: verification.GitHubOIDCIssuer, - Owner: "sigstore", - SAN: SigstoreSanValue, - SigstoreVerifier: verification.NewMockSigstoreVerifier(t), - } - require.Nil(t, runVerify(&opts)) - }) - - // TODO: this test can only be tested with a live SigstoreVerifier - // add integration tests or HTTP mocked sigstore verifier tests - // to test this case - t.Run("with invalid SAN", func(t *testing.T) { - t.Skip() - opts := publicGoodOpts - opts.SAN = "fake san" - require.Error(t, runVerify(&opts)) - }) - - // TODO: this test can only be tested with a live SigstoreVerifier - // add integration tests or HTTP mocked sigstore verifier tests - // to test this case - t.Run("with SAN regex enforcement", func(t *testing.T) { - t.Skip() - opts := publicGoodOpts - opts.SANRegex = SigstoreSanRegex - require.Nil(t, runVerify(&opts)) - }) - - // TODO: this test can only be tested with a live SigstoreVerifier - // add integration tests or HTTP mocked sigstore verifier tests - // to test this case - t.Run("with invalid SAN regex", func(t *testing.T) { - t.Skip() - opts := publicGoodOpts - opts.SANRegex = "^https://github.com/sigstore/not-real/" - require.Error(t, runVerify(&opts)) - }) - - // TODO: this test can only be tested with a live SigstoreVerifier - // add integration tests or HTTP mocked sigstore verifier tests - // to test this case - t.Run("with no matching OIDC issuer", func(t *testing.T) { - t.Skip() - opts := publicGoodOpts - opts.OIDCIssuer = "some-other-issuer" - require.Error(t, runVerify(&opts)) - }) - t.Run("with missing API client", func(t *testing.T) { customOpts := publicGoodOpts customOpts.APIClient = nil From ce5bde4379e4ce7c9ba3ba520e39d7b14258c98d Mon Sep 17 00:00:00 2001 From: Meredith Lancaster Date: Mon, 28 Oct 2024 12:59:04 -0600 Subject: [PATCH 2/6] simplify signer workflow validation tests Signed-off-by: Meredith Lancaster --- pkg/cmd/attestation/verify/policy_test.go | 44 ++++++++++------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/pkg/cmd/attestation/verify/policy_test.go b/pkg/cmd/attestation/verify/policy_test.go index ae1e5295554..bc831e46d09 100644 --- a/pkg/cmd/attestation/verify/policy_test.go +++ b/pkg/cmd/attestation/verify/policy_test.go @@ -36,18 +36,28 @@ func TestValidateSignerWorkflow(t *testing.T) { providedSignerWorkflow string expectedWorkflowRegex string host string + expectErr bool + errContains string } testcases := []testcase{ { name: "workflow with no host specified", providedSignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml", + expectErr: true, + errContains: "unknown host", + }, + { + name: "workflow with default host", + providedSignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml", expectedWorkflowRegex: "^https://github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml", + host: "github.com", }, { - name: "workflow with host specified", + name: "workflow with workflow URL included", providedSignerWorkflow: "github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml", expectedWorkflowRegex: "^https://github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml", + host: "github.com", }, { name: "workflow with GH_HOST set", @@ -61,12 +71,6 @@ func TestValidateSignerWorkflow(t *testing.T) { expectedWorkflowRegex: "^https://authedhost.github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml", host: "authedhost.github.com", }, - { - name: "workflow with authenticated host", - providedSignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml", - expectedWorkflowRegex: "^https://authedhost.github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml", - host: "authedhost.github.com", - }, } for _, tc := range testcases { @@ -78,28 +82,16 @@ func TestValidateSignerWorkflow(t *testing.T) { } // All host resolution is done verify.go:RunE - if tc.host == "" { - // Set to default host - tc.host = "github.com" - } opts.Hostname = tc.host - workflowRegex, err := validateSignerWorkflow(opts) - require.NoError(t, err) require.Equal(t, tc.expectedWorkflowRegex, workflowRegex) + if tc.expectErr { + require.Error(t, err) + require.ErrorContains(t, err, tc.errContains) + } else { + require.NoError(t, err) + require.Equal(t, tc.expectedWorkflowRegex, workflowRegex) + } } } - -func TestValidateSignerWorkflowNoHost(t *testing.T) { - cmdFactory := factory.New("test") - opts := &Options{ - Config: cmdFactory.Config, - SignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml", - } - - workflowRegex, err := validateSignerWorkflow(opts) - require.Error(t, err) - require.ErrorContains(t, err, "unknown host") - require.Equal(t, "", workflowRegex) -} From f8b0f5e68738623a0bf3dc31718adb1be3a3cd58 Mon Sep 17 00:00:00 2001 From: Meredith Lancaster Date: Mon, 28 Oct 2024 13:02:12 -0600 Subject: [PATCH 3/6] clean up test Signed-off-by: Meredith Lancaster --- pkg/cmd/attestation/verify/policy_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/cmd/attestation/verify/policy_test.go b/pkg/cmd/attestation/verify/policy_test.go index bc831e46d09..39e5d37073d 100644 --- a/pkg/cmd/attestation/verify/policy_test.go +++ b/pkg/cmd/attestation/verify/policy_test.go @@ -74,10 +74,8 @@ func TestValidateSignerWorkflow(t *testing.T) { } for _, tc := range testcases { - cmdFactory := factory.New("test") - opts := &Options{ - Config: cmdFactory.Config, + Config: factory.New("test").Config, SignerWorkflow: tc.providedSignerWorkflow, } From 502856082e7a1afda89b57da8040ce5d31cbba27 Mon Sep 17 00:00:00 2001 From: Meredith Lancaster Date: Mon, 28 Oct 2024 13:40:23 -0600 Subject: [PATCH 4/6] table tests Signed-off-by: Meredith Lancaster --- .../verification/sigstore_integration_test.go | 90 +++++++++---------- 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/pkg/cmd/attestation/verification/sigstore_integration_test.go b/pkg/cmd/attestation/verification/sigstore_integration_test.go index 1d3ec2d75d8..d4a92c54106 100644 --- a/pkg/cmd/attestation/verification/sigstore_integration_test.go +++ b/pkg/cmd/attestation/verification/sigstore_integration_test.go @@ -15,32 +15,52 @@ import ( ) func TestLiveSigstoreVerifier(t *testing.T) { - t.Run("with invalid signature", func(t *testing.T) { - attestations := getAttestationsFor(t, "../test/data/sigstoreBundle-invalid-signature.json") - require.NotNil(t, attestations) - + type testcase struct { + name string + attestations []*api.Attestation + expectErr bool + errContains string + } + + testcases := []testcase{ + { + name: "with invalid signature", + attestations: getAttestationsFor(t, "../test/data/sigstoreBundle-invalid-signature.json"), + expectErr: true, + errContains: "verifying with issuer \"sigstore.dev\"", + }, + { + name: "with valid artifact and JSON lines file containing multiple Sigstore bundles", + attestations: getAttestationsFor(t, "../test/data/sigstore-js-2.1.0_with_2_bundles.jsonl"), + }, + { + name: "with invalid bundle version", + attestations: getAttestationsFor(t, "../test/data/sigstore-js-2.1.0-bundle-v0.1.json"), + expectErr: true, + errContains: "unsupported bundle version", + }, + { + name: "with no attestations", + attestations: []*api.Attestation{}, + }, + } + + for _, tc := range testcases { verifier := NewLiveSigstoreVerifier(SigstoreConfig{ Logger: io.NewTestHandler(), }) - res := verifier.Verify(attestations, publicGoodPolicy(t)) - require.Error(t, res.Error) - require.ErrorContains(t, res.Error, "verifying with issuer \"sigstore.dev\"") - require.Nil(t, res.VerifyResults) - }) - - t.Run("with valid artifact and JSON lines file containing multiple Sigstore bundles", func(t *testing.T) { - attestations := getAttestationsFor(t, "../test/data/sigstore-js-2.1.0_with_2_bundles.jsonl") - require.Len(t, attestations, 2) - - verifier := NewLiveSigstoreVerifier(SigstoreConfig{ - Logger: io.NewTestHandler(), - }) + res := verifier.Verify(tc.attestations, publicGoodPolicy(t)) - res := verifier.Verify(attestations, publicGoodPolicy(t)) - require.Len(t, res.VerifyResults, 2) - require.NoError(t, res.Error) - }) + if tc.expectErr { + require.Error(t, res.Error) + require.ErrorContains(t, res.Error, "verifying with issuer \"sigstore.dev\"") + require.Nil(t, res.VerifyResults) + } else { + require.Len(t, len(tc.attestations), len(res.VerifyResults)) + require.NoError(t, res.Error) + } + } t.Run("with GitHub Sigstore artifact", func(t *testing.T) { githubArtifactPath := test.NormalizeRelativePath("../test/data/github_provenance_demo-0.0.12-py3-none-any.whl") @@ -72,34 +92,6 @@ func TestLiveSigstoreVerifier(t *testing.T) { require.Len(t, res.VerifyResults, 2) require.NoError(t, res.Error) }) - - t.Run("with invalid bundle version", func(t *testing.T) { - attestations := getAttestationsFor(t, "../test/data/sigstore-js-2.1.0-bundle-v0.1.json") - require.Len(t, attestations, 1) - - verifier := NewLiveSigstoreVerifier(SigstoreConfig{ - Logger: io.NewTestHandler(), - }) - - res := verifier.Verify(attestations, publicGoodPolicy(t)) - require.Len(t, res.VerifyResults, 0) - require.ErrorContains(t, res.Error, "unsupported bundle version") - }) - - t.Run("with no attestations", func(t *testing.T) { - attestations := []*api.Attestation{} - require.Len(t, attestations, 0) - - verifier := NewLiveSigstoreVerifier(SigstoreConfig{ - Logger: io.NewTestHandler(), - TrustedRoot: test.NormalizeRelativePath("../test/data/trusted_root.json"), - }) - - res := verifier.Verify(attestations, publicGoodPolicy(t)) - require.Len(t, res.VerifyResults, 0) - require.NotNil(t, res.Error) - }) - } func publicGoodPolicy(t *testing.T) verify.PolicyBuilder { From 4ec696dacd7a387f658c139a538ac414f2379033 Mon Sep 17 00:00:00 2001 From: Meredith Lancaster Date: Mon, 28 Oct 2024 13:40:48 -0600 Subject: [PATCH 5/6] create common test fixture, organize tests Signed-off-by: Meredith Lancaster --- pkg/cmd/attestation/verify/options_test.go | 112 +++++++++------------ 1 file changed, 46 insertions(+), 66 deletions(-) diff --git a/pkg/cmd/attestation/verify/options_test.go b/pkg/cmd/attestation/verify/options_test.go index b9be054a512..77c0e3b2300 100644 --- a/pkg/cmd/attestation/verify/options_test.go +++ b/pkg/cmd/attestation/verify/options_test.go @@ -13,29 +13,28 @@ var ( publicGoodBundlePath = test.NormalizeRelativePath("../test/data/psigstore-js-2.1.0-bundle.json") ) +var baseOptions = Options{ + ArtifactPath: publicGoodArtifactPath, + BundlePath: publicGoodBundlePath, + DigestAlgorithm: "sha512", + Limit: 1, + Owner: "sigstore", + OIDCIssuer: "some issuer", +} + func TestAreFlagsValid(t *testing.T) { t.Run("has invalid Repo value", func(t *testing.T) { - opts := Options{ - ArtifactPath: publicGoodArtifactPath, - DigestAlgorithm: "sha512", - OIDCIssuer: "some issuer", - Repo: "sigstoresigstore-js", - } + opts := baseOptions + opts.Repo = "sigstoresigstore-js" err := opts.AreFlagsValid() require.Error(t, err) require.ErrorContains(t, err, "invalid value provided for repo") }) - t.Run("invalid limit < 0", func(t *testing.T) { - opts := Options{ - ArtifactPath: publicGoodArtifactPath, - BundlePath: publicGoodBundlePath, - DigestAlgorithm: "sha512", - Owner: "sigstore", - OIDCIssuer: "some issuer", - Limit: 0, - } + t.Run("invalid limit == 0", func(t *testing.T) { + opts := baseOptions + opts.Limit = 0 err := opts.AreFlagsValid() require.Error(t, err) @@ -43,19 +42,43 @@ func TestAreFlagsValid(t *testing.T) { }) t.Run("invalid limit > 1000", func(t *testing.T) { - opts := Options{ - ArtifactPath: publicGoodArtifactPath, - BundlePath: publicGoodBundlePath, - DigestAlgorithm: "sha512", - Owner: "sigstore", - OIDCIssuer: "some issuer", - Limit: 1001, - } + opts := baseOptions + opts.Limit = 1001 err := opts.AreFlagsValid() require.Error(t, err) require.ErrorContains(t, err, "limit 1001 not allowed, must be between 1 and 1000") }) + + t.Run("returns error when UseBundleFromRegistry is true and ArtifactPath is not an OCI path", func(t *testing.T) { + opts := baseOptions + opts.BundlePath = "" + opts.UseBundleFromRegistry = true + + err := opts.AreFlagsValid() + require.Error(t, err) + require.ErrorContains(t, err, "bundle-from-oci flag can only be used with OCI artifact paths") + }) + + t.Run("does not return error when UseBundleFromRegistry is true and ArtifactPath is an OCI path", func(t *testing.T) { + opts := baseOptions + opts.ArtifactPath = "oci://sigstore/sigstore-js:2.1.0" + opts.BundlePath = "" + opts.UseBundleFromRegistry = true + + err := opts.AreFlagsValid() + require.NoError(t, err) + }) + + t.Run("returns error when UseBundleFromRegistry is true and BundlePath is provided", func(t *testing.T) { + opts := baseOptions + opts.ArtifactPath = "oci://sigstore/sigstore-js:2.1.0" + opts.UseBundleFromRegistry = true + + err := opts.AreFlagsValid() + require.Error(t, err) + require.ErrorContains(t, err, "bundle-from-oci flag cannot be used with bundle-path flag") + }) } func TestSetPolicyFlags(t *testing.T) { @@ -116,47 +139,4 @@ func TestSetPolicyFlags(t *testing.T) { require.Equal(t, "sigstore", opts.Owner) require.Equal(t, "^https://github/foo", opts.SANRegex) }) - - t.Run("returns error when UseBundleFromRegistry is true and ArtifactPath is not an OCI path", func(t *testing.T) { - opts := Options{ - ArtifactPath: publicGoodArtifactPath, - DigestAlgorithm: "sha512", - Owner: "sigstore", - UseBundleFromRegistry: true, - Limit: 1, - } - - err := opts.AreFlagsValid() - require.Error(t, err) - require.ErrorContains(t, err, "bundle-from-oci flag can only be used with OCI artifact paths") - }) - - t.Run("does not return error when UseBundleFromRegistry is true and ArtifactPath is an OCI path", func(t *testing.T) { - opts := Options{ - ArtifactPath: "oci://sigstore/sigstore-js:2.1.0", - DigestAlgorithm: "sha512", - OIDCIssuer: "some issuer", - Owner: "sigstore", - UseBundleFromRegistry: true, - Limit: 1, - } - - err := opts.AreFlagsValid() - require.NoError(t, err) - }) - - t.Run("returns error when UseBundleFromRegistry is true and BundlePath is provided", func(t *testing.T) { - opts := Options{ - ArtifactPath: "oci://sigstore/sigstore-js:2.1.0", - BundlePath: publicGoodBundlePath, - DigestAlgorithm: "sha512", - Owner: "sigstore", - UseBundleFromRegistry: true, - Limit: 1, - } - - err := opts.AreFlagsValid() - require.Error(t, err) - require.ErrorContains(t, err, "bundle-from-oci flag cannot be used with bundle-path flag") - }) } From 8a8f224a7ae36e01f15c48898f0f7be7625ba621 Mon Sep 17 00:00:00 2001 From: Meredith Lancaster Date: Mon, 28 Oct 2024 15:28:00 -0600 Subject: [PATCH 6/6] fix test Signed-off-by: Meredith Lancaster --- .../verification/sigstore_integration_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/cmd/attestation/verification/sigstore_integration_test.go b/pkg/cmd/attestation/verification/sigstore_integration_test.go index d4a92c54106..b7057505ef7 100644 --- a/pkg/cmd/attestation/verification/sigstore_integration_test.go +++ b/pkg/cmd/attestation/verification/sigstore_integration_test.go @@ -42,6 +42,8 @@ func TestLiveSigstoreVerifier(t *testing.T) { { name: "with no attestations", attestations: []*api.Attestation{}, + expectErr: true, + errContains: "no attestations were verified", }, } @@ -53,12 +55,12 @@ func TestLiveSigstoreVerifier(t *testing.T) { res := verifier.Verify(tc.attestations, publicGoodPolicy(t)) if tc.expectErr { - require.Error(t, res.Error) - require.ErrorContains(t, res.Error, "verifying with issuer \"sigstore.dev\"") - require.Nil(t, res.VerifyResults) + require.Error(t, res.Error, "test case: %s", tc.name) + require.ErrorContains(t, res.Error, tc.errContains, "test case: %s", tc.name) + require.Nil(t, res.VerifyResults, "test case: %s", tc.name) } else { - require.Len(t, len(tc.attestations), len(res.VerifyResults)) - require.NoError(t, res.Error) + require.Equal(t, len(tc.attestations), len(res.VerifyResults), "test case: %s", tc.name) + require.NoError(t, res.Error, "test case: %s", tc.name) } }