Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Clean up skipped online tests for gh attestation verify #9838

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Oct 30, 2024
Merged

Clean up skipped online tests for gh attestation verify #9838

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a820457
clean up skipped online tests
malancas Oct 28, 2024
ce5bde4
simplify signer workflow validation tests
malancas Oct 28, 2024
f8b0f5e
clean up test
malancas Oct 28, 2024
5028560
table tests
malancas Oct 28, 2024
4ec696d
create common test fixture, organize tests
malancas Oct 28, 2024
8a8f224
fix test
malancas Oct 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 43 additions & 49 deletions pkg/cmd/attestation/verification/sigstore_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,54 @@ import (
)

func TestLiveSigstoreVerifier(t *testing.T) {
t.Run("with invalid signature", func(t *testing.T) {
attestations := getAttestationsFor(t, "../test/data/sigstoreBundle-invalid-signature.json")
require.NotNil(t, attestations)

type testcase struct {
name string
attestations []*api.Attestation
expectErr bool
errContains string
}

testcases := []testcase{
{
name: "with invalid signature",
attestations: getAttestationsFor(t, "../test/data/sigstoreBundle-invalid-signature.json"),
expectErr: true,
errContains: "verifying with issuer \"sigstore.dev\"",
},
{
name: "with valid artifact and JSON lines file containing multiple Sigstore bundles",
attestations: getAttestationsFor(t, "../test/data/sigstore-js-2.1.0_with_2_bundles.jsonl"),
},
{
name: "with invalid bundle version",
attestations: getAttestationsFor(t, "../test/data/sigstore-js-2.1.0-bundle-v0.1.json"),
expectErr: true,
errContains: "unsupported bundle version",
},
{
name: "with no attestations",
attestations: []*api.Attestation{},
expectErr: true,
errContains: "no attestations were verified",
},
}

for _, tc := range testcases {
verifier := NewLiveSigstoreVerifier(SigstoreConfig{
Logger: io.NewTestHandler(),
})

res := verifier.Verify(attestations, publicGoodPolicy(t))
require.Error(t, res.Error)
require.ErrorContains(t, res.Error, "verifying with issuer \"sigstore.dev\"")
require.Nil(t, res.VerifyResults)
})

t.Run("with valid artifact and JSON lines file containing multiple Sigstore bundles", func(t *testing.T) {
attestations := getAttestationsFor(t, "../test/data/sigstore-js-2.1.0_with_2_bundles.jsonl")
require.Len(t, attestations, 2)

verifier := NewLiveSigstoreVerifier(SigstoreConfig{
Logger: io.NewTestHandler(),
})
res := verifier.Verify(tc.attestations, publicGoodPolicy(t))

res := verifier.Verify(attestations, publicGoodPolicy(t))
require.Len(t, res.VerifyResults, 2)
require.NoError(t, res.Error)
})
if tc.expectErr {
require.Error(t, res.Error, "test case: %s", tc.name)
require.ErrorContains(t, res.Error, tc.errContains, "test case: %s", tc.name)
require.Nil(t, res.VerifyResults, "test case: %s", tc.name)
} else {
require.Equal(t, len(tc.attestations), len(res.VerifyResults), "test case: %s", tc.name)
require.NoError(t, res.Error, "test case: %s", tc.name)
}
}

t.Run("with GitHub Sigstore artifact", func(t *testing.T) {
githubArtifactPath := test.NormalizeRelativePath("../test/data/github_provenance_demo-0.0.12-py3-none-any.whl")
Expand Down Expand Up @@ -72,34 +94,6 @@ func TestLiveSigstoreVerifier(t *testing.T) {
require.Len(t, res.VerifyResults, 2)
require.NoError(t, res.Error)
})

t.Run("with invalid bundle version", func(t *testing.T) {
attestations := getAttestationsFor(t, "../test/data/sigstore-js-2.1.0-bundle-v0.1.json")
require.Len(t, attestations, 1)

verifier := NewLiveSigstoreVerifier(SigstoreConfig{
Logger: io.NewTestHandler(),
})

res := verifier.Verify(attestations, publicGoodPolicy(t))
require.Len(t, res.VerifyResults, 0)
require.ErrorContains(t, res.Error, "unsupported bundle version")
})

t.Run("with no attestations", func(t *testing.T) {
attestations := []*api.Attestation{}
require.Len(t, attestations, 0)

verifier := NewLiveSigstoreVerifier(SigstoreConfig{
Logger: io.NewTestHandler(),
TrustedRoot: test.NormalizeRelativePath("../test/data/trusted_root.json"),
})

res := verifier.Verify(attestations, publicGoodPolicy(t))
require.Len(t, res.VerifyResults, 0)
require.NotNil(t, res.Error)
})

}

func publicGoodPolicy(t *testing.T) verify.PolicyBuilder {
Expand Down
112 changes: 46 additions & 66 deletions pkg/cmd/attestation/verify/options_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,49 +13,72 @@ var (
publicGoodBundlePath = test.NormalizeRelativePath("../test/data/psigstore-js-2.1.0-bundle.json")
)

var baseOptions = Options{
ArtifactPath: publicGoodArtifactPath,
BundlePath: publicGoodBundlePath,
DigestAlgorithm: "sha512",
Limit: 1,
Owner: "sigstore",
OIDCIssuer: "some issuer",
}

func TestAreFlagsValid(t *testing.T) {
t.Run("has invalid Repo value", func(t *testing.T) {
opts := Options{
ArtifactPath: publicGoodArtifactPath,
DigestAlgorithm: "sha512",
OIDCIssuer: "some issuer",
Repo: "sigstoresigstore-js",
}
opts := baseOptions
opts.Repo = "sigstoresigstore-js"

err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "invalid value provided for repo")
})

t.Run("invalid limit < 0", func(t *testing.T) {
opts := Options{
ArtifactPath: publicGoodArtifactPath,
BundlePath: publicGoodBundlePath,
DigestAlgorithm: "sha512",
Owner: "sigstore",
OIDCIssuer: "some issuer",
Limit: 0,
}
t.Run("invalid limit == 0", func(t *testing.T) {
opts := baseOptions
opts.Limit = 0

err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "limit 0 not allowed, must be between 1 and 1000")
})

t.Run("invalid limit > 1000", func(t *testing.T) {
opts := Options{
ArtifactPath: publicGoodArtifactPath,
BundlePath: publicGoodBundlePath,
DigestAlgorithm: "sha512",
Owner: "sigstore",
OIDCIssuer: "some issuer",
Limit: 1001,
}
opts := baseOptions
opts.Limit = 1001

err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "limit 1001 not allowed, must be between 1 and 1000")
})

t.Run("returns error when UseBundleFromRegistry is true and ArtifactPath is not an OCI path", func(t *testing.T) {
opts := baseOptions
opts.BundlePath = ""
opts.UseBundleFromRegistry = true

err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "bundle-from-oci flag can only be used with OCI artifact paths")
})

t.Run("does not return error when UseBundleFromRegistry is true and ArtifactPath is an OCI path", func(t *testing.T) {
opts := baseOptions
opts.ArtifactPath = "oci://sigstore/sigstore-js:2.1.0"
opts.BundlePath = ""
opts.UseBundleFromRegistry = true

err := opts.AreFlagsValid()
require.NoError(t, err)
})

t.Run("returns error when UseBundleFromRegistry is true and BundlePath is provided", func(t *testing.T) {
opts := baseOptions
opts.ArtifactPath = "oci://sigstore/sigstore-js:2.1.0"
opts.UseBundleFromRegistry = true

err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "bundle-from-oci flag cannot be used with bundle-path flag")
})
}

func TestSetPolicyFlags(t *testing.T) {
Expand Down Expand Up @@ -116,47 +139,4 @@ func TestSetPolicyFlags(t *testing.T) {
require.Equal(t, "sigstore", opts.Owner)
require.Equal(t, "^https://github/foo", opts.SANRegex)
})

t.Run("returns error when UseBundleFromRegistry is true and ArtifactPath is not an OCI path", func(t *testing.T) {
opts := Options{
ArtifactPath: publicGoodArtifactPath,
DigestAlgorithm: "sha512",
Owner: "sigstore",
UseBundleFromRegistry: true,
Limit: 1,
}

err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "bundle-from-oci flag can only be used with OCI artifact paths")
})

t.Run("does not return error when UseBundleFromRegistry is true and ArtifactPath is an OCI path", func(t *testing.T) {
opts := Options{
ArtifactPath: "oci://sigstore/sigstore-js:2.1.0",
DigestAlgorithm: "sha512",
OIDCIssuer: "some issuer",
Owner: "sigstore",
UseBundleFromRegistry: true,
Limit: 1,
}

err := opts.AreFlagsValid()
require.NoError(t, err)
})

t.Run("returns error when UseBundleFromRegistry is true and BundlePath is provided", func(t *testing.T) {
opts := Options{
ArtifactPath: "oci://sigstore/sigstore-js:2.1.0",
BundlePath: publicGoodBundlePath,
DigestAlgorithm: "sha512",
Owner: "sigstore",
UseBundleFromRegistry: true,
Limit: 1,
}

err := opts.AreFlagsValid()
require.Error(t, err)
require.ErrorContains(t, err, "bundle-from-oci flag cannot be used with bundle-path flag")
})
}
48 changes: 19 additions & 29 deletions pkg/cmd/attestation/verify/policy_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,18 +36,28 @@ func TestValidateSignerWorkflow(t *testing.T) {
providedSignerWorkflow string
expectedWorkflowRegex string
host string
expectErr bool
errContains string
}

testcases := []testcase{
{
name: "workflow with no host specified",
providedSignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml",
expectErr: true,
errContains: "unknown host",
},
{
name: "workflow with default host",
providedSignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml",
expectedWorkflowRegex: "^https://github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
host: "github.com",
},
{
name: "workflow with host specified",
name: "workflow with workflow URL included",
providedSignerWorkflow: "github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
expectedWorkflowRegex: "^https://github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
host: "github.com",
},
{
name: "workflow with GH_HOST set",
Expand All @@ -61,45 +71,25 @@ func TestValidateSignerWorkflow(t *testing.T) {
expectedWorkflowRegex: "^https://authedhost.github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
host: "authedhost.github.com",
},
{
name: "workflow with authenticated host",
providedSignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml",
expectedWorkflowRegex: "^https://authedhost.github.com/github/artifact-attestations-workflows/.github/workflows/attest.yml",
host: "authedhost.github.com",
},
}

for _, tc := range testcases {
cmdFactory := factory.New("test")

opts := &Options{
Config: cmdFactory.Config,
Config: factory.New("test").Config,
SignerWorkflow: tc.providedSignerWorkflow,
}

// All host resolution is done verify.go:RunE
if tc.host == "" {
// Set to default host
tc.host = "github.com"
}
opts.Hostname = tc.host

workflowRegex, err := validateSignerWorkflow(opts)
require.NoError(t, err)
require.Equal(t, tc.expectedWorkflowRegex, workflowRegex)

if tc.expectErr {
require.Error(t, err)
require.ErrorContains(t, err, tc.errContains)
} else {
require.NoError(t, err)
require.Equal(t, tc.expectedWorkflowRegex, workflowRegex)
}
}
}

func TestValidateSignerWorkflowNoHost(t *testing.T) {
cmdFactory := factory.New("test")
opts := &Options{
Config: cmdFactory.Config,
SignerWorkflow: "github/artifact-attestations-workflows/.github/workflows/attest.yml",
}

workflowRegex, err := validateSignerWorkflow(opts)
require.Error(t, err)
require.ErrorContains(t, err, "unknown host")
require.Equal(t, "", workflowRegex)
}
27 changes: 27 additions & 0 deletions pkg/cmd/attestation/verify/verify_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,33 @@ func TestVerifyIntegration(t *testing.T) {
require.Error(t, err)
require.ErrorContains(t, err, "expected SourceRepositoryURI to be https://github.com/fakeowner/fakerepo, got https://github.com/sigstore/sigstore-js")
})

t.Run("with no matching OIDC issuer", func(t *testing.T) {
opts := publicGoodOpts
opts.OIDCIssuer = "some-other-issuer"

err := runVerify(&opts)
require.Error(t, err)
require.ErrorContains(t, err, "expected Issuer to be some-other-issuer, got https://token.actions.githubusercontent.com")
})

t.Run("with invalid SAN", func(t *testing.T) {
opts := publicGoodOpts
opts.SAN = "fake san"

err := runVerify(&opts)
require.Error(t, err)
require.ErrorContains(t, err, "verifying with issuer \"sigstore.dev\"")
})

t.Run("with invalid SAN regex", func(t *testing.T) {
opts := publicGoodOpts
opts.SANRegex = "^https://github.com/sigstore/not-real/"

err := runVerify(&opts)
require.Error(t, err)
require.ErrorContains(t, err, "verifying with issuer \"sigstore.dev\"")
})
}

func TestVerifyIntegrationCustomIssuer(t *testing.T) {
Expand Down
Loading