From ab7178cc0aa708459cdf9f3b100999ba1ed12525 Mon Sep 17 00:00:00 2001 From: Marcus Pettersen Irgens Date: Fri, 19 May 2023 18:38:27 +0200 Subject: [PATCH 01/20] Pass BUILDTAGS argument to go build Signed-off-by: Marcus Pettersen Irgens --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fb54b68138d..d6c193951b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN --mount=type=bind,target=/go/src/github.com/docker/distribution,rw \ --mount=type=cache,target=/root/.cache/go-build \ --mount=target=/go/pkg/mod,type=cache \ --mount=type=bind,source=/tmp/.ldflags,target=/tmp/.ldflags,from=version \ - set -x ; xx-go build -trimpath -ldflags "$(cat /tmp/.ldflags) ${LDFLAGS}" -o /usr/bin/registry ./cmd/registry \ + set -x ; xx-go build -trimpath -ldflags "$(cat /tmp/.ldflags) ${LDFLAGS}" -tags="${BUILDTAGS}" -o /usr/bin/registry ./cmd/registry \ && xx-verify --static /usr/bin/registry FROM scratch AS binary From 2548973b1d22e2f1de9b708f90218be8c4efd1b8 Mon Sep 17 00:00:00 2001 From: Milos Gajdos Date: Wed, 28 Jun 2023 11:41:22 +0100 Subject: [PATCH 02/20] Enable Go build tags This enables go build tags so the GCS and OSS driver support is available in the binary distributed via the image build by Dockerfile. This led to quite a few fixes in the GCS and OSS packages raised as warning by golang-ci linter. Signed-off-by: Milos Gajdos (cherry picked from commit 6b388b1ba604bf5b970bbb41878f97cc3fc93376) Signed-off-by: Sebastiaan van Stijn --- BUILDING.md | 2 +- Dockerfile | 4 +-- registry/storage/driver/gcs/gcs.go | 33 ++++++++++++------------- registry/storage/driver/gcs/gcs_test.go | 8 +++++- registry/storage/driver/oss/oss.go | 14 ++++------- registry/storage/driver/oss/oss_test.go | 6 +++++ 6 files changed, 37 insertions(+), 30 deletions(-) diff --git a/BUILDING.md b/BUILDING.md index 2981d016b0d..4c43b03cb7a 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -114,4 +114,4 @@ the registry binary generated in the "./bin" directory: ### Optional build tags Optional [build tags](http://golang.org/pkg/go/build/) can be provided using -the environment variable `DOCKER_BUILDTAGS`. +the environment variable `BUILDTAGS`. diff --git a/Dockerfile b/Dockerfile index d6c193951b5..42b87c064cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,12 +22,12 @@ RUN --mount=target=. \ FROM base AS build ARG TARGETPLATFORM ARG LDFLAGS="-s -w" -ARG BUILDTAGS="include_oss include_gcs" +ARG BUILDTAGS="include_oss,include_gcs" RUN --mount=type=bind,target=/go/src/github.com/docker/distribution,rw \ --mount=type=cache,target=/root/.cache/go-build \ --mount=target=/go/pkg/mod,type=cache \ --mount=type=bind,source=/tmp/.ldflags,target=/tmp/.ldflags,from=version \ - set -x ; xx-go build -trimpath -ldflags "$(cat /tmp/.ldflags) ${LDFLAGS}" -tags="${BUILDTAGS}" -o /usr/bin/registry ./cmd/registry \ + set -x ; xx-go build -tags "${BUILDTAGS}" -trimpath -ldflags "$(cat /tmp/.ldflags) ${LDFLAGS}" -o /usr/bin/registry ./cmd/registry \ && xx-verify --static /usr/bin/registry FROM scratch AS binary diff --git a/registry/storage/driver/gcs/gcs.go b/registry/storage/driver/gcs/gcs.go index 66d0d1ad710..2a6619f5379 100644 --- a/registry/storage/driver/gcs/gcs.go +++ b/registry/storage/driver/gcs/gcs.go @@ -1,18 +1,17 @@ +//go:build include_gcs +// +build include_gcs + // Package gcs provides a storagedriver.StorageDriver implementation to // store blobs in Google cloud storage. // // This package leverages the google.golang.org/cloud/storage client library -//for interfacing with gcs. +// for interfacing with gcs. // // Because gcs is a key, value store the Stat call does not support last modification // time for directories (directories are an abstraction for key, value stores) // // Note that the contents of incomplete uploads are not accessible even though // Stat returns their length -// -//go:build include_gcs -// +build include_gcs - package gcs import ( @@ -62,7 +61,6 @@ var rangeHeader = regexp.MustCompile(`^bytes=([0-9])+-([0-9]+)$`) // driverParameters is a struct that encapsulates all of the driver parameters after all values have been set type driverParameters struct { bucket string - config *jwt.Config email string privateKey []byte client *http.Client @@ -88,6 +86,8 @@ func (factory *gcsDriverFactory) Create(parameters map[string]interface{}) (stor return FromParameters(parameters) } +var _ storagedriver.StorageDriver = &driver{} + // driver is a storagedriver.StorageDriver implementation backed by GCS // Objects are stored at absolute keys in the provided bucket. type driver struct { @@ -298,7 +298,7 @@ func (d *driver) Reader(context context.Context, path string, offset int64) (io. if err != nil { return nil, err } - if offset == int64(obj.Size) { + if offset == obj.Size { return ioutil.NopCloser(bytes.NewReader([]byte{})), nil } return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset} @@ -434,7 +434,6 @@ func putContentsClose(wc *storage.Writer, contents []byte) error { } } if err != nil { - wc.CloseWithError(err) return err } return wc.Close() @@ -614,10 +613,10 @@ func (d *driver) Stat(context context.Context, path string) (storagedriver.FileI //try to get as folder dirpath := d.pathToDirKey(path) - var query *storage.Query - query = &storage.Query{} - query.Prefix = dirpath - query.MaxResults = 1 + query := &storage.Query{ + Prefix: dirpath, + MaxResults: 1, + } objects, err := storageListObjects(gcsContext, d.bucket, query) if err != nil { @@ -639,12 +638,12 @@ func (d *driver) Stat(context context.Context, path string) (storagedriver.FileI } // List returns a list of the objects that are direct descendants of the -//given path. +// given path. func (d *driver) List(context context.Context, path string) ([]string, error) { - var query *storage.Query - query = &storage.Query{} - query.Delimiter = "/" - query.Prefix = d.pathToDirKey(path) + query := &storage.Query{ + Delimiter: "/", + Prefix: d.pathToDirKey(path), + } list := make([]string, 0, 64) for { objects, err := storageListObjects(d.context(context), d.bucket, query) diff --git a/registry/storage/driver/gcs/gcs_test.go b/registry/storage/driver/gcs/gcs_test.go index 4ae9aa3fdda..ee4a67a8f3a 100644 --- a/registry/storage/driver/gcs/gcs_test.go +++ b/registry/storage/driver/gcs/gcs_test.go @@ -59,7 +59,7 @@ func init() { panic(fmt.Sprintf("Error reading JWT config : %s", err)) } email = jwtConfig.Email - privateKey = []byte(jwtConfig.PrivateKey) + privateKey = jwtConfig.PrivateKey if len(privateKey) == 0 { panic("Error reading JWT config : missing private_key property") } @@ -260,6 +260,9 @@ func TestEmptyRootList(t *testing.T) { } }() keys, err := emptyRootDriver.List(ctx, "/") + if err != nil { + t.Fatalf("unexpected error listing empty root content: %v", err) + } for _, path := range keys { if !storagedriver.PathRegexp.MatchString(path) { t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp) @@ -267,6 +270,9 @@ func TestEmptyRootList(t *testing.T) { } keys, err = slashRootDriver.List(ctx, "/") + if err != nil { + t.Fatalf("unexpected error listing slash root content: %v", err) + } for _, path := range keys { if !storagedriver.PathRegexp.MatchString(path) { t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp) diff --git a/registry/storage/driver/oss/oss.go b/registry/storage/driver/oss/oss.go index 8738b1e0c18..55f6edc0e9f 100644 --- a/registry/storage/driver/oss/oss.go +++ b/registry/storage/driver/oss/oss.go @@ -1,3 +1,6 @@ +//go:build include_oss +// +build include_oss + // Package oss provides a storagedriver.StorageDriver implementation to // store blobs in Aliyun OSS cloud storage. // @@ -6,10 +9,6 @@ // // Because OSS is a key, value store the Stat call does not support last modification // time for directories (directories are an abstraction for key, value stores) -// -//go:build include_oss -// +build include_oss - package oss import ( @@ -68,6 +67,8 @@ func (factory *ossDriverFactory) Create(parameters map[string]interface{}) (stor return FromParameters(parameters) } +var _ storagedriver.StorageDriver = &driver{} + type driver struct { Client *oss.Client Bucket *oss.Bucket @@ -498,11 +499,6 @@ func parseError(path string, err error) error { return err } -func hasCode(err error, code string) bool { - ossErr, ok := err.(*oss.Error) - return ok && ossErr.Code == code -} - func (d *driver) getOptions() oss.Options { return oss.Options{ServerSideEncryption: d.Encrypt} } diff --git a/registry/storage/driver/oss/oss_test.go b/registry/storage/driver/oss/oss_test.go index e042bb754fa..aedca987f90 100644 --- a/registry/storage/driver/oss/oss_test.go +++ b/registry/storage/driver/oss/oss_test.go @@ -128,6 +128,9 @@ func TestEmptyRootList(t *testing.T) { defer rootedDriver.Delete(ctx, filename) keys, err := emptyRootDriver.List(ctx, "/") + if err != nil { + t.Fatalf("unexpected error listing empty root content: %v", err) + } for _, path := range keys { if !storagedriver.PathRegexp.MatchString(path) { t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp) @@ -135,6 +138,9 @@ func TestEmptyRootList(t *testing.T) { } keys, err = slashRootDriver.List(ctx, "/") + if err != nil { + t.Fatalf("unexpected error listing slash root content: %v", err) + } for _, path := range keys { if !storagedriver.PathRegexp.MatchString(path) { t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp) From 2d62a4027add9ed41920784cf10bac0b6486a84b Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 21 Aug 2023 13:54:13 +0200 Subject: [PATCH 03/20] s3: add interface assertion This was added for the other drivers in 6b388b1ba604bf5b970bbb41878f97cc3fc93376, but it missed the s3 storage driver. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 5b3be398701517100f417b4949bba6b7c824a7df) Signed-off-by: Sebastiaan van Stijn --- registry/storage/driver/s3-aws/s3.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/registry/storage/driver/s3-aws/s3.go b/registry/storage/driver/s3-aws/s3.go index 77b821f82aa..5f2dd753379 100644 --- a/registry/storage/driver/s3-aws/s3.go +++ b/registry/storage/driver/s3-aws/s3.go @@ -137,6 +137,8 @@ func (factory *s3DriverFactory) Create(parameters map[string]interface{}) (stora return FromParameters(parameters) } +var _ storagedriver.StorageDriver = &driver{} + type driver struct { S3 *s3.S3 Bucket string From 110cb7538dc91e18fdf59893d9274e81cbba7201 Mon Sep 17 00:00:00 2001 From: Milos Gajdos Date: Fri, 19 May 2023 17:51:37 +0100 Subject: [PATCH 04/20] Enable build tags in 2.8 It would appear we were missing the Go build tags on 2.8.X branch so the images would not have the necessary support for some storage drivers causing breakages to end users trying to use them. This commit fixes both the build and linting issues. Signed-off-by: Milos Gajdos --- .github/workflows/ci.yml | 2 +- Makefile | 2 +- registry/storage/driver/oss/oss.go | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae866df9d88..337f1e812af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: build: runs-on: ubuntu-latest env: - DOCKER_BUILDTAGS: "include_oss include_gcs" + BUILDTAGS: "include_oss,include_gcs" CGO_ENABLED: 1 GO111MODULE: "auto" GOPATH: ${{ github.workspace }} diff --git a/Makefile b/Makefile index 75e11820152..dcdbcb54799 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ version/version.go: check: ## run all linters (TODO: enable "unused", "varcheck", "ineffassign", "unconvert", "staticheck", "goimports", "structcheck") @echo "$(WHALE) $@" - @GO111MODULE=off golangci-lint run + @GO111MODULE=off golangci-lint --build-tags "${BUILDTAGS}" run test: ## run tests, except integration test with test.short @echo "$(WHALE) $@" diff --git a/registry/storage/driver/oss/oss.go b/registry/storage/driver/oss/oss.go index 55f6edc0e9f..dfe6fd966a3 100644 --- a/registry/storage/driver/oss/oss.go +++ b/registry/storage/driver/oss/oss.go @@ -37,12 +37,11 @@ const driverName = "oss" const minChunkSize = 5 << 20 const defaultChunkSize = 2 * minChunkSize -const defaultTimeout = 2 * time.Minute // 2 minute timeout per chunk // listMax is the largest amount of objects you can request from OSS in a list call const listMax = 1000 -//DriverParameters A struct that encapsulates all of the driver parameters after all values have been set +// DriverParameters A struct that encapsulates all of the driver parameters after all values have been set type DriverParameters struct { AccessKeyID string AccessKeySecret string From 2c4bf1a66407e64b016d37fed681eadbaffff108 Mon Sep 17 00:00:00 2001 From: zounengren Date: Mon, 27 Sep 2021 04:29:46 +0800 Subject: [PATCH 05/20] replace deprecated function Signed-off-by: Zou Nengren Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 79d19015492357887313f21c64d6bfa2202549e4) Signed-off-by: Sebastiaan van Stijn --- reference/reference.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/reference/reference.go b/reference/reference.go index b7cd00b0d68..4fdf4fd46b8 100644 --- a/reference/reference.go +++ b/reference/reference.go @@ -320,11 +320,13 @@ func WithDigest(name Named, digest digest.Digest) (Canonical, error) { // TrimNamed removes any tag or digest from the named reference. func TrimNamed(ref Named) Named { - domain, path := SplitHostname(ref) - return repository{ - domain: domain, - path: path, + repo := repository{} + if r, ok := ref.(namedRepository); ok { + repo.domain, repo.path = r.Domain(), r.Path() + } else { + repo.domain, repo.path = splitDomain(ref.Name()) } + return repo } func getBestReferenceType(ref reference) Reference { From b57133cc21dd406aa8a927953ee06a4e99c14772 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sun, 6 Nov 2022 22:52:01 +0100 Subject: [PATCH 06/20] referene: fix formatting of "deprecated" comment. Go requires "deprecated" comments to have an empty line before them, and to not be all-caps. This updates to the comment so that it's correctly picked up as deprecated. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 3c71f4933db1c49201765df66faf5dc6dc9ba884) Signed-off-by: Sebastiaan van Stijn --- reference/reference.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reference/reference.go b/reference/reference.go index 4fdf4fd46b8..fa5d66ced86 100644 --- a/reference/reference.go +++ b/reference/reference.go @@ -175,7 +175,8 @@ func splitDomain(name string) (string, string) { // hostname and name string. If no valid hostname is // found, the hostname is empty and the full value // is returned as name -// DEPRECATED: Use Domain or Path +// +// Deprecated: Use [Domain] or [Path]. func SplitHostname(named Named) (string, string) { if r, ok := named.(namedRepository); ok { return r.Domain(), r.Path() From cb121c3f2061817ae1968d1b1e18dde58cfd4ca8 Mon Sep 17 00:00:00 2001 From: Milos Gajdos Date: Tue, 15 Aug 2023 08:46:48 +0100 Subject: [PATCH 07/20] Set Content-Type header in registry client ReadFrom Client ReadFrom doesn't set Content-Type header leading to server side implementor to assume it's application/octet-stream. This commit makes this explicit on the client side. Signed-off-by: Milos Gajdos (cherry picked from commit 24de708d229e4b67fb434151895e909f31aa08e6) Signed-off-by: Sebastiaan van Stijn --- registry/client/blob_writer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/registry/client/blob_writer.go b/registry/client/blob_writer.go index 695bf852f16..dac030c7382 100644 --- a/registry/client/blob_writer.go +++ b/registry/client/blob_writer.go @@ -42,6 +42,8 @@ func (hbu *httpBlobUpload) ReadFrom(r io.Reader) (n int64, err error) { } defer req.Body.Close() + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := hbu.client.Do(req) if err != nil { return 0, err From 2ec0471bb5bb3e0380f5902ee38b92956fc56dff Mon Sep 17 00:00:00 2001 From: Milos Gajdos Date: Thu, 24 Aug 2023 08:08:04 +0100 Subject: [PATCH 08/20] Dont parse errors as JSON unless Content-Type is set to JSON Client attempts to parse the body of every error it receives as JSON regardless of the content-type. This commit rectifies by only parsing he error body as JSON if the Content-Type header is set to either "application/json" or "application/vnd.api+json". Signed-off-by: Milos Gajdos (cherry picked from commit 45b7b9cec3a45cbf3343ce773fd5c88bc50f8a7f) Signed-off-by: Sebastiaan van Stijn --- registry/client/blob_writer_test.go | 2 ++ registry/client/errors.go | 51 ++++++++++++++++++++--------- registry/client/errors_test.go | 39 +++++++++++++++++----- registry/client/repository_test.go | 1 + 4 files changed, 68 insertions(+), 25 deletions(-) diff --git a/registry/client/blob_writer_test.go b/registry/client/blob_writer_test.go index f4469b6f5c2..be961a72ebf 100644 --- a/registry/client/blob_writer_test.go +++ b/registry/client/blob_writer_test.go @@ -85,6 +85,7 @@ func TestUploadReadFrom(t *testing.T) { }, Response: testutil.Response{ StatusCode: http.StatusBadRequest, + Headers: http.Header{"Content-Type": []string{"application/json; charset=utf-8"}}, Body: []byte(` { "errors": [ @@ -106,6 +107,7 @@ func TestUploadReadFrom(t *testing.T) { }, Response: testutil.Response{ StatusCode: http.StatusBadRequest, + Headers: http.Header{"Content-Type": []string{"application/json"}}, Body: []byte("something bad happened"), }, }, diff --git a/registry/client/errors.go b/registry/client/errors.go index 024df43dd92..ce9902034d3 100644 --- a/registry/client/errors.go +++ b/registry/client/errors.go @@ -4,8 +4,8 @@ import ( "encoding/json" "errors" "fmt" - "io" "io/ioutil" + "mime" "net/http" "github.com/docker/distribution/registry/api/errcode" @@ -38,13 +38,29 @@ func (e *UnexpectedHTTPResponseError) Error() string { return fmt.Sprintf("error parsing HTTP %d response body: %s: %q", e.StatusCode, e.ParseErr.Error(), string(e.Response)) } -func parseHTTPErrorResponse(statusCode int, r io.Reader) error { +func parseHTTPErrorResponse(resp *http.Response) error { var errors errcode.Errors - body, err := ioutil.ReadAll(r) + body, err := ioutil.ReadAll(resp.Body) if err != nil { return err } + statusCode := resp.StatusCode + ctHeader := resp.Header.Get("Content-Type") + + if ctHeader == "" { + return makeError(statusCode, string(body)) + } + + contentType, _, err := mime.ParseMediaType(ctHeader) + if err != nil { + return fmt.Errorf("failed parsing content-type: %w", err) + } + + if contentType != "application/json" && contentType != "application/vnd.api+json" { + return makeError(statusCode, string(body)) + } + // For backward compatibility, handle irregularly formatted // messages that contain a "details" field. var detailsErr struct { @@ -52,16 +68,7 @@ func parseHTTPErrorResponse(statusCode int, r io.Reader) error { } err = json.Unmarshal(body, &detailsErr) if err == nil && detailsErr.Details != "" { - switch statusCode { - case http.StatusUnauthorized: - return errcode.ErrorCodeUnauthorized.WithMessage(detailsErr.Details) - case http.StatusForbidden: - return errcode.ErrorCodeDenied.WithMessage(detailsErr.Details) - case http.StatusTooManyRequests: - return errcode.ErrorCodeTooManyRequests.WithMessage(detailsErr.Details) - default: - return errcode.ErrorCodeUnknown.WithMessage(detailsErr.Details) - } + return makeError(statusCode, detailsErr.Details) } if err := json.Unmarshal(body, &errors); err != nil { @@ -85,6 +92,19 @@ func parseHTTPErrorResponse(statusCode int, r io.Reader) error { return errors } +func makeError(statusCode int, details string) error { + switch statusCode { + case http.StatusUnauthorized: + return errcode.ErrorCodeUnauthorized.WithMessage(details) + case http.StatusForbidden: + return errcode.ErrorCodeDenied.WithMessage(details) + case http.StatusTooManyRequests: + return errcode.ErrorCodeTooManyRequests.WithMessage(details) + default: + return errcode.ErrorCodeUnknown.WithMessage(details) + } +} + func makeErrorList(err error) []error { if errL, ok := err.(errcode.Errors); ok { return []error(errL) @@ -121,11 +141,10 @@ func HandleErrorResponse(resp *http.Response) error { } else { err.Message = err.Code.Message() } - - return mergeErrors(err, parseHTTPErrorResponse(resp.StatusCode, resp.Body)) + return mergeErrors(err, parseHTTPErrorResponse(resp)) } } - err := parseHTTPErrorResponse(resp.StatusCode, resp.Body) + err := parseHTTPErrorResponse(resp) if uErr, ok := err.(*UnexpectedHTTPResponseError); ok && resp.StatusCode == 401 { return errcode.ErrorCodeUnauthorized.WithDetail(uErr.Response) } diff --git a/registry/client/errors_test.go b/registry/client/errors_test.go index 6be399d0350..135608c7700 100644 --- a/registry/client/errors_test.go +++ b/registry/client/errors_test.go @@ -15,17 +15,18 @@ type nopCloser struct { func (nopCloser) Close() error { return nil } func TestHandleErrorResponse401ValidBody(t *testing.T) { - json := "{\"errors\":[{\"code\":\"UNAUTHORIZED\",\"message\":\"action requires authentication\"}]}" + json := `{"errors":[{"code":"UNAUTHORIZED","message":"action requires authentication"}]}` response := &http.Response{ Status: "401 Unauthorized", StatusCode: 401, Body: nopCloser{bytes.NewBufferString(json)}, + Header: http.Header{"Content-Type": []string{"application/json; charset=utf-8"}}, } err := HandleErrorResponse(response) expectedMsg := "unauthorized: action requires authentication" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error()) + t.Errorf("Expected %q, got: %q", expectedMsg, err.Error()) } } @@ -35,27 +36,29 @@ func TestHandleErrorResponse401WithInvalidBody(t *testing.T) { Status: "401 Unauthorized", StatusCode: 401, Body: nopCloser{bytes.NewBufferString(json)}, + Header: http.Header{"Content-Type": []string{"application/json; charset=utf-8"}}, } err := HandleErrorResponse(response) expectedMsg := "unauthorized: authentication required" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error()) + t.Errorf("Expected %q, got: %q", expectedMsg, err.Error()) } } func TestHandleErrorResponseExpectedStatusCode400ValidBody(t *testing.T) { - json := "{\"errors\":[{\"code\":\"DIGEST_INVALID\",\"message\":\"provided digest does not match\"}]}" + json := `{"errors":[{"code":"DIGEST_INVALID","message":"provided digest does not match"}]}` response := &http.Response{ Status: "400 Bad Request", StatusCode: 400, Body: nopCloser{bytes.NewBufferString(json)}, + Header: http.Header{"Content-Type": []string{"application/json"}}, } err := HandleErrorResponse(response) expectedMsg := "digest invalid: provided digest does not match" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error()) + t.Errorf("Expected %q, got: %q", expectedMsg, err.Error()) } } @@ -65,12 +68,13 @@ func TestHandleErrorResponseExpectedStatusCode404EmptyErrorSlice(t *testing.T) { Status: "404 Not Found", StatusCode: 404, Body: nopCloser{bytes.NewBufferString(json)}, + Header: http.Header{"Content-Type": []string{"application/json; charset=utf-8"}}, } err := HandleErrorResponse(response) expectedMsg := `error parsing HTTP 404 response body: no error details found in HTTP response body: "{\"randomkey\": \"randomvalue\"}"` if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error()) + t.Errorf("Expected %q, got: %q", expectedMsg, err.Error()) } } @@ -80,12 +84,13 @@ func TestHandleErrorResponseExpectedStatusCode404InvalidBody(t *testing.T) { Status: "404 Not Found", StatusCode: 404, Body: nopCloser{bytes.NewBufferString(json)}, + Header: http.Header{"Content-Type": []string{"application/json"}}, } err := HandleErrorResponse(response) expectedMsg := "error parsing HTTP 404 response body: invalid character 'i' looking for beginning of object key string: \"{invalid json}\"" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error()) + t.Errorf("Expected %q, got: %q", expectedMsg, err.Error()) } } @@ -94,12 +99,13 @@ func TestHandleErrorResponseUnexpectedStatusCode501(t *testing.T) { Status: "501 Not Implemented", StatusCode: 501, Body: nopCloser{bytes.NewBufferString("{\"Error Encountered\" : \"Function not implemented.\"}")}, + Header: http.Header{"Content-Type": []string{"application/json"}}, } err := HandleErrorResponse(response) expectedMsg := "received unexpected HTTP status: 501 Not Implemented" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error()) + t.Errorf("Expected %q, got: %q", expectedMsg, err.Error()) } } @@ -109,11 +115,26 @@ func TestHandleErrorResponseInsufficientPrivileges403(t *testing.T) { Status: "403 Forbidden", StatusCode: 403, Body: nopCloser{bytes.NewBufferString(json)}, + Header: http.Header{"Content-Type": []string{"application/json"}}, } err := HandleErrorResponse(response) expectedMsg := "denied: requesting higher privileges than access token allows" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("Expected \"%s\", got: \"%s\"", expectedMsg, err.Error()) + t.Errorf("Expected %q, got: %q", expectedMsg, err.Error()) + } +} + +func TestHandleErrorResponseNonJson(t *testing.T) { + msg := `{"details":"requesting higher privileges than access token allows"}` + response := &http.Response{ + Status: "403 Forbidden", + StatusCode: 403, + Body: nopCloser{bytes.NewBufferString(msg)}, + } + err := HandleErrorResponse(response) + + if !strings.Contains(err.Error(), msg) { + t.Errorf("Expected %q, got: %q", msg, err.Error()) } } diff --git a/registry/client/repository_test.go b/registry/client/repository_test.go index e142fc21b33..0051500bac1 100644 --- a/registry/client/repository_test.go +++ b/registry/client/repository_test.go @@ -1214,6 +1214,7 @@ func TestManifestUnauthorized(t *testing.T) { }, Response: testutil.Response{ StatusCode: http.StatusUnauthorized, + Headers: http.Header{"Content-Type": []string{"application/json; charset=utf-8"}}, Body: []byte("garbage"), }, }) From 0a98a00d175aa912275d603eadcf821168cf2208 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 9 May 2023 13:23:43 +0200 Subject: [PATCH 09/20] Ignore SA1019: SplitHostname is deprecated. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 84a85a40484b10a2540c0b68bc8fedda77ac5e2a) Signed-off-by: Sebastiaan van Stijn --- reference/normalize_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference/normalize_test.go b/reference/normalize_test.go index a636236eee0..110c8e8b57b 100644 --- a/reference/normalize_test.go +++ b/reference/normalize_test.go @@ -532,7 +532,7 @@ func TestNormalizedSplitHostname(t *testing.T) { t.Fail() } - named, err := ParseNormalizedNamed(testcase.input) + named, err := ParseNormalizedNamed(testcase.input) //nolint:staticcheck // Ignore SA1019: SplitHostname is deprecated. if err != nil { failf("error parsing name: %s", err) } From b800af44090d4b70be7c5a94b15677327df84fcd Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 9 May 2023 13:19:48 +0200 Subject: [PATCH 10/20] ignore SA1019: ac.(*accessController).rootCerts.Subjects has been deprecated We need to look into this; can we remove it, or is there a replacement? Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ebe9d67446a676d4b06f60a9c86e3ede66cc95fd) Signed-off-by: Sebastiaan van Stijn --- registry/auth/token/token_test.go | 2 +- registry/registry.go | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/registry/auth/token/token_test.go b/registry/auth/token/token_test.go index ec80d1bc872..49b836e0a0c 100644 --- a/registry/auth/token/token_test.go +++ b/registry/auth/token/token_test.go @@ -527,7 +527,7 @@ func TestNewAccessControllerPemBlock(t *testing.T) { t.Fatal(err) } - if len(ac.(*accessController).rootCerts.Subjects()) != 2 { + if len(ac.(*accessController).rootCerts.Subjects()) != 2 { //nolint:staticcheck // FIXME(thaJeztah): ignore SA1019: ac.(*accessController).rootCerts.Subjects has been deprecated since Go 1.18: if s was returned by SystemCertPool, Subjects will not include the system roots. (staticcheck) t.Fatal("accessController has the wrong number of certificates") } } diff --git a/registry/registry.go b/registry/registry.go index dc156f462a7..9486d8bba5d 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -236,11 +236,10 @@ func (registry *Registry) ListenAndServe() error { dcontext.GetLogger(registry.app).Infof("restricting TLS cipher suites to: %s", strings.Join(getCipherSuiteNames(tlsCipherSuites), ",")) tlsConf := &tls.Config{ - ClientAuth: tls.NoClientCert, - NextProtos: nextProtos(config), - MinVersion: tlsMinVersion, - PreferServerCipherSuites: true, - CipherSuites: tlsCipherSuites, + ClientAuth: tls.NoClientCert, + NextProtos: nextProtos(config), + MinVersion: tlsMinVersion, + CipherSuites: tlsCipherSuites, } if config.HTTP.TLS.LetsEncrypt.CacheFile != "" { @@ -282,7 +281,7 @@ func (registry *Registry) ListenAndServe() error { } } - for _, subj := range pool.Subjects() { + for _, subj := range pool.Subjects() { //nolint:staticcheck // FIXME(thaJeztah): ignore SA1019: ac.(*accessController).rootCerts.Subjects has been deprecated since Go 1.18: if s was returned by SystemCertPool, Subjects will not include the system roots. (staticcheck) dcontext.GetLogger(registry.app).Debugf("CA Subject: %s", string(subj)) } From 444d053e12d64179b6d3c7d610d493b53268f71c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 9 May 2023 12:36:32 +0200 Subject: [PATCH 11/20] update golangci-lint to v1.52 Removing the "structcheck" and "varcheck" linters as they've been deprecated. level=warning msg="[runner] The linter 'structcheck' is deprecated (since v1.49.0) due to: The owner seems to have abandoned the linter. Replaced by unused." level=warning msg="[runner] The linter 'varcheck' is deprecated (since v1.49.0) due to: The owner seems to have abandoned the linter. Replaced by unused." Signed-off-by: Sebastiaan van Stijn (cherry picked from commit dec03ea3d85d66f7681648c502260ea3cf370143) Signed-off-by: Sebastiaan van Stijn --- .golangci.yml | 10 ++++++++-- script/setup/install-dev-tools | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 36c083b0fc4..61dd0e00eb7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,7 +1,5 @@ linters: enable: - - structcheck - - varcheck - staticcheck - unconvert - gofmt @@ -14,6 +12,14 @@ linters: disable: - errcheck +linters-settings: + revive: + rules: + # TODO(thaJeztah): temporarily disabled the "unused-parameter" check. + # It produces many warnings, and some of those may need to be looked at. + - name: unused-parameter + disabled: true + run: deadline: 2m skip-dirs: diff --git a/script/setup/install-dev-tools b/script/setup/install-dev-tools index 7737836bb97..460718b3ee1 100755 --- a/script/setup/install-dev-tools +++ b/script/setup/install-dev-tools @@ -1,6 +1,6 @@ #!/usr/bin/env bash -GOLANGCI_LINT_VERSION="v1.50.1" +GOLANGCI_LINT_VERSION="v1.52.0" # # Install developer tools to $GOBIN (or $GOPATH/bin if unset) From 3316b19810e1b84bee372a8f285a59dd506381ef Mon Sep 17 00:00:00 2001 From: Ben Manuel Date: Thu, 29 Jun 2023 15:34:00 -0500 Subject: [PATCH 12/20] Update to golang 1.19.10 This addresses CVE-2023-29402, CVE-2023-29403, CVE-2023-29404, CVE-2023-29405 which were patched in 1.19.10. Signed-off-by: Ben Manuel (cherry picked from commit 36dd5b79ca81c7d945fb701d8eb958c0e212e24c) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/ci.yml | 2 +- Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 337f1e812af..8ad613d8136 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.19.9 + go-version: 1.19.10 - name: Dependencies run: | diff --git a/Dockerfile b/Dockerfile index 42b87c064cb..b655a06279f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.19.9 -ARG ALPINE_VERSION=3.16 +ARG GO_VERSION=1.19.10 +ARG ALPINE_VERSION=3.18 ARG XX_VERSION=1.2.1 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx From 29b8ba0b937417c25700d641623ca09bf9b44f04 Mon Sep 17 00:00:00 2001 From: James Hewitt Date: Sun, 27 Aug 2023 10:17:46 +0100 Subject: [PATCH 13/20] Update to go 1.20 Signed-off-by: James Hewitt (cherry picked from commit 0eb8fee87ecac7b7436c936c853f39c8f5eb9fb0) Signed-off-by: Sebastiaan van Stijn --- .github/workflows/ci.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ad613d8136..55b30d8c7ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.19.10 + go-version: 1.20.7 - name: Dependencies run: | diff --git a/Dockerfile b/Dockerfile index b655a06279f..6affb3ba9a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.19.10 +ARG GO_VERSION=1.20.7 ARG ALPINE_VERSION=3.18 ARG XX_VERSION=1.2.1 From 31f5cd4865c8053f8c5f62fba2ce13f7493aba1a Mon Sep 17 00:00:00 2001 From: James Hewitt Date: Sun, 27 Aug 2023 11:06:16 +0100 Subject: [PATCH 14/20] Handle rand deprecations in go 1.20 Signed-off-by: James Hewitt (cherry picked from commit 1a3e73cb84fd7ccb4f061f95507357cc63b6eb0e) Signed-off-by: Sebastiaan van Stijn --- registry/api/v2/routes_test.go | 2 -- registry/proxy/proxyblobstore_test.go | 5 +++-- registry/storage/driver/s3-aws/s3_test.go | 2 +- registry/storage/driver/testsuites/testsuites.go | 3 ++- registry/storage/filereader_test.go | 3 ++- script/setup/install-dev-tools | 2 +- testutil/tarfile.go | 3 ++- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/registry/api/v2/routes_test.go b/registry/api/v2/routes_test.go index 6c77e28155e..dc23c2e0477 100644 --- a/registry/api/v2/routes_test.go +++ b/registry/api/v2/routes_test.go @@ -9,7 +9,6 @@ import ( "reflect" "strings" "testing" - "time" "github.com/gorilla/mux" ) @@ -218,7 +217,6 @@ func TestRouterWithBadCharacters(t *testing.T) { // with random UTF8 characters not in the 128 bit ASCII range. // These are not valid characters for the router and we expect // 404s on every test. - rand.Seed(time.Now().UTC().UnixNano()) testCases := make([]routeTestCase, 1000) for idx := range testCases { testCases[idx] = routeTestCase{ diff --git a/registry/proxy/proxyblobstore_test.go b/registry/proxy/proxyblobstore_test.go index 6e90dbe57fd..13fea957b61 100644 --- a/registry/proxy/proxyblobstore_test.go +++ b/registry/proxy/proxyblobstore_test.go @@ -21,6 +21,7 @@ import ( ) var sbsMu sync.Mutex +var randSource rand.Rand type statsBlobStore struct { stats map[string]int @@ -195,13 +196,13 @@ func makeTestEnv(t *testing.T, name string) *testEnv { func makeBlob(size int) []byte { blob := make([]byte, size) for i := 0; i < size; i++ { - blob[i] = byte('A' + rand.Int()%48) + blob[i] = byte('A' + randSource.Int()%48) } return blob } func init() { - rand.Seed(42) + randSource = *rand.New(rand.NewSource(42)) } func populate(t *testing.T, te *testEnv, blobCount, size, numUnique int) { diff --git a/registry/storage/driver/s3-aws/s3_test.go b/registry/storage/driver/s3-aws/s3_test.go index be02772e952..20f4a6f0f84 100644 --- a/registry/storage/driver/s3-aws/s3_test.go +++ b/registry/storage/driver/s3-aws/s3_test.go @@ -2,8 +2,8 @@ package s3 import ( "bytes" + "crypto/rand" "io/ioutil" - "math/rand" "os" "strconv" "testing" diff --git a/registry/storage/driver/testsuites/testsuites.go b/registry/storage/driver/testsuites/testsuites.go index 5e37c5f3cfa..496d2b742ec 100644 --- a/registry/storage/driver/testsuites/testsuites.go +++ b/registry/storage/driver/testsuites/testsuites.go @@ -3,6 +3,7 @@ package testsuites import ( "bytes" "context" + crand "crypto/rand" "crypto/sha1" "io" "io/ioutil" @@ -1214,7 +1215,7 @@ func randomFilename(length int64) string { var randomBytes = make([]byte, 128<<20) func init() { - _, _ = rand.Read(randomBytes) // always returns len(randomBytes) and nil error + _, _ = crand.Read(randomBytes) // always returns len(randomBytes) and nil error } func randomContents(length int64) []byte { diff --git a/registry/storage/filereader_test.go b/registry/storage/filereader_test.go index 305366f434d..9145c3d73df 100644 --- a/registry/storage/filereader_test.go +++ b/registry/storage/filereader_test.go @@ -2,6 +2,7 @@ package storage import ( "bytes" + crand "crypto/rand" "io" mrand "math/rand" "testing" @@ -14,7 +15,7 @@ import ( func TestSimpleRead(t *testing.T) { ctx := context.Background() content := make([]byte, 1<<20) - n, err := mrand.Read(content) + n, err := crand.Read(content) if err != nil { t.Fatalf("unexpected error building random data: %v", err) } diff --git a/script/setup/install-dev-tools b/script/setup/install-dev-tools index 460718b3ee1..f0dce872e62 100755 --- a/script/setup/install-dev-tools +++ b/script/setup/install-dev-tools @@ -1,6 +1,6 @@ #!/usr/bin/env bash -GOLANGCI_LINT_VERSION="v1.52.0" +GOLANGCI_LINT_VERSION="v1.54.2" # # Install developer tools to $GOBIN (or $GOPATH/bin if unset) diff --git a/testutil/tarfile.go b/testutil/tarfile.go index 2ebd364a2d0..7120856382e 100644 --- a/testutil/tarfile.go +++ b/testutil/tarfile.go @@ -3,6 +3,7 @@ package testutil import ( "archive/tar" "bytes" + crand "crypto/rand" "fmt" "io" mrand "math/rand" @@ -45,7 +46,7 @@ func CreateRandomTarFile() (rs io.ReadSeeker, dgst digest.Digest, err error) { randomData := make([]byte, fileSize) // Fill up the buffer with some random data. - n, err := mrand.Read(randomData) + n, err := crand.Read(randomData) if n != len(randomData) { return nil, "", fmt.Errorf("short read creating random reader: %v bytes != %v bytes", n, len(randomData)) From 3c6f77884209e50bbe15cd95657c50cbd3a463bf Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 12 Sep 2023 00:07:34 +0200 Subject: [PATCH 15/20] update to go1.20.8 go1.20.8 (released 2023-09-06) includes two security fixes to the html/template package, as well as bug fixes to the compiler, the go command, the runtime, and the crypto/tls, go/types, net/http, and path/filepath packages. See the Go 1.20.8 milestone on our issue tracker for details: https://github.com/golang/go/issues?q=milestone%3AGo1.20.8+label%3ACherryPickApproved full diff: https://github.com/golang/go/compare/go1.20.7...go1.20.8 From the security mailing: [security] Go 1.21.1 and Go 1.20.8 are released Hello gophers, We have just released Go versions 1.21.1 and 1.20.8, minor point releases. These minor releases include 4 security fixes following the security policy: - cmd/go: go.mod toolchain directive allows arbitrary execution The go.mod toolchain directive, introduced in Go 1.21, could be leveraged to execute scripts and binaries relative to the root of the module when the "go" command was executed within the module. This applies to modules downloaded using the "go" command from the module proxy, as well as modules downloaded directly using VCS software. Thanks to Juho Nurminen of Mattermost for reporting this issue. This is CVE-2023-39320 and Go issue https://go.dev/issue/62198. - html/template: improper handling of HTML-like comments within script contexts The html/template package did not properly handle HMTL-like "" comment tokens, nor hashbang "#!" comment tokens, in