From fe2204d3ff3182bd6b625054b8894721bd1edce9 Mon Sep 17 00:00:00 2001 From: Fergus Strange Date: Sat, 11 Jun 2022 11:41:36 +1000 Subject: [PATCH 1/6] Add test for real corrupted data. --- decompression_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/decompression_test.go b/decompression_test.go index cbd0cb8..2be61b3 100644 --- a/decompression_test.go +++ b/decompression_test.go @@ -132,3 +132,35 @@ func Test_decompressTarXz_ErrorWhenFileToCopyToNotExists(t *testing.T) { assert.Regexp(t, "^unable to extract postgres archive:.+$", err) } + +func Test_decompressTarXz_ErrorWhenArchiveCorrupted(t *testing.T) { + tempDir, err := ioutil.TempDir("", "temp_tar_test") + if err != nil { + panic(err) + } + + archive, cleanup := createTempXzArchive() + + defer cleanup() + + file, err := os.OpenFile(archive, os.O_WRONLY, 0664) + if err != nil { + panic(err) + } + + if _, err := file.Seek(35, 0); err != nil { + panic(err) + } + + if _, err := file.WriteString("someJunk"); err != nil { + panic(err) + } + + if err := file.Close(); err != nil { + panic(err) + } + + err = decompressTarXz(defaultTarReader, archive, tempDir) + + assert.EqualError(t, err, "unable to extract postgres archive: xz: data is corrupt") +} From f0b414108fb1d4848d8fc84bd4254f19ab54a893 Mon Sep 17 00:00:00 2001 From: Fergus Strange Date: Sat, 11 Jun 2022 15:07:45 +1000 Subject: [PATCH 2/6] Try adding integrity check on downloaded files. --- remote_fetch.go | 49 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/remote_fetch.go b/remote_fetch.go index a33a1ee..7394a0f 100644 --- a/remote_fetch.go +++ b/remote_fetch.go @@ -3,6 +3,9 @@ package embeddedpostgres import ( "archive/zip" "bytes" + "crypto/sha256" + "encoding/hex" + "errors" "fmt" "io/ioutil" "log" @@ -33,27 +36,51 @@ func defaultRemoteFetchStrategy(remoteFetchHost string, versionStrategy VersionS return fmt.Errorf("unable to connect to %s", remoteFetchHost) } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Fatal(err) - } - }() + defer closeBody(resp)() + + downloadSHAURL := fmt.Sprintf("%s.sha256", downloadURL) + + shaResp, err := http.Get(downloadSHAURL) + if err != nil { + return fmt.Errorf("unable to connect to %s", remoteFetchHost) + } + + defer closeBody(shaResp)() if resp.StatusCode != http.StatusOK { return fmt.Errorf("no version found matching %s", version) } - return decompressResponse(resp, cacheLocator, downloadURL) + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errorFetchingPostgres(err) + } + + if shaResp.StatusCode == http.StatusOK { + shaBytes, err := ioutil.ReadAll(shaResp.Body) + if err == nil { + bodyBytesHash := sha256.Sum256(bodyBytes) + downloadedHash := hex.EncodeToString(bodyBytesHash[:]) + if !strings.EqualFold(string(shaBytes), downloadedHash) { + return errors.New("downloaded checksums do not match") + } + } + } + + return decompressResponse(bodyBytes, resp.ContentLength, cacheLocator, downloadURL) } } -func decompressResponse(resp *http.Response, cacheLocator CacheLocator, downloadURL string) error { - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return errorFetchingPostgres(err) +func closeBody(resp *http.Response) func() { + return func() { + if err := resp.Body.Close(); err != nil { + log.Fatal(err) + } } +} - zipReader, err := zip.NewReader(bytes.NewReader(bodyBytes), resp.ContentLength) +func decompressResponse(bodyBytes []byte, contentLength int64, cacheLocator CacheLocator, downloadURL string) error { + zipReader, err := zip.NewReader(bytes.NewReader(bodyBytes), contentLength) if err != nil { return errorFetchingPostgres(err) } From 112289a4b11475a2a6860e8ab9743bbb20b6aa3d Mon Sep 17 00:00:00 2001 From: Fergus Strange Date: Sat, 11 Jun 2022 15:08:14 +1000 Subject: [PATCH 3/6] Lint --- remote_fetch.go | 1 + 1 file changed, 1 insertion(+) diff --git a/remote_fetch.go b/remote_fetch.go index 7394a0f..7b383c3 100644 --- a/remote_fetch.go +++ b/remote_fetch.go @@ -61,6 +61,7 @@ func defaultRemoteFetchStrategy(remoteFetchHost string, versionStrategy VersionS if err == nil { bodyBytesHash := sha256.Sum256(bodyBytes) downloadedHash := hex.EncodeToString(bodyBytesHash[:]) + if !strings.EqualFold(string(shaBytes), downloadedHash) { return errors.New("downloaded checksums do not match") } From ef859b10052eb3fa782149e1eb59dc41a1e39a70 Mon Sep 17 00:00:00 2001 From: Fergus Strange Date: Sat, 11 Jun 2022 16:43:54 +1000 Subject: [PATCH 4/6] Add coverage for checking sha --- remote_fetch_test.go | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/remote_fetch_test.go b/remote_fetch_test.go index 98588f0..8e520d5 100644 --- a/remote_fetch_test.go +++ b/remote_fetch_test.go @@ -2,11 +2,14 @@ package embeddedpostgres import ( "archive/zip" + "crypto/sha256" + "encoding/hex" "io/ioutil" "net/http" "net/http/httptest" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -54,7 +57,10 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenBodyReadIssue(t *testing.T) { func Test_defaultRemoteFetchStrategy_ErrorWhenCannotUnzipSubFile(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - + if strings.HasSuffix(r.RequestURI, ".sha256") { + w.WriteHeader(http.StatusNotFound) + return + } })) defer server.Close() @@ -69,6 +75,11 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotUnzipSubFile(t *testing.T) { func Test_defaultRemoteFetchStrategy_ErrorWhenCannotUnzip(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.RequestURI, ".sha256") { + w.WriteHeader(404) + return + } + if _, err := w.Write([]byte("lolz")); err != nil { panic(err) } @@ -86,6 +97,11 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotUnzip(t *testing.T) { func Test_defaultRemoteFetchStrategy_ErrorWhenNoSubTarArchive(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.RequestURI, ".sha256") { + w.WriteHeader(http.StatusNotFound) + return + } + MyZipWriter := zip.NewWriter(w) if err := MyZipWriter.Close(); err != nil { @@ -114,6 +130,11 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotExtractSubArchive(t *testing } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.RequestURI, ".sha256") { + w.WriteHeader(http.StatusNotFound) + return + } + bytes, err := ioutil.ReadFile(jarFile) if err != nil { panic(err) @@ -148,6 +169,11 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotCreateCacheDirectory(t *test cacheLocation := filepath.Join(fileBlockingExtractDirectory, "cache_file.jar") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.RequestURI, ".sha256") { + w.WriteHeader(http.StatusNotFound) + return + } + bytes, err := ioutil.ReadFile(jarFile) if err != nil { panic(err) @@ -181,6 +207,11 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotCreateSubArchiveFile(t *test } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.RequestURI, ".sha256") { + w.WriteHeader(http.StatusNotFound) + return + } + bytes, err := ioutil.ReadFile(jarFile) if err != nil { panic(err) @@ -213,6 +244,17 @@ func Test_defaultRemoteFetchStrategy(t *testing.T) { if err != nil { panic(err) } + + if strings.HasSuffix(r.RequestURI, ".sha256") { + w.WriteHeader(200) + contentHash := sha256.Sum256(bytes) + if _, err := w.Write([]byte(hex.EncodeToString(contentHash[:]))); err != nil { + panic(err) + } + + return + } + if _, err := w.Write(bytes); err != nil { panic(err) } From 3b106562e776fcc10ef0c7993f93e07bf979091c Mon Sep 17 00:00:00 2001 From: Fergus Strange Date: Sun, 12 Jun 2022 11:23:51 +1000 Subject: [PATCH 5/6] Up coverage --- remote_fetch.go | 17 +++++++---------- remote_fetch_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/remote_fetch.go b/remote_fetch.go index 7b383c3..d74648a 100644 --- a/remote_fetch.go +++ b/remote_fetch.go @@ -22,6 +22,7 @@ type RemoteFetchStrategy func() error func defaultRemoteFetchStrategy(remoteFetchHost string, versionStrategy VersionStrategy, cacheLocator CacheLocator) RemoteFetchStrategy { return func() error { operatingSystem, architecture, version := versionStrategy() + downloadURL := fmt.Sprintf("%s/io/zonky/test/postgres/embedded-postgres-binaries-%s-%s/%s/embedded-postgres-binaries-%s-%s-%s.jar", remoteFetchHost, operatingSystem, @@ -38,15 +39,6 @@ func defaultRemoteFetchStrategy(remoteFetchHost string, versionStrategy VersionS defer closeBody(resp)() - downloadSHAURL := fmt.Sprintf("%s.sha256", downloadURL) - - shaResp, err := http.Get(downloadSHAURL) - if err != nil { - return fmt.Errorf("unable to connect to %s", remoteFetchHost) - } - - defer closeBody(shaResp)() - if resp.StatusCode != http.StatusOK { return fmt.Errorf("no version found matching %s", version) } @@ -56,7 +48,12 @@ func defaultRemoteFetchStrategy(remoteFetchHost string, versionStrategy VersionS return errorFetchingPostgres(err) } - if shaResp.StatusCode == http.StatusOK { + downloadSHAURL := fmt.Sprintf("%s.sha256", downloadURL) + shaResp, err := http.Get(downloadSHAURL) + + defer closeBody(shaResp)() + + if err == nil && shaResp.StatusCode == http.StatusOK { shaBytes, err := ioutil.ReadAll(shaResp.Body) if err == nil { bodyBytesHash := sha256.Sum256(bodyBytes) diff --git a/remote_fetch_test.go b/remote_fetch_test.go index 8e520d5..57f7b0a 100644 --- a/remote_fetch_test.go +++ b/remote_fetch_test.go @@ -233,6 +233,44 @@ func Test_defaultRemoteFetchStrategy_ErrorWhenCannotCreateSubArchiveFile(t *test assert.Regexp(t, "^unable to extract postgres archive:.+$", err) } +func Test_defaultRemoteFetchStrategy_ErrorWhenSHA256NotMatch(t *testing.T) { + jarFile, cleanUp := createTempZipArchive() + defer cleanUp() + + cacheLocation := filepath.Join(filepath.Dir(jarFile), "extract_location", "cache.jar") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + bytes, err := ioutil.ReadFile(jarFile) + if err != nil { + panic(err) + } + + if strings.HasSuffix(r.RequestURI, ".sha256") { + w.WriteHeader(200) + if _, err := w.Write([]byte("literallyN3verGonnaWork")); err != nil { + panic(err) + } + + return + } + + if _, err := w.Write(bytes); err != nil { + panic(err) + } + })) + defer server.Close() + + remoteFetchStrategy := defaultRemoteFetchStrategy(server.URL+"/maven2", + testVersionStrategy(), + func() (s string, b bool) { + return cacheLocation, false + }) + + err := remoteFetchStrategy() + + assert.EqualError(t, err, "downloaded checksums do not match") +} + func Test_defaultRemoteFetchStrategy(t *testing.T) { jarFile, cleanUp := createTempZipArchive() defer cleanUp() From 72c16cc8239c6665b2099a74b86aada659eeadba Mon Sep 17 00:00:00 2001 From: Fergus Strange Date: Sun, 12 Jun 2022 12:53:21 +1000 Subject: [PATCH 6/6] Check via bytes rather than string and tidy code --- remote_fetch.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/remote_fetch.go b/remote_fetch.go index d74648a..ba0499e 100644 --- a/remote_fetch.go +++ b/remote_fetch.go @@ -23,7 +23,7 @@ func defaultRemoteFetchStrategy(remoteFetchHost string, versionStrategy VersionS return func() error { operatingSystem, architecture, version := versionStrategy() - downloadURL := fmt.Sprintf("%s/io/zonky/test/postgres/embedded-postgres-binaries-%s-%s/%s/embedded-postgres-binaries-%s-%s-%s.jar", + jarDownloadURL := fmt.Sprintf("%s/io/zonky/test/postgres/embedded-postgres-binaries-%s-%s/%s/embedded-postgres-binaries-%s-%s-%s.jar", remoteFetchHost, operatingSystem, architecture, @@ -32,40 +32,37 @@ func defaultRemoteFetchStrategy(remoteFetchHost string, versionStrategy VersionS architecture, version) - resp, err := http.Get(downloadURL) + jarDownloadResponse, err := http.Get(jarDownloadURL) if err != nil { return fmt.Errorf("unable to connect to %s", remoteFetchHost) } - defer closeBody(resp)() + defer closeBody(jarDownloadResponse)() - if resp.StatusCode != http.StatusOK { + if jarDownloadResponse.StatusCode != http.StatusOK { return fmt.Errorf("no version found matching %s", version) } - bodyBytes, err := ioutil.ReadAll(resp.Body) + jarBodyBytes, err := ioutil.ReadAll(jarDownloadResponse.Body) if err != nil { return errorFetchingPostgres(err) } - downloadSHAURL := fmt.Sprintf("%s.sha256", downloadURL) - shaResp, err := http.Get(downloadSHAURL) + shaDownloadURL := fmt.Sprintf("%s.sha256", jarDownloadURL) + shaDownloadResponse, err := http.Get(shaDownloadURL) - defer closeBody(shaResp)() + defer closeBody(shaDownloadResponse)() - if err == nil && shaResp.StatusCode == http.StatusOK { - shaBytes, err := ioutil.ReadAll(shaResp.Body) - if err == nil { - bodyBytesHash := sha256.Sum256(bodyBytes) - downloadedHash := hex.EncodeToString(bodyBytesHash[:]) - - if !strings.EqualFold(string(shaBytes), downloadedHash) { + if err == nil && shaDownloadResponse.StatusCode == http.StatusOK { + if shaBodyBytes, err := ioutil.ReadAll(shaDownloadResponse.Body); err == nil { + jarChecksum := sha256.Sum256(jarBodyBytes) + if !bytes.Equal(shaBodyBytes, []byte(hex.EncodeToString(jarChecksum[:]))) { return errors.New("downloaded checksums do not match") } } } - return decompressResponse(bodyBytes, resp.ContentLength, cacheLocator, downloadURL) + return decompressResponse(jarBodyBytes, jarDownloadResponse.ContentLength, cacheLocator, jarDownloadURL) } }