Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit df34858

Browse files
johnstcnjohnstcn
authored
chore(coderd): extract fileszip to package archive for reuse (#15229)
Related to #15087 As part of sniffing the workspace tags from an uploaded file, we need to be able to handle both zip and tar files. Extracting the functions to a separate `archive` package will be helpful here.
1 parent 5ad4747 commit df34858

File tree

8 files changed

+156
-126
lines changed

8 files changed

+156
-126
lines changed

‎coderd/fileszip.go renamed to ‎archive/archive.go

+14-11
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package coderd
1+
package archive
22

33
import (
44
"archive/tar"
@@ -10,29 +10,30 @@ import (
1010
"strings"
1111
)
1212

13-
func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) {
13+
// CreateTarFromZip converts the given zipReader to a tar archive.
14+
func CreateTarFromZip(zipReader *zip.Reader, maxSize int64) ([]byte, error) {
1415
var tarBuffer bytes.Buffer
15-
err := writeTarArchive(&tarBuffer, zipReader)
16+
err := writeTarArchive(&tarBuffer, zipReader, maxSize)
1617
if err != nil {
1718
return nil, err
1819
}
1920
return tarBuffer.Bytes(), nil
2021
}
2122

22-
func writeTarArchive(w io.Writer, zipReader *zip.Reader) error {
23+
func writeTarArchive(w io.Writer, zipReader *zip.Reader, maxSize int64) error {
2324
tarWriter := tar.NewWriter(w)
2425
defer tarWriter.Close()
2526

2627
for _, file := range zipReader.File {
27-
err := processFileInZipArchive(file, tarWriter)
28+
err := processFileInZipArchive(file, tarWriter, maxSize)
2829
if err != nil {
2930
return err
3031
}
3132
}
3233
return nil
3334
}
3435

35-
func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
36+
func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer, maxSize int64) error {
3637
fileReader, err := file.Open()
3738
if err != nil {
3839
return err
@@ -52,24 +53,26 @@ func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
5253
return err
5354
}
5455

55-
n, err := io.CopyN(tarWriter, fileReader, httpFileMaxBytes)
56+
n, err := io.CopyN(tarWriter, fileReader, maxSize)
5657
log.Println(file.Name, n, err)
5758
if errors.Is(err, io.EOF) {
5859
err = nil
5960
}
6061
return err
6162
}
6263

63-
func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) {
64+
// CreateZipFromTar converts the given tarReader to a zip archive.
65+
func CreateZipFromTar(tarReader *tar.Reader, maxSize int64) ([]byte, error) {
6466
var zipBuffer bytes.Buffer
65-
err := WriteZipArchive(&zipBuffer, tarReader)
67+
err := WriteZip(&zipBuffer, tarReader, maxSize)
6668
if err != nil {
6769
return nil, err
6870
}
6971
return zipBuffer.Bytes(), nil
7072
}
7173

72-
func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
74+
// WriteZip writes the given tarReader to w.
75+
func WriteZip(w io.Writer, tarReader *tar.Reader, maxSize int64) error {
7376
zipWriter := zip.NewWriter(w)
7477
defer zipWriter.Close()
7578

@@ -100,7 +103,7 @@ func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
100103
return err
101104
}
102105

103-
_, err = io.CopyN(zipEntry, tarReader, httpFileMaxBytes)
106+
_, err = io.CopyN(zipEntry, tarReader, maxSize)
104107
if errors.Is(err, io.EOF) {
105108
err = nil
106109
}

‎coderd/fileszip_test.go renamed to ‎archive/archive_test.go

+12-97
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,22 @@
1-
package coderd_test
1+
package archive_test
22

33
import (
44
"archive/tar"
55
"archive/zip"
66
"bytes"
7-
"io"
87
"io/fs"
98
"os"
109
"os/exec"
1110
"path/filepath"
1211
"runtime"
1312
"strings"
1413
"testing"
15-
"time"
1614

1715
"github.com/stretchr/testify/assert"
1816
"github.com/stretchr/testify/require"
19-
"golang.org/x/xerrors"
2017

21-
"github.com/coder/coder/v2/coderd"
18+
"github.com/coder/coder/v2/archive"
19+
"github.com/coder/coder/v2/archive/archivetest"
2220
"github.com/coder/coder/v2/testutil"
2321
)
2422

@@ -30,18 +28,17 @@ func TestCreateTarFromZip(t *testing.T) {
3028

3129
// Read a zip file we prepared earlier
3230
ctx := testutil.Context(t, testutil.WaitShort)
33-
zipBytes, err := os.ReadFile(filepath.Join("testdata", "test.zip"))
34-
require.NoError(t, err, "failed to read sample zip file")
31+
zipBytes := archivetest.TestZipFileBytes()
3532
// Assert invariant
36-
assertSampleZipFile(t, zipBytes)
33+
archivetest.AssertSampleZipFile(t, zipBytes)
3734

3835
zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
3936
require.NoError(t, err, "failed to parse sample zip file")
4037

41-
tarBytes, err := coderd.CreateTarFromZip(zr)
38+
tarBytes, err := archive.CreateTarFromZip(zr, int64(len(zipBytes)))
4239
require.NoError(t, err, "failed to convert zip to tar")
4340

44-
assertSampleTarFile(t, tarBytes)
41+
archivetest.AssertSampleTarFile(t, tarBytes)
4542

4643
tempDir := t.TempDir()
4744
tempFilePath := filepath.Join(tempDir, "test.tar")
@@ -60,14 +57,13 @@ func TestCreateZipFromTar(t *testing.T) {
6057
}
6158
t.Run("OK", func(t *testing.T) {
6259
t.Parallel()
63-
tarBytes, err := os.ReadFile(filepath.Join(".", "testdata", "test.tar"))
64-
require.NoError(t, err, "failed to read sample tar file")
60+
tarBytes := archivetest.TestTarFileBytes()
6561

6662
tr := tar.NewReader(bytes.NewReader(tarBytes))
67-
zipBytes, err := coderd.CreateZipFromTar(tr)
63+
zipBytes, err := archive.CreateZipFromTar(tr, int64(len(tarBytes)))
6864
require.NoError(t, err)
6965

70-
assertSampleZipFile(t, zipBytes)
66+
archivetest.AssertSampleZipFile(t, zipBytes)
7167

7268
tempDir := t.TempDir()
7369
tempFilePath := filepath.Join(tempDir, "test.zip")
@@ -99,7 +95,7 @@ func TestCreateZipFromTar(t *testing.T) {
9995

10096
// When: we convert this to a zip
10197
tr := tar.NewReader(&tarBytes)
102-
zipBytes, err := coderd.CreateZipFromTar(tr)
98+
zipBytes, err := archive.CreateZipFromTar(tr, int64(tarBytes.Len()))
10399
require.NoError(t, err)
104100

105101
// Then: the resulting zip should contain a corresponding directory
@@ -133,7 +129,7 @@ func assertExtractedFiles(t *testing.T, dir string, checkModePerm bool) {
133129
if checkModePerm {
134130
assert.Equal(t, fs.ModePerm&0o755, stat.Mode().Perm(), "expected mode 0755 on directory")
135131
}
136-
assert.Equal(t, archiveRefTime(t).UTC(), stat.ModTime().UTC(), "unexpected modtime of %q", path)
132+
assert.Equal(t, archivetest.ArchiveRefTime(t).UTC(), stat.ModTime().UTC(), "unexpected modtime of %q", path)
137133
case "/test/hello.txt":
138134
stat, err := os.Stat(path)
139135
assert.NoError(t, err, "failed to stat path %q", path)
@@ -168,84 +164,3 @@ func assertExtractedFiles(t *testing.T, dir string, checkModePerm bool) {
168164
return nil
169165
})
170166
}
171-
172-
func assertSampleTarFile(t *testing.T, tarBytes []byte) {
173-
t.Helper()
174-
175-
tr := tar.NewReader(bytes.NewReader(tarBytes))
176-
for {
177-
hdr, err := tr.Next()
178-
if err != nil {
179-
if err == io.EOF {
180-
return
181-
}
182-
require.NoError(t, err)
183-
}
184-
185-
// Note: ignoring timezones here.
186-
require.Equal(t, archiveRefTime(t).UTC(), hdr.ModTime.UTC())
187-
188-
switch hdr.Name {
189-
case "test/":
190-
require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
191-
case "test/hello.txt":
192-
require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
193-
bs, err := io.ReadAll(tr)
194-
if err != nil && !xerrors.Is(err, io.EOF) {
195-
require.NoError(t, err)
196-
}
197-
require.Equal(t, "hello", string(bs))
198-
case "test/dir/":
199-
require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
200-
case "test/dir/world.txt":
201-
require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
202-
bs, err := io.ReadAll(tr)
203-
if err != nil && !xerrors.Is(err, io.EOF) {
204-
require.NoError(t, err)
205-
}
206-
require.Equal(t, "world", string(bs))
207-
default:
208-
require.Failf(t, "unexpected file in tar", hdr.Name)
209-
}
210-
}
211-
}
212-
213-
func assertSampleZipFile(t *testing.T, zipBytes []byte) {
214-
t.Helper()
215-
216-
zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
217-
require.NoError(t, err)
218-
219-
for _, f := range zr.File {
220-
// Note: ignoring timezones here.
221-
require.Equal(t, archiveRefTime(t).UTC(), f.Modified.UTC())
222-
switch f.Name {
223-
case "test/", "test/dir/":
224-
// directory
225-
case "test/hello.txt":
226-
rc, err := f.Open()
227-
require.NoError(t, err)
228-
bs, err := io.ReadAll(rc)
229-
_ = rc.Close()
230-
require.NoError(t, err)
231-
require.Equal(t, "hello", string(bs))
232-
case "test/dir/world.txt":
233-
rc, err := f.Open()
234-
require.NoError(t, err)
235-
bs, err := io.ReadAll(rc)
236-
_ = rc.Close()
237-
require.NoError(t, err)
238-
require.Equal(t, "world", string(bs))
239-
default:
240-
require.Failf(t, "unexpected file in zip", f.Name)
241-
}
242-
}
243-
}
244-
245-
// archiveRefTime is the Go reference time. The contents of the sample tar and zip files
246-
// in testdata/ all have their modtimes set to the below in some timezone.
247-
func archiveRefTime(t *testing.T) time.Time {
248-
locMST, err := time.LoadLocation("MST")
249-
require.NoError(t, err, "failed to load MST timezone")
250-
return time.Date(2006, 1, 2, 3, 4, 5, 0, locMST)
251-
}

‎archive/archivetest/archivetest.go

+113
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package archivetest
2+
3+
import (
4+
"archive/tar"
5+
"archive/zip"
6+
"bytes"
7+
_ "embed"
8+
"io"
9+
"testing"
10+
"time"
11+
12+
"github.com/stretchr/testify/require"
13+
"golang.org/x/xerrors"
14+
)
15+
16+
//go:embed testdata/test.tar
17+
var testTarFileBytes []byte
18+
19+
//go:embed testdata/test.zip
20+
var testZipFileBytes []byte
21+
22+
// TestTarFileBytes returns the content of testdata/test.tar
23+
func TestTarFileBytes() []byte {
24+
return append([]byte{}, testTarFileBytes...)
25+
}
26+
27+
// TestZipFileBytes returns the content of testdata/test.zip
28+
func TestZipFileBytes() []byte {
29+
return append([]byte{}, testZipFileBytes...)
30+
}
31+
32+
// AssertSampleTarfile compares the content of tarBytes against testdata/test.tar.
33+
func AssertSampleTarFile(t *testing.T, tarBytes []byte) {
34+
t.Helper()
35+
36+
tr := tar.NewReader(bytes.NewReader(tarBytes))
37+
for {
38+
hdr, err := tr.Next()
39+
if err != nil {
40+
if err == io.EOF {
41+
return
42+
}
43+
require.NoError(t, err)
44+
}
45+
46+
// Note: ignoring timezones here.
47+
require.Equal(t, ArchiveRefTime(t).UTC(), hdr.ModTime.UTC())
48+
49+
switch hdr.Name {
50+
case "test/":
51+
require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
52+
case "test/hello.txt":
53+
require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
54+
bs, err := io.ReadAll(tr)
55+
if err != nil && !xerrors.Is(err, io.EOF) {
56+
require.NoError(t, err)
57+
}
58+
require.Equal(t, "hello", string(bs))
59+
case "test/dir/":
60+
require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
61+
case "test/dir/world.txt":
62+
require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
63+
bs, err := io.ReadAll(tr)
64+
if err != nil && !xerrors.Is(err, io.EOF) {
65+
require.NoError(t, err)
66+
}
67+
require.Equal(t, "world", string(bs))
68+
default:
69+
require.Failf(t, "unexpected file in tar", hdr.Name)
70+
}
71+
}
72+
}
73+
74+
// AssertSampleZipFile compares the content of zipBytes against testdata/test.zip.
75+
func AssertSampleZipFile(t *testing.T, zipBytes []byte) {
76+
t.Helper()
77+
78+
zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
79+
require.NoError(t, err)
80+
81+
for _, f := range zr.File {
82+
// Note: ignoring timezones here.
83+
require.Equal(t, ArchiveRefTime(t).UTC(), f.Modified.UTC())
84+
switch f.Name {
85+
case "test/", "test/dir/":
86+
// directory
87+
case "test/hello.txt":
88+
rc, err := f.Open()
89+
require.NoError(t, err)
90+
bs, err := io.ReadAll(rc)
91+
_ = rc.Close()
92+
require.NoError(t, err)
93+
require.Equal(t, "hello", string(bs))
94+
case "test/dir/world.txt":
95+
rc, err := f.Open()
96+
require.NoError(t, err)
97+
bs, err := io.ReadAll(rc)
98+
_ = rc.Close()
99+
require.NoError(t, err)
100+
require.Equal(t, "world", string(bs))
101+
default:
102+
require.Failf(t, "unexpected file in zip", f.Name)
103+
}
104+
}
105+
}
106+
107+
// archiveRefTime is the Go reference time. The contents of the sample tar and zip files
108+
// in testdata/ all have their modtimes set to the below in some timezone.
109+
func ArchiveRefTime(t *testing.T) time.Time {
110+
locMST, err := time.LoadLocation("MST")
111+
require.NoError(t, err, "failed to load MST timezone")
112+
return time.Date(2006, 1, 2, 3, 4, 5, 0, locMST)
113+
}

‎coderd/testdata/test.tar renamed to ‎archive/archivetest/testdata/test.tar

File renamed without changes.

‎coderd/testdata/test.zip renamed to ‎archive/archivetest/testdata/test.zip

File renamed without changes.

‎cli/templatepull_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/google/uuid"
1414
"github.com/stretchr/testify/require"
1515

16+
"github.com/coder/coder/v2/archive"
1617
"github.com/coder/coder/v2/cli/clitest"
1718
"github.com/coder/coder/v2/coderd"
1819
"github.com/coder/coder/v2/coderd/coderdtest"
@@ -95,7 +96,7 @@ func TestTemplatePull_Stdout(t *testing.T) {
9596

9697
// Verify .zip format
9798
tarReader := tar.NewReader(bytes.NewReader(expected))
98-
expectedZip, err := coderd.CreateZipFromTar(tarReader)
99+
expectedZip, err := archive.CreateZipFromTar(tarReader, coderd.HTTPFileMaxBytes)
99100
require.NoError(t, err)
100101

101102
inv, root = clitest.New(t, "templates", "pull", "--zip", template.Name)

0 commit comments

Comments
 (0)