From 8ca578165eab06ec44e32d3b60f2a016f3d3b428 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Sat, 15 Mar 2025 15:10:07 +0100 Subject: [PATCH 01/34] Update afero to v1.14.0 --- gcsfs/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcsfs/go.mod b/gcsfs/go.mod index 504c4d4b..64cb4b76 100644 --- a/gcsfs/go.mod +++ b/gcsfs/go.mod @@ -6,7 +6,7 @@ replace github.com/spf13/afero => ../ require ( cloud.google.com/go/storage v1.51.0 - github.com/spf13/afero v1.13.0 + github.com/spf13/afero v1.14.0 golang.org/x/oauth2 v0.28.0 google.golang.org/api v0.226.0 ) From 487a81d9cb25bd50ee622aaca02c6f71d1f2a62f Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Sat, 15 Mar 2025 15:10:38 +0100 Subject: [PATCH 02/34] Update afero to v1.14.0 --- sftpfs/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sftpfs/go.mod b/sftpfs/go.mod index 3de2394f..ee2d5469 100644 --- a/sftpfs/go.mod +++ b/sftpfs/go.mod @@ -6,7 +6,7 @@ replace github.com/spf13/afero => ../ require ( github.com/pkg/sftp v1.13.8 - github.com/spf13/afero v1.13.0 + github.com/spf13/afero v1.14.0 golang.org/x/crypto v0.36.0 ) From ade566cea3ee363ff6d4c09d57ce6aa8a553f1d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Mar 2025 21:09:15 +0000 Subject: [PATCH 03/34] Bump golangci/golangci-lint-action from 6.5.1 to 6.5.2 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 6.5.1 to 6.5.2. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/4696ba8babb6127d732c3c6dde519db15edab9ea...55c2c1448f86e01eaae002a5a3a9624417608d84) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fde1ee93..e56359a0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,6 +50,6 @@ jobs: go-version: "1.24" - name: Lint - uses: golangci/golangci-lint-action@4696ba8babb6127d732c3c6dde519db15edab9ea # v6.5.1 + uses: golangci/golangci-lint-action@55c2c1448f86e01eaae002a5a3a9624417608d84 # v6.5.2 with: version: v1.64.7 From e521b7d9b2ef8491dc1734a9b017d15a838dca26 Mon Sep 17 00:00:00 2001 From: messikiller Date: Tue, 8 Apr 2025 17:00:15 +0800 Subject: [PATCH 04/34] support aliyun oss [draft] --- go.mod | 14 +- go.sum | 16 + ossfs/file.go | 288 ++++++++++++++++ ossfs/file_info.go | 17 + ossfs/file_test.go | 296 +++++++++++++++++ ossfs/fs.go | 188 +++++++++++ ossfs/fs_test.go | 462 ++++++++++++++++++++++++++ ossfs/fs_utils.go | 36 ++ ossfs/init.go | 14 + ossfs/internal/mocks/FileInfo.go | 139 ++++++++ ossfs/internal/mocks/ObjectManager.go | 293 ++++++++++++++++ ossfs/internal/utils/contract.go | 21 ++ ossfs/internal/utils/init.go | 6 + ossfs/internal/utils/oss.go | 210 ++++++++++++ 14 files changed, 1999 insertions(+), 1 deletion(-) create mode 100644 ossfs/file.go create mode 100644 ossfs/file_info.go create mode 100644 ossfs/file_test.go create mode 100644 ossfs/fs.go create mode 100644 ossfs/fs_test.go create mode 100644 ossfs/fs_utils.go create mode 100644 ossfs/init.go create mode 100644 ossfs/internal/mocks/FileInfo.go create mode 100644 ossfs/internal/mocks/ObjectManager.go create mode 100644 ossfs/internal/utils/contract.go create mode 100644 ossfs/internal/utils/init.go create mode 100644 ossfs/internal/utils/oss.go diff --git a/go.mod b/go.mod index 101c2865..fa24da40 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,16 @@ module github.com/spf13/afero go 1.23.0 -require golang.org/x/text v0.23.0 +require ( + github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1 + github.com/stretchr/testify v1.10.0 + golang.org/x/text v0.23.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + golang.org/x/time v0.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index d00bb390..1862dd3d 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,18 @@ +github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1 h1:sOhpJdR/+lbQniznp3cYSfwQlXbVkT0ccuiZScBrI6Y= +github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= +golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ossfs/file.go b/ossfs/file.go new file mode 100644 index 00000000..630beff8 --- /dev/null +++ b/ossfs/file.go @@ -0,0 +1,288 @@ +package ossfs + +import ( + "errors" + "io" + "os" + "strconv" + "syscall" + + "github.com/spf13/afero" +) + +type File struct { + name string + fs *Fs + openFlag int + offset int64 + fi os.FileInfo + dirty bool + closed bool + isDir bool + preloaded bool + preloadedFd afero.File +} + +func NewOssFile(name string, flag int, fs *Fs) (*File, error) { + return &File{ + name: fs.normFileName(name), + fs: fs, + openFlag: flag, + offset: 0, + dirty: false, + closed: false, + isDir: fs.isDir(fs.normFileName(name)), + preloaded: false, + preloadedFd: nil, + }, nil +} + +func (f *File) preload() error { + pfs := f.fs.preloadFs + if _, err := pfs.Stat(f.name); err == nil { + if e := pfs.Remove(f.name); e != nil { + return e + } + } + pfd, err := f.fs.preloadFs.Create(f.name) + if err != nil { + return err + } + + r, clean, e := f.fs.manager.GetObject(f.fs.ctx, f.fs.bucketName, f.name) + if e != nil { + return e + } + defer clean() + + if _, err := io.Copy(pfd, r); err != nil { + return err + } + + if _, err := pfd.Seek(f.offset, io.SeekStart); err != nil { + return err + } + + f.preloadedFd = pfd + f.preloaded = true + return nil +} + +func (f *File) freshFileInfo() error { + fi, err := f.fs.Stat(f.name) + if err != nil { + return err + } + f.fi = fi + f.dirty = false + return nil +} + +func (f *File) getFileInfo() (os.FileInfo, error) { + if f.dirty { + if err := f.freshFileInfo(); err != nil { + return nil, err + } + } + return f.fi, nil +} + +func (f *File) isReadable() bool { + return !f.closed && (f.openFlag == os.O_RDONLY || f.openFlag == os.O_RDWR) +} + +func (f *File) isWriteable() bool { + return !f.closed && (f.openFlag == os.O_WRONLY || f.openFlag == os.O_RDWR) +} + +func (f *File) isAppendOnly() bool { + return f.isWriteable() && f.openFlag&os.O_APPEND != 0 +} + +func (f *File) Read(p []byte) (int, error) { + if !f.isReadable() || f.isDir { + return 0, syscall.EPERM + } + n, err := f.ReadAt(p, f.offset) + if err != nil { + return 0, err + } + f.offset += int64(n) + return n, err +} + +func (f *File) ReadAt(p []byte, off int64) (int, error) { + if !f.isReadable() || f.isDir { + return 0, syscall.EPERM + } + reader, cleanUp, err := f.fs.manager.GetObjectPart(f.fs.ctx, f.fs.bucketName, f.name, off, off+int64(len(p))) + if err != nil { + return 0, err + } + defer cleanUp() + return reader.Read(p) +} + +func (f *File) Seek(offset int64, whence int) (int64, error) { + if (!f.isReadable() && !f.isWriteable()) || f.isDir { + return 0, syscall.EPERM + } + fi, err := f.getFileInfo() + if err != nil { + return 0, err + } + max := fi.Size() + var newOffset int64 + switch whence { + case io.SeekCurrent: + newOffset = f.offset + offset + case io.SeekStart: + newOffset = offset + case io.SeekEnd: + newOffset = max + offset + default: + return 0, errors.New("Invalid whence value: " + strconv.Itoa(whence)) + } + if newOffset < 0 || newOffset > max { + return 0, afero.ErrOutOfRange + } + f.offset = newOffset + return f.offset, nil +} + +func (f *File) doAppend(p []byte) (int, error) { + if !f.isWriteable() { + return 0, syscall.EPERM + } + fi, err := f.getFileInfo() + if err != nil { + return 0, err + } + return f.doWriteAt(p, fi.Size()) +} + +func (f *File) Write(p []byte) (int, error) { + if !f.isWriteable() { + return 0, syscall.EPERM + } + if f.isAppendOnly() { + return f.doAppend(p) + } + n, e := f.doWriteAt(p, f.offset) + if e != nil { + return 0, e + } + f.offset += int64(n) + return n, e +} + +func (f *File) doWriteAt(p []byte, off int64) (int, error) { + if f.isDir { + return 0, syscall.EPERM + } + + if !f.preloaded { + if err := f.preload(); err != nil { + return 0, err + } + } + + n, e := f.preloadedFd.WriteAt(p, off) + f.dirty = true + if f.fs.autoSync { + f.Sync() + } + return n, e +} + +func (f *File) WriteAt(p []byte, off int64) (int, error) { + if !f.isWriteable() || f.isAppendOnly() { + return 0, syscall.EPERM + } + return f.doWriteAt(p, off) +} + +func (f *File) Close() error { + f.Sync() + f.closed = true + delete(f.fs.openedFiles, f.name) + if f.preloaded { + err := f.fs.preloadFs.Remove(f.name) + if err != nil { + return err + } + err = f.preloadedFd.Close() + if err != nil { + return err + } + f.preloadedFd = nil + f.preloaded = false + } + return nil +} + +func (f *File) Name() string { + return f.name +} + +func (f *File) Readdir(count int) ([]os.FileInfo, error) { + if !f.isReadable() { + return nil, syscall.EPERM + } + + fis, err := f.fs.manager.ListObjects(f.fs.ctx, f.fs.bucketName, f.fs.ensureAsDir(f.name), count) + return fis, err +} + +func (f *File) Readdirnames(n int) ([]string, error) { + if !f.isReadable() { + return nil, syscall.EPERM + } + + fis, err := f.Readdir(n) + if err != nil { + return nil, err + } + var fNames []string + for _, fi := range fis { + fNames = append(fNames, fi.Name()) + } + + return fNames, nil +} + +func (f *File) Stat() (os.FileInfo, error) { + if f.dirty { + err := f.freshFileInfo() + if err != nil { + return nil, err + } + } + return f.fi, nil +} + +func (f *File) Sync() error { + if f.preloaded { + if _, err := f.fs.manager.PutObject(f.fs.ctx, f.fs.bucketName, f.name, f.preloadedFd); err != nil { + return err + } + } + if f.dirty { + if err := f.freshFileInfo(); err != nil { + return err + } + } + return nil +} + +func (f *File) Truncate(size int64) error { + if !f.isWriteable() || f.isDir { + return syscall.EPERM + } + _, err := f.WriteAt([]byte(""), 0) + return err +} + +func (f *File) WriteString(s string) (int, error) { + return f.Write([]byte(s)) +} diff --git a/ossfs/file_info.go b/ossfs/file_info.go new file mode 100644 index 00000000..ea31d2da --- /dev/null +++ b/ossfs/file_info.go @@ -0,0 +1,17 @@ +package ossfs + +import ( + "time" + + "github.com/spf13/afero/ossfs/internal/utils" +) + +type FileInfo struct { + *utils.OssObjectMeta +} + +func NewFileInfo(name string, size int64, updatedAt time.Time) *FileInfo { + return &FileInfo{ + OssObjectMeta: utils.NewOssObjectMeta(name, size, updatedAt), + } +} diff --git a/ossfs/file_test.go b/ossfs/file_test.go new file mode 100644 index 00000000..fba21bd2 --- /dev/null +++ b/ossfs/file_test.go @@ -0,0 +1,296 @@ +package ossfs + +import ( + "io" + "os" + "strings" + "syscall" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/spf13/afero/ossfs/internal/mocks" + "github.com/spf13/afero/ossfs/internal/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func getMockedFs() *Fs { + fs := NewOssFs("test-ak", "test-sk", "test-region", "test-bucket") + fs.manager = &mocks.ObjectManager{} + return fs +} + +func getMockedFile(name string, flag int, fs *Fs) *File { + f, _ := NewOssFile(name, flag, fs) + return f +} + +func TestNewOssFile(t *testing.T) { + t.Run("create new file with read flag", func(t *testing.T) { + fs := &Fs{} + file, err := NewOssFile("testfile", os.O_RDONLY, fs) + assert.NoError(t, err) + assert.Equal(t, "testfile", file.name) + assert.Equal(t, os.O_RDONLY, file.openFlag) + assert.Equal(t, fs, file.fs) + assert.False(t, file.dirty) + assert.False(t, file.closed) + assert.False(t, file.isDir) + assert.False(t, file.preloaded) + assert.Nil(t, file.preloadedFd) + }) + + t.Run("create new file with write flag", func(t *testing.T) { + fs := &Fs{} + file, err := NewOssFile("testfile", os.O_WRONLY, fs) + assert.NoError(t, err) + assert.Equal(t, "testfile", file.name) + assert.Equal(t, os.O_WRONLY, file.openFlag) + assert.Equal(t, fs, file.fs) + assert.False(t, file.dirty) + assert.False(t, file.closed) + assert.False(t, file.isDir) + assert.False(t, file.preloaded) + assert.Nil(t, file.preloadedFd) + }) + + t.Run("create new directory", func(t *testing.T) { + fs := &Fs{} + file, err := NewOssFile("testdir/", os.O_RDONLY, fs) + assert.NoError(t, err) + assert.Equal(t, "testdir/", file.name) + assert.Equal(t, os.O_RDONLY, file.openFlag) + assert.Equal(t, fs, file.fs) + assert.False(t, file.dirty) + assert.False(t, file.closed) + assert.True(t, file.isDir) + assert.False(t, file.preloaded) + assert.Nil(t, file.preloadedFd) + }) + + t.Run("normalize file name", func(t *testing.T) { + fs := &Fs{} + file, err := NewOssFile("/path/testfile", os.O_RDONLY, fs) + assert.NoError(t, err) + assert.Equal(t, "path/testfile", file.name) + }) +} + +func TestRead(t *testing.T) { + t.Run("Read with unreadable flag return error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_WRONLY, fs) + + p := make([]byte, 0) + _, e := f.Read(p) + + assert.Error(t, e) + assert.NotNil(t, e) + }) + + t.Run("Read on directory return error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testdir", os.O_RDONLY, fs) + f.isDir = true + + p := make([]byte, 10) + _, e := f.Read(p) + + assert.Error(t, e) + assert.Equal(t, syscall.EPERM, e) + }) + + t.Run("Read on closed file return error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDONLY, fs) + f.closed = true + + p := make([]byte, 10) + _, e := f.Read(p) + + assert.Error(t, e) + assert.Equal(t, syscall.EPERM, e) + }) + + t.Run("Successful read updates offset", func(t *testing.T) { + fs := getMockedFs() + var cu utils.CleanUp = func() {} + mockManager := fs.manager.(*mocks.ObjectManager) + mockManager.On("GetObjectPart", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + // Return(&mockReadCloser{data: []byte("testdata")}, cu, nil) + Return(strings.NewReader("testdata"), cu, nil) + + f := getMockedFile("testfile", os.O_RDONLY, fs) + p := make([]byte, 8) + n, err := f.Read(p) + + assert.NoError(t, err) + assert.Equal(t, 8, n) + assert.Equal(t, int64(8), f.offset) + }) + + t.Run("ReadAt error propagates", func(t *testing.T) { + fs := getMockedFs() + mockManager := fs.manager.(*mocks.ObjectManager) + mockManager.On("GetObjectPart", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil, nil, syscall.EIO) + + f := getMockedFile("testfile", os.O_RDONLY, fs) + p := make([]byte, 8) + _, err := f.Read(p) + + assert.Error(t, err) + assert.Equal(t, syscall.EIO, err) + }) +} + +func TestReadAt(t *testing.T) { + t.Run("ReadAt with unreadable flag return error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_WRONLY, fs) + + p := make([]byte, 0) + _, e := f.ReadAt(p, 0) + + assert.Error(t, e) + assert.NotNil(t, e) + }) + + t.Run("ReadAt on dir return error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("/path/to/dir/", os.O_WRONLY, fs) + + p := make([]byte, 0) + _, e := f.ReadAt(p, 0) + + assert.Error(t, e) + assert.NotNil(t, e) + }) + + t.Run("ReadAt success", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDONLY, fs) + + p := make([]byte, 4) + + var cu utils.CleanUp = func() {} + off := int64(5) + m := &mocks.ObjectManager{} + m. + On("GetObjectPart", f.fs.ctx, f.fs.bucketName, f.name, off, off+int64(len(p))). + Return(strings.NewReader("test result"), cu, nil) + fs.manager = m + + n, e := f.ReadAt(p, off) + + assert.Nil(t, e) + assert.Equal(t, 4, n) + assert.Equal(t, "test", string(p)) + }) +} + +func TestSeek(t *testing.T) { + t.Run("Seek on unreadable/unwritable file returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_WRONLY, fs) + f.closed = true + + _, err := f.Seek(0, io.SeekStart) + assert.Error(t, err) + assert.Equal(t, syscall.EPERM, err) + }) + + t.Run("Seek on directory returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testdir", os.O_RDONLY, fs) + f.isDir = true + + _, err := f.Seek(0, io.SeekStart) + assert.Error(t, err) + assert.Equal(t, syscall.EPERM, err) + }) + + t.Run("SeekStart sets correct offset", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + + offset, err := f.Seek(10, io.SeekStart) + assert.NoError(t, err) + assert.Equal(t, int64(10), offset) + assert.Equal(t, int64(10), f.offset) + }) + + t.Run("SeekCurrent adjusts offset correctly", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + f.offset = 5 + + offset, err := f.Seek(5, io.SeekCurrent) + assert.NoError(t, err) + assert.Equal(t, int64(10), offset) + assert.Equal(t, int64(10), f.offset) + }) + + t.Run("SeekEnd adjusts offset correctly", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + + offset, err := f.Seek(-10, io.SeekEnd) + assert.NoError(t, err) + assert.Equal(t, int64(90), offset) + assert.Equal(t, int64(90), f.offset) + }) + + t.Run("Seek beyond file size returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + + _, err := f.Seek(101, io.SeekStart) + assert.Error(t, err) + assert.Equal(t, afero.ErrOutOfRange, err) + }) + + t.Run("Seek negative offset returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + + _, err := f.Seek(-1, io.SeekStart) + assert.Error(t, err) + assert.Equal(t, afero.ErrOutOfRange, err) + }) + + t.Run("Seek with invalid whence returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + + _, err := f.Seek(0, 3) + assert.Error(t, err) + }) +} + +func TestWrite(t *testing.T) { + t.Run("Write unwritable file returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDONLY, fs) + _, e := f.Write([]byte("test input string")) + + assert.Error(t, e) + assert.NotNil(t, e) + }) + + t.Run("Write dir returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("/path/to/test_dir/", os.O_WRONLY, fs) + _, e := f.Write([]byte("test input string")) + + assert.Error(t, e) + assert.NotNil(t, e) + }) +} diff --git a/ossfs/fs.go b/ossfs/fs.go new file mode 100644 index 00000000..05ef2ffd --- /dev/null +++ b/ossfs/fs.go @@ -0,0 +1,188 @@ +package ossfs + +import ( + "context" + "errors" + "os" + "strings" + "time" + + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials" + "github.com/spf13/afero" + "github.com/spf13/afero/ossfs/internal/utils" +) + +const ( + defaultFileMode = 0o755 + defaultFileFlag = os.O_RDWR +) + +type Fs struct { + manager utils.ObjectManager + bucketName string + separator string + autoSync bool + openedFiles map[string]afero.File + preloadFs afero.Fs + ctx context.Context +} + +func NewOssFs(accessKeyId, accessKeySecret, region, bucket string) *Fs { + ossCfg := oss.LoadDefaultConfig(). + WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyId, accessKeySecret)). + WithRegion(region) + + return &Fs{ + manager: &utils.OssObjectManager{ + Client: oss.NewClient(ossCfg), + }, + bucketName: bucket, + separator: "/", + autoSync: true, + openedFiles: make(map[string]afero.File), + preloadFs: afero.NewMemMapFs(), + ctx: context.Background(), + } +} + +func (fs *Fs) WithContext(ctx context.Context) *Fs { + fs.ctx = ctx + return fs +} + +// Create creates a new empty file and open it, return the open file and error +// if any happens. +func (fs *Fs) Create(name string) (afero.File, error) { + n := fs.normFileName(name) + r := strings.NewReader("") + if _, err := fs.manager.PutObject(fs.ctx, fs.bucketName, n, r); err != nil { + return nil, err + } + return NewOssFile(n, defaultFileFlag, fs) +} + +// Mkdir creates a directory in the filesystem, return an error if any +// happens. +func (fs *Fs) Mkdir(name string, perm os.FileMode) error { + return fs.MkdirAll(fs.ensureAsDir(name), perm) +} + +// MkdirAll creates a directory path and all parents that does not exist +// yet. +func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { + dirName := fs.ensureAsDir(path) + r := strings.NewReader("") + _, err := fs.manager.PutObject(fs.ctx, fs.bucketName, dirName, r) + return err +} + +// Open opens a file, returning it or an error, if any happens. +func (fs *Fs) Open(name string) (afero.File, error) { + return fs.OpenFile(name, defaultFileFlag, defaultFileMode) +} + +// OpenFile opens a file using the given flags and the given mode. +func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + name = fs.normFileName(name) + file, found := fs.openedFiles[name] + if found && file.(*File).openFlag == flag { + return file, nil + } + + f, err := NewOssFile(name, flag, fs) + if err != nil { + return nil, err + } + + existed := false + existed, err = fs.manager.IsObjectExist(fs.ctx, fs.bucketName, name) + if err != nil { + return nil, err + } + + if !existed && f.openFlag&os.O_CREATE == 0 { + return nil, afero.ErrFileNotFound + } + + if !existed && f.openFlag*os.O_CREATE != 0 { + if _, err := fs.Create(f.name); err != nil { + return nil, err + } + } + + if f.openFlag&os.O_TRUNC != 0 { + _, err := f.fs.manager.PutObject(fs.ctx, fs.bucketName, f.name, strings.NewReader("")) + if err != nil { + return nil, err + } + } + + fs.openedFiles[name] = f + + return f, nil +} + +// Remove removes a file identified by name, returning an error, if any +// happens. +func (fs *Fs) Remove(name string) error { + return fs.manager.DeleteObject(fs.ctx, fs.bucketName, fs.normFileName(name)) +} + +// RemoveAll removes a directory path and any children it contains. It +// does not fail if the path does not exist (return nil). +func (fs *Fs) RemoveAll(path string) error { + dir := fs.ensureAsDir(path) + fis, err := fs.manager.ListAllObjects(fs.ctx, fs.bucketName, dir) + if err != nil { + return err + } + for _, fi := range fis { + err = fs.manager.DeleteObject(fs.ctx, fs.bucketName, fi.Name()) + if err != nil { + return err + } + } + return nil +} + +// Rename renames a file. +func (fs *Fs) Rename(oldname, newname string) error { + err := fs.manager.CopyObject(fs.ctx, fs.bucketName, oldname, newname) + if err != nil { + return err + } + err = fs.manager.DeleteObject(fs.ctx, fs.bucketName, oldname) + return err +} + +// Stat returns a FileInfo describing the named file, or an error, if any +// happens. +func (fs *Fs) Stat(name string) (os.FileInfo, error) { + fi, err := fs.manager.GetObjectMeta(fs.ctx, fs.bucketName, fs.normFileName(name)) + if err != nil { + return nil, err + } + + return fi, err +} + +// The name of this FileSystem +func (fs *Fs) Name() string { + return "OssFs" +} + +// Chmod changes the mode of the named file to mode. +func (fs *Fs) Chmod(name string, mode os.FileMode) error { + return errors.New("OSS: method Chmod is not implemented") +} + +// Chown changes the uid and gid of the named file. +func (fs *Fs) Chown(name string, uid, gid int) error { + return errors.New("OSS: method Chown is not implemented") +} + +// Chtimes changes the access and modification times of the named file +func (fs *Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return errors.New("OSS: method Chtimes is not implemented") +} diff --git a/ossfs/fs_test.go b/ossfs/fs_test.go new file mode 100644 index 00000000..8d09ecf8 --- /dev/null +++ b/ossfs/fs_test.go @@ -0,0 +1,462 @@ +package ossfs + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/spf13/afero/ossfs/internal/mocks" + "github.com/stretchr/testify/assert" +) + +func TestNewOssFs(t *testing.T) { + tests := []struct { + name string + accessKeyId string + accessKeySecret string + region string + bucket string + expected *Fs + }{ + { + name: "valid credentials", + accessKeyId: "testKeyId", + accessKeySecret: "testKeySecret", + region: "test-region", + bucket: "test-bucket", + expected: &Fs{ + bucketName: "test-bucket", + separator: "/", + autoSync: true, + openedFiles: make(map[string]afero.File), + preloadFs: afero.NewMemMapFs(), + ctx: context.Background(), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewOssFs(tt.accessKeyId, tt.accessKeySecret, tt.region, tt.bucket) + assert.NotNil(t, got.manager) + assert.Equal(t, tt.expected.bucketName, got.bucketName) + assert.Equal(t, tt.expected.separator, got.separator) + assert.Equal(t, tt.expected.autoSync, got.autoSync) + assert.NotNil(t, got.openedFiles) + assert.NotNil(t, got.preloadFs) + assert.NotNil(t, got.ctx) + }) + } +} + +func TestFsWithContext(t *testing.T) { + type bgMeta string + tests := []struct { + name string + fs *Fs + ctx context.Context + expected *Fs + }{ + { + name: "set new context", + fs: &Fs{ + ctx: context.Background(), + }, + ctx: context.WithValue(context.Background(), bgMeta("testKey"), bgMeta("testValue")), + expected: &Fs{ + ctx: context.WithValue(context.Background(), bgMeta("testKey"), bgMeta("testValue")), + }, + }, + { + name: "set nil context", + fs: &Fs{ + ctx: context.Background(), + }, + ctx: nil, + expected: &Fs{ + ctx: nil, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.fs.WithContext(tt.ctx) + assert.Equal(t, tt.expected.ctx, got.ctx) + assert.Equal(t, tt.fs, got) + }) + } +} + +func TestFsCreate(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("create simple success", func(t *testing.T) { + m.On("PutObject", fs.ctx, bucket, "test.txt", strings.NewReader("")).Return(true, nil).Once() + file, err := fs.Create("test.txt") + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + m.AssertExpectations(t) + }) + + t.Run("create prefixed file path success", func(t *testing.T) { + m. + On("PutObject", fs.ctx, bucket, "path/to/test.txt", strings.NewReader("")). + Return(true, nil). + Once() + file, err := fs.Create("/path/to/test.txt") + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + m.AssertExpectations(t) + }) + + t.Run("create dir path success", func(t *testing.T) { + m. + On("PutObject", ctx, bucket, "path/to/test_dir/", strings.NewReader("")). + Return(true, nil). + Once() + file, err := fs.Create("/path/to/test_dir/") + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + assert.Equal(t, "path/to/test_dir/", file.Name()) + m.AssertExpectations(t) + }) + + t.Run("create failure", func(t *testing.T) { + m.On("PutObject", fs.ctx, bucket, "test2.txt", strings.NewReader("")).Return(false, afero.ErrFileNotFound).Once() + _, err := fs.Create("test2.txt") + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) +} + +func TestFsMkdirAll(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("MkDirAll simple success", func(t *testing.T) { + m. + On("PutObject", fs.ctx, bucket, "path/to/test_dir/", strings.NewReader("")). + Return(true, nil). + Once() + err := fs.MkdirAll("/path/to/test_dir/", defaultFileMode) + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("MkDirAll failure", func(t *testing.T) { + m. + On("PutObject", fs.ctx, bucket, "path/to/test_dir/", strings.NewReader("")). + Return(false, afero.ErrFileClosed). + Once() + err := fs.MkdirAll("/path/to/test_dir/", defaultFileMode) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileClosed) + m.AssertExpectations(t) + }) +} + +func TestFsOpenFile(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + openedFiles: make(map[string]afero.File), + } + + t.Run("open existing file success", func(t *testing.T) { + m.On("IsObjectExist", ctx, bucket, "test.txt").Return(true, nil).Once() + file, err := fs.OpenFile("test.txt", os.O_RDONLY, 0644) + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + m.AssertExpectations(t) + }) + + t.Run("open non-existing file with create flag success", func(t *testing.T) { + m.On("IsObjectExist", ctx, bucket, "new.txt").Return(false, nil).Once() + m.On("PutObject", ctx, bucket, "new.txt", strings.NewReader("")).Return(true, nil).Once() + file, err := fs.OpenFile("new.txt", os.O_CREATE|os.O_RDWR, 0644) + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + m.AssertExpectations(t) + }) + + t.Run("open file with truncate flag success", func(t *testing.T) { + m.On("IsObjectExist", ctx, bucket, "trunc.txt").Return(true, nil).Once() + m.On("PutObject", ctx, bucket, "trunc.txt", strings.NewReader("")).Return(true, nil).Once() + file, err := fs.OpenFile("trunc.txt", os.O_TRUNC|os.O_RDWR, 0644) + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + m.AssertExpectations(t) + }) + + t.Run("open existing file from cache", func(t *testing.T) { + cachedFile := &File{name: "cached.txt", openFlag: os.O_RDONLY} + fs.openedFiles["cached.txt"] = cachedFile + file, err := fs.OpenFile("cached.txt", os.O_RDONLY, 0644) + assert.Nil(t, err) + assert.Equal(t, cachedFile, file) + assert.Implements(t, (*afero.File)(nil), file) + }) + + t.Run("open non-existing file without create flag fails", func(t *testing.T) { + m.On("IsObjectExist", ctx, bucket, "nonexist.txt").Return(false, nil).Once() + _, err := fs.OpenFile("nonexist.txt", os.O_RDONLY, 0644) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) + + t.Run("open file with check exist error fails", func(t *testing.T) { + m.On("IsObjectExist", ctx, bucket, "error.txt").Return(false, afero.ErrFileNotFound).Once() + _, err := fs.OpenFile("error.txt", os.O_RDONLY, 0644) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) +} + +func TestFsRemove(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("remove file success", func(t *testing.T) { + m.On("DeleteObject", fs.ctx, bucket, "test.txt").Return(nil).Once() + err := fs.Remove("test.txt") + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("remove prefixed file success", func(t *testing.T) { + m.On("DeleteObject", fs.ctx, bucket, "path/to/test.txt").Return(nil).Once() + err := fs.Remove("/path/to/test.txt") + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("remove non-existent file", func(t *testing.T) { + m.On("DeleteObject", fs.ctx, bucket, "nonexistent.txt").Return(afero.ErrFileNotFound).Once() + err := fs.Remove("nonexistent.txt") + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) +} + +func TestFsRemoveAll(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("remove non-empty directory", func(t *testing.T) { + dirPath := "path/to/dir/" + files := []os.FileInfo{ + NewFileInfo("path/to/dir/file1.txt", 100, time.Now()), + NewFileInfo("path/to/dir/file2.txt", 200, time.Now()), + NewFileInfo("path/to/dir/subdir/", 0, time.Now()), + } + + m.On("ListAllObjects", ctx, bucket, dirPath).Return(files, nil).Once() + m.On("DeleteObject", ctx, bucket, "path/to/dir/file1.txt").Return(nil).Once() + m.On("DeleteObject", ctx, bucket, "path/to/dir/file2.txt").Return(nil).Once() + m.On("DeleteObject", ctx, bucket, "path/to/dir/subdir/").Return(nil).Once() + + err := fs.RemoveAll(dirPath) + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("remove empty directory", func(t *testing.T) { + dirPath := "empty/dir/" + m.On("ListAllObjects", ctx, bucket, dirPath).Return([]os.FileInfo{}, nil).Once() + + err := fs.RemoveAll(dirPath) + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("remove non-existent path", func(t *testing.T) { + nonExistentPath := "nonexistent/path/" + m.On("ListAllObjects", ctx, bucket, nonExistentPath).Return([]os.FileInfo{}, nil).Once() + + err := fs.RemoveAll(nonExistentPath) + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("list objects failure", func(t *testing.T) { + dirPath := "path/to/dir/" + m.On("ListAllObjects", ctx, bucket, dirPath).Return(nil, afero.ErrFileNotFound).Once() + + err := fs.RemoveAll(dirPath) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) + + t.Run("delete object failure", func(t *testing.T) { + dirPath := "path/to/dir/" + files := []os.FileInfo{ + NewFileInfo("path/to/dir/file1.txt", 0, time.Now()), + } + + m.On("ListAllObjects", ctx, bucket, dirPath).Return(files, nil).Once() + m.On("DeleteObject", ctx, bucket, "path/to/dir/file1.txt").Return(afero.ErrFileNotFound).Once() + + err := fs.RemoveAll(dirPath) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) +} + +func TestFsRename(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("successful rename", func(t *testing.T) { + oldname := "old/file.txt" + newname := "new/file.txt" + + m.On("CopyObject", ctx, bucket, oldname, newname).Return(nil).Once() + m.On("DeleteObject", ctx, bucket, oldname).Return(nil).Once() + + err := fs.Rename(oldname, newname) + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("copy failure", func(t *testing.T) { + oldname := "old/file.txt" + newname := "new/file.txt" + + m.On("CopyObject", ctx, bucket, oldname, newname).Return(afero.ErrFileNotFound).Once() + + err := fs.Rename(oldname, newname) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) + + t.Run("delete failure after successful copy", func(t *testing.T) { + oldname := "old/file.txt" + newname := "new/file.txt" + + m.On("CopyObject", ctx, bucket, oldname, newname).Return(nil).Once() + m.On("DeleteObject", ctx, bucket, oldname).Return(afero.ErrFileNotFound).Once() + + err := fs.Rename(oldname, newname) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) +} + +func TestFsStat(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("stat file success", func(t *testing.T) { + expectedInfo := &mocks.FileInfo{} + m.On("GetObjectMeta", fs.ctx, bucket, "test.txt").Return(expectedInfo, nil).Once() + info, err := fs.Stat("test.txt") + assert.Nil(t, err) + assert.Equal(t, expectedInfo, info) + m.AssertExpectations(t) + }) + + t.Run("stat prefixed file path success", func(t *testing.T) { + expectedInfo := &mocks.FileInfo{} + m.On("GetObjectMeta", fs.ctx, bucket, "path/to/test.txt").Return(expectedInfo, nil).Once() + info, err := fs.Stat("/path/to/test.txt") + assert.Nil(t, err) + assert.Equal(t, expectedInfo, info) + m.AssertExpectations(t) + }) + + t.Run("stat dir path success", func(t *testing.T) { + expectedInfo := &mocks.FileInfo{} + m.On("GetObjectMeta", fs.ctx, bucket, "path/to/dir/").Return(expectedInfo, nil).Once() + info, err := fs.Stat("/path/to/dir/") + assert.Nil(t, err) + assert.Equal(t, expectedInfo, info) + m.AssertExpectations(t) + }) + + t.Run("stat non-existent file", func(t *testing.T) { + m.On("GetObjectMeta", fs.ctx, bucket, "nonexistent.txt").Return(nil, os.ErrNotExist).Once() + _, err := fs.Stat("nonexistent.txt") + assert.NotNil(t, err) + assert.ErrorIs(t, err, os.ErrNotExist) + m.AssertExpectations(t) + }) +} + +func TestFsName(t *testing.T) { + fs := &Fs{} + name := fs.Name() + assert.Equal(t, "OssFs", name) +} diff --git a/ossfs/fs_utils.go b/ossfs/fs_utils.go new file mode 100644 index 00000000..2a14609c --- /dev/null +++ b/ossfs/fs_utils.go @@ -0,0 +1,36 @@ +package ossfs + +import ( + "strings" +) + +func (fs *Fs) isDir(s string) bool { + sep := fs.separator + if fs.separator == "" { + sep = "/" + } + return strings.HasSuffix(s, sep) +} + +func (fs *Fs) ensureAsDir(s string) string { + sep := fs.separator + if fs.separator == "" { + sep = "/" + } + s = fs.normFileName(s) + if !strings.HasSuffix(s, sep) { + s = s + sep + } + return s +} + +func (fs *Fs) normFileName(s string) string { + sep := fs.separator + if fs.separator == "" { + sep = "/" + } + s = strings.TrimLeft(s, "/\\") + s = strings.Replace(s, "\\", sep, -1) + s = strings.Replace(s, "/", sep, -1) + return s +} diff --git a/ossfs/init.go b/ossfs/init.go new file mode 100644 index 00000000..3a8ae873 --- /dev/null +++ b/ossfs/init.go @@ -0,0 +1,14 @@ +package ossfs + +import ( + "os" + + "github.com/spf13/afero" +) + +func init() { + // Ensure oss.Fs implements afero.Fs interface + var _ afero.Fs = (*Fs)(nil) + var _ afero.File = (*File)(nil) + var _ os.FileInfo = (*FileInfo)(nil) +} diff --git a/ossfs/internal/mocks/FileInfo.go b/ossfs/internal/mocks/FileInfo.go new file mode 100644 index 00000000..64328f2b --- /dev/null +++ b/ossfs/internal/mocks/FileInfo.go @@ -0,0 +1,139 @@ +// Code generated by mockery v2.53.3. DO NOT EDIT. + +package mocks + +import ( + "os" + time "time" + + mock "github.com/stretchr/testify/mock" +) + +// FileInfo is an autogenerated mock type for the FileInfo type +type FileInfo struct { + mock.Mock +} + +// IsDir provides a mock function with no fields +func (_m *FileInfo) IsDir() bool { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for IsDir") + } + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// ModTime provides a mock function with no fields +func (_m *FileInfo) ModTime() time.Time { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ModTime") + } + + var r0 time.Time + if rf, ok := ret.Get(0).(func() time.Time); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Time) + } + + return r0 +} + +// Mode provides a mock function with no fields +func (_m *FileInfo) Mode() os.FileMode { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Mode") + } + + var r0 os.FileMode + if rf, ok := ret.Get(0).(func() os.FileMode); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(os.FileMode) + } + + return r0 +} + +// Name provides a mock function with no fields +func (_m *FileInfo) Name() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Name") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// Size provides a mock function with no fields +func (_m *FileInfo) Size() int64 { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Size") + } + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + return r0 +} + +// Sys provides a mock function with no fields +func (_m *FileInfo) Sys() interface{} { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Sys") + } + + var r0 interface{} + if rf, ok := ret.Get(0).(func() interface{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interface{}) + } + } + + return r0 +} + +// NewFileInfo creates a new instance of FileInfo. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewFileInfo(t interface { + mock.TestingT + Cleanup(func()) +}) *FileInfo { + mock := &FileInfo{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/ossfs/internal/mocks/ObjectManager.go b/ossfs/internal/mocks/ObjectManager.go new file mode 100644 index 00000000..bb3e087b --- /dev/null +++ b/ossfs/internal/mocks/ObjectManager.go @@ -0,0 +1,293 @@ +// Code generated by mockery v2.53.3. DO NOT EDIT. + +package mocks + +import ( + context "context" + fs "io/fs" + + io "io" + + mock "github.com/stretchr/testify/mock" + + utils "github.com/spf13/afero/ossfs/internal/utils" +) + +// ObjectManager is an autogenerated mock type for the ObjectManager type +type ObjectManager struct { + mock.Mock +} + +// CopyObject provides a mock function with given fields: ctx, bucket, srcName, targetName +func (_m *ObjectManager) CopyObject(ctx context.Context, bucket string, srcName string, targetName string) error { + ret := _m.Called(ctx, bucket, srcName, targetName) + + if len(ret) == 0 { + panic("no return value specified for CopyObject") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string) error); ok { + r0 = rf(ctx, bucket, srcName, targetName) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DeleteObject provides a mock function with given fields: ctx, bucket, name +func (_m *ObjectManager) DeleteObject(ctx context.Context, bucket string, name string) error { + ret := _m.Called(ctx, bucket, name) + + if len(ret) == 0 { + panic("no return value specified for DeleteObject") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, bucket, name) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetObject provides a mock function with given fields: ctx, bucket, name +func (_m *ObjectManager) GetObject(ctx context.Context, bucket string, name string) (io.Reader, utils.CleanUp, error) { + ret := _m.Called(ctx, bucket, name) + + if len(ret) == 0 { + panic("no return value specified for GetObject") + } + + var r0 io.Reader + var r1 utils.CleanUp + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (io.Reader, utils.CleanUp, error)); ok { + return rf(ctx, bucket, name) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) io.Reader); ok { + r0 = rf(ctx, bucket, name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.Reader) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) utils.CleanUp); ok { + r1 = rf(ctx, bucket, name) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(utils.CleanUp) + } + } + + if rf, ok := ret.Get(2).(func(context.Context, string, string) error); ok { + r2 = rf(ctx, bucket, name) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// GetObjectMeta provides a mock function with given fields: ctx, bucket, name +func (_m *ObjectManager) GetObjectMeta(ctx context.Context, bucket string, name string) (fs.FileInfo, error) { + ret := _m.Called(ctx, bucket, name) + + if len(ret) == 0 { + panic("no return value specified for GetObjectMeta") + } + + var r0 fs.FileInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (fs.FileInfo, error)); ok { + return rf(ctx, bucket, name) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) fs.FileInfo); ok { + r0 = rf(ctx, bucket, name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(fs.FileInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, bucket, name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetObjectPart provides a mock function with given fields: ctx, bucket, name, start, end +func (_m *ObjectManager) GetObjectPart(ctx context.Context, bucket string, name string, start int64, end int64) (io.Reader, utils.CleanUp, error) { + ret := _m.Called(ctx, bucket, name, start, end) + + if len(ret) == 0 { + panic("no return value specified for GetObjectPart") + } + + var r0 io.Reader + var r1 utils.CleanUp + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64) (io.Reader, utils.CleanUp, error)); ok { + return rf(ctx, bucket, name, start, end) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64) io.Reader); ok { + r0 = rf(ctx, bucket, name, start, end) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.Reader) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, int64, int64) utils.CleanUp); ok { + r1 = rf(ctx, bucket, name, start, end) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(utils.CleanUp) + } + } + + if rf, ok := ret.Get(2).(func(context.Context, string, string, int64, int64) error); ok { + r2 = rf(ctx, bucket, name, start, end) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// IsObjectExist provides a mock function with given fields: ctx, bucket, name +func (_m *ObjectManager) IsObjectExist(ctx context.Context, bucket string, name string) (bool, error) { + ret := _m.Called(ctx, bucket, name) + + if len(ret) == 0 { + panic("no return value specified for IsObjectExist") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (bool, error)); ok { + return rf(ctx, bucket, name) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) bool); ok { + r0 = rf(ctx, bucket, name) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, bucket, name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListAllObjects provides a mock function with given fields: ctx, bucket, prefix +func (_m *ObjectManager) ListAllObjects(ctx context.Context, bucket string, prefix string) ([]fs.FileInfo, error) { + ret := _m.Called(ctx, bucket, prefix) + + if len(ret) == 0 { + panic("no return value specified for ListAllObjects") + } + + var r0 []fs.FileInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]fs.FileInfo, error)); ok { + return rf(ctx, bucket, prefix) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) []fs.FileInfo); ok { + r0 = rf(ctx, bucket, prefix) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]fs.FileInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, bucket, prefix) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListObjects provides a mock function with given fields: ctx, bucket, prefix, count +func (_m *ObjectManager) ListObjects(ctx context.Context, bucket string, prefix string, count int) ([]fs.FileInfo, error) { + ret := _m.Called(ctx, bucket, prefix, count) + + if len(ret) == 0 { + panic("no return value specified for ListObjects") + } + + var r0 []fs.FileInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, int) ([]fs.FileInfo, error)); ok { + return rf(ctx, bucket, prefix, count) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, int) []fs.FileInfo); ok { + r0 = rf(ctx, bucket, prefix, count) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]fs.FileInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, int) error); ok { + r1 = rf(ctx, bucket, prefix, count) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// PutObject provides a mock function with given fields: ctx, bucket, name, reader +func (_m *ObjectManager) PutObject(ctx context.Context, bucket string, name string, reader io.Reader) (bool, error) { + ret := _m.Called(ctx, bucket, name, reader) + + if len(ret) == 0 { + panic("no return value specified for PutObject") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, io.Reader) (bool, error)); ok { + return rf(ctx, bucket, name, reader) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, io.Reader) bool); ok { + r0 = rf(ctx, bucket, name, reader) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, io.Reader) error); ok { + r1 = rf(ctx, bucket, name, reader) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewObjectManager creates a new instance of ObjectManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewObjectManager(t interface { + mock.TestingT + Cleanup(func()) +}) *ObjectManager { + mock := &ObjectManager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/ossfs/internal/utils/contract.go b/ossfs/internal/utils/contract.go new file mode 100644 index 00000000..9c458ec0 --- /dev/null +++ b/ossfs/internal/utils/contract.go @@ -0,0 +1,21 @@ +package utils + +import ( + "context" + "io" + "os" +) + +type CleanUp func() + +type ObjectManager interface { + GetObject(ctx context.Context, bucket, name string) (io.Reader, CleanUp, error) + GetObjectPart(ctx context.Context, bucket, name string, start, end int64) (io.Reader, CleanUp, error) + DeleteObject(ctx context.Context, bucket, name string) error + IsObjectExist(ctx context.Context, bucket, name string) (bool, error) + PutObject(ctx context.Context, bucket, name string, reader io.Reader) (bool, error) + CopyObject(ctx context.Context, bucket, srcName, targetName string) error + GetObjectMeta(ctx context.Context, bucket, name string) (os.FileInfo, error) + ListObjects(ctx context.Context, bucket, prefix string, count int) ([]os.FileInfo, error) + ListAllObjects(ctx context.Context, bucket, prefix string) ([]os.FileInfo, error) +} diff --git a/ossfs/internal/utils/init.go b/ossfs/internal/utils/init.go new file mode 100644 index 00000000..72709b88 --- /dev/null +++ b/ossfs/internal/utils/init.go @@ -0,0 +1,6 @@ +package utils + +func init() { + // Ensure OssObjectManager implements ObjectManager interface + var _ ObjectManager = (*OssObjectManager)(nil) +} diff --git a/ossfs/internal/utils/oss.go b/ossfs/internal/utils/oss.go new file mode 100644 index 00000000..6014d71d --- /dev/null +++ b/ossfs/internal/utils/oss.go @@ -0,0 +1,210 @@ +package utils + +import ( + "context" + "fmt" + "io" + "io/fs" + "os" + "strings" + "time" + + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" + "github.com/spf13/afero" +) + +var ossDirSeparator string = "/" +var ossDefaultFileMode fs.FileMode = 0o755 + +type OssObjectManager struct { + ObjectManager + Client *oss.Client +} + +func (m *OssObjectManager) GetObject(ctx context.Context, bucket, name string) (io.Reader, CleanUp, error) { + req := &oss.GetObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(name), + } + res, err := m.Client.GetObject(ctx, req) + cleanUp := func() { + res.Body.Close() + } + return res.Body, cleanUp, err +} + +func (m *OssObjectManager) GetObjectPart(ctx context.Context, bucket, name string, start, end int64) (io.Reader, CleanUp, error) { + if start > end { + return nil, nil, afero.ErrOutOfRange + } + req := &oss.GetObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(name), + Range: oss.Ptr(fmt.Sprintf("bytes=%v-%v", start, end)), + RangeBehavior: oss.Ptr("standard"), + } + res, err := m.Client.GetObject(ctx, req) + cleanUp := func() { + res.Body.Close() + } + return res.Body, cleanUp, err +} + +func (m *OssObjectManager) DeleteObject(ctx context.Context, bucket, name string) error { + req := &oss.DeleteObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(name), + } + _, err := m.Client.DeleteObject(ctx, req) + return err +} + +func (m *OssObjectManager) IsObjectExist(ctx context.Context, bucket, name string) (bool, error) { + return m.Client.IsObjectExist(ctx, bucket, name) +} + +func (m *OssObjectManager) PutObject(ctx context.Context, bucket, name string, reader io.Reader) (bool, error) { + req := &oss.PutObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(name), + Body: reader, + } + _, err := m.Client.PutObject(ctx, req) + if err != nil { + return false, err + } + return true, nil +} + +func (m *OssObjectManager) CopyObject(ctx context.Context, bucket, srcName, targetName string) error { + req := &oss.CopyObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(srcName), + SourceKey: oss.Ptr(targetName), + SourceBucket: oss.Ptr(bucket), + StorageClass: oss.StorageClassStandard, + } + _, err := m.Client.CopyObject(ctx, req) + return err +} + +func (m *OssObjectManager) GetObjectMeta(ctx context.Context, bucket, name string) (os.FileInfo, error) { + req := &oss.HeadObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(name), + } + + res, err := m.Client.HeadObject(ctx, req) + if err != nil { + return nil, err + } + return &OssObjectMeta{ + name: name, + size: res.ContentLength, + lastModifiedAt: *res.LastModified, + }, nil +} + +func (m *OssObjectManager) ListObjects(ctx context.Context, bucket, prefix string, count int) ([]os.FileInfo, error) { + req := &oss.ListObjectsV2Request{ + Bucket: oss.Ptr(bucket), + Delimiter: oss.Ptr(ossDirSeparator), + Prefix: oss.Ptr(prefix), + } + p := m.Client.NewListObjectsV2Paginator(req) + + s := make([]os.FileInfo, 0) + + var i int + +loop: + for p.HasNext() { + page, err := p.NextPage(ctx) + if err != nil { + return nil, err + } + + for _, obj := range page.Contents { + i++ + if i == count { + break loop + } + s = append(s, &OssObjectMeta{ + name: oss.ToString(obj.Key), + size: obj.Size, + lastModifiedAt: oss.ToTime(obj.LastModified), + }) + } + } + + return s, nil +} + +func (m *OssObjectManager) ListAllObjects(ctx context.Context, bucket, prefix string) ([]os.FileInfo, error) { + req := &oss.ListObjectsV2Request{ + Bucket: oss.Ptr(bucket), + Prefix: oss.Ptr(prefix), + } + p := m.Client.NewListObjectsV2Paginator(req) + + s := make([]os.FileInfo, 0) + + for p.HasNext() { + page, err := p.NextPage(ctx) + if err != nil { + return nil, err + } + + for _, obj := range page.Contents { + s = append(s, &OssObjectMeta{ + name: oss.ToString(obj.Key), + size: obj.Size, + lastModifiedAt: oss.ToTime(obj.LastModified), + }) + } + } + + return s, nil +} + +type OssObjectMeta struct { + os.FileInfo + name string + size int64 + lastModifiedAt time.Time +} + +func NewOssObjectMeta(name string, size int64, updatedAt time.Time) *OssObjectMeta { + return &OssObjectMeta{ + name: name, + size: size, + lastModifiedAt: updatedAt, + } +} + +func (objMeta *OssObjectMeta) isDir() bool { + return strings.HasSuffix(objMeta.name, ossDirSeparator) +} + +func (objMeta *OssObjectMeta) ModTime() time.Time { + return objMeta.lastModifiedAt +} + +func (objMeta *OssObjectMeta) Mode() fs.FileMode { + if objMeta.isDir() { + return ossDefaultFileMode | fs.ModeDir + } + return ossDefaultFileMode +} + +func (objMeta *OssObjectMeta) Name() string { + return objMeta.name +} + +func (objMeta *OssObjectMeta) Size() int64 { + return objMeta.size +} + +func (objMeta *OssObjectMeta) Sys() any { + return nil +} From 63f1b49342436dd6b28b304a64a2548e8f1cad11 Mon Sep 17 00:00:00 2001 From: ahkui <14049597+ahkui@users.noreply.github.com> Date: Wed, 2 Apr 2025 11:49:02 +0000 Subject: [PATCH 05/34] fix(gcsfs): update object not exist check logic Signed-off-by: ahkui --- gcsfs/errors.go | 4 +++- gcsfs/file_info.go | 3 ++- gcsfs/gcs_mocks.go | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/gcsfs/errors.go b/gcsfs/errors.go index 201cd676..16f9d29c 100644 --- a/gcsfs/errors.go +++ b/gcsfs/errors.go @@ -19,13 +19,15 @@ package gcsfs import ( "errors" "syscall" + + "cloud.google.com/go/storage" ) var ( ErrNoBucketInName = errors.New("no bucket name found in the name") ErrFileClosed = errors.New("file is closed") ErrOutOfRange = errors.New("out of range") - ErrObjectDoesNotExist = errors.New("storage: object doesn't exist") + ErrObjectDoesNotExist = storage.ErrObjectNotExist ErrEmptyObjectName = errors.New("storage: object name is empty") ErrFileNotFound = syscall.ENOENT ) diff --git a/gcsfs/file_info.go b/gcsfs/file_info.go index 92e30460..0591b312 100644 --- a/gcsfs/file_info.go +++ b/gcsfs/file_info.go @@ -17,6 +17,7 @@ package gcsfs import ( + "errors" "os" "path/filepath" "strings" @@ -58,7 +59,7 @@ func newFileInfo(name string, fs *Fs, fileMode os.FileMode) (*FileInfo, error) { res.name = fs.ensureTrailingSeparator(res.name) res.isDir = true return res, nil - } else if err.Error() == ErrObjectDoesNotExist.Error() { + } else if errors.Is(err, ErrObjectDoesNotExist) { // Folders do not actually "exist" in GCloud, so we have to check, if something exists with // such a prefix bucketName, bucketPath := fs.splitName(name) diff --git a/gcsfs/gcs_mocks.go b/gcsfs/gcs_mocks.go index 9e64970a..d13f391f 100644 --- a/gcsfs/gcs_mocks.go +++ b/gcsfs/gcs_mocks.go @@ -130,7 +130,7 @@ func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) { if info.IsDir() { // we have to mock it here, because of FileInfo logic - return nil, ErrObjectDoesNotExist + return nil, storage.ErrObjectNotExist } return res, nil From 2ee23eec5eee2797effb029d2c545ff81f9505bc Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Sat, 12 Apr 2025 14:08:58 +0200 Subject: [PATCH 06/34] chore: update golangci-lint Signed-off-by: Mark Sagi-Kazar --- .editorconfig | 3 ++ .github/workflows/ci.yaml | 4 +-- .golangci.yaml | 61 +++++++++++++++++++++++++++++---------- 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/.editorconfig b/.editorconfig index 4492e9f9..a85749f1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,3 +10,6 @@ trim_trailing_whitespace = true [*.go] indent_style = tab + +[{*.yml,*.yaml}] +indent_size = 2 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e56359a0..b991f783 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,6 +50,6 @@ jobs: go-version: "1.24" - name: Lint - uses: golangci/golangci-lint-action@55c2c1448f86e01eaae002a5a3a9624417608d84 # v6.5.2 + uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd # v7.0.0 with: - version: v1.64.7 + version: v2.0.2 diff --git a/.golangci.yaml b/.golangci.yaml index 806289a2..fbba3201 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,18 +1,47 @@ -linters-settings: - gci: - sections: - - standard - - default - - prefix(github.com/spf13/afero) +version: "2" + +run: + timeout: 10m linters: - disable-all: true - enable: - - gci - - gofmt - - gofumpt - - staticcheck - -issues: - exclude-dirs: - - gcsfs/internal/stiface + enable: + - govet + - ineffassign + # - misspell + - nolintlint + # - revive + - unused + + disable: + - errcheck + - staticcheck + + settings: + misspell: + locale: US + nolintlint: + allow-unused: false # report any unused nolint directives + require-specific: false # don't require nolint directives to be specific about which linter is being skipped + + exclusions: + paths: + - gcsfs/internal/stiface + +formatters: + enable: + - gci + - gofmt + # - gofumpt + - goimports + # - golines + + settings: + gci: + sections: + - standard + - default + - localmodule + + exclusions: + paths: + - gcsfs/internal/stiface From a991449002ca0cee55185ae48879a06e6b9442ae Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Sat, 12 Apr 2025 14:09:23 +0200 Subject: [PATCH 07/34] ci: add dependency review Signed-off-by: Mark Sagi-Kazar --- .github/workflows/ci.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b991f783..a9e7123b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,3 +53,15 @@ jobs: uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd # v7.0.0 with: version: v2.0.2 + + dependency-review: + name: Dependency review + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Dependency Review + uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0 From 66924a372a4ed1e700a65d7c859d6acd0db98354 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Sat, 12 Apr 2025 14:09:58 +0200 Subject: [PATCH 08/34] ci: add scorecard analysis Signed-off-by: Mark Sagi-Kazar --- .github/workflows/analysis-scorecard.yaml | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/analysis-scorecard.yaml diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml new file mode 100644 index 00000000..5fce75fb --- /dev/null +++ b/.github/workflows/analysis-scorecard.yaml @@ -0,0 +1,47 @@ +name: OpenSSF Scorecard + +on: + branch_protection_rule: + push: + branches: [main] + schedule: + - cron: "30 0 * * 5" + +permissions: + contents: read + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + permissions: + actions: read + contents: read + id-token: write + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload results as artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: OpenSSF Scorecard results + path: results.sarif + retention-days: 5 + + - name: Upload results to GitHub Security tab + uses: github/codeql-action/upload-sarif@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4 + with: + sarif_file: results.sarif From 3fa25f7ea79f9aea7505ad5fe4a58ac03c6f8a7a Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Sat, 12 Apr 2025 14:11:02 +0200 Subject: [PATCH 09/34] docs: update badges Signed-off-by: Mark Sagi-Kazar --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 86f15455..e043a346 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,10 @@ A FileSystem Abstraction System for Go -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/spf13/afero/ci.yaml?branch=master&style=flat-square)](https://github.com/spf13/afero/actions?query=workflow%3ACI) -[![Join the chat at https://gitter.im/spf13/afero](https://badges.gitter.im/Dev%20Chat.svg)](https://gitter.im/spf13/afero?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![Go Report Card](https://goreportcard.com/badge/github.com/spf13/afero?style=flat-square)](https://goreportcard.com/report/github.com/spf13/afero) -![Go Version](https://img.shields.io/badge/go%20version-%3E=1.23-61CFDD.svg?style=flat-square) -[![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/spf13/afero)](https://pkg.go.dev/mod/github.com/spf13/afero) +[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/spf13/afero/ci.yaml?style=flat-square)](https://github.com/spf13/afero/actions/workflows/ci.yaml) +[![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/mod/github.com/spf13/afero) +![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/spf13/afero?style=flat-square&color=61CFDD) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/spf13/afero/badge?style=flat-square)](https://deps.dev/go/github.com%252Fspf13%252Fafero) # Overview From 638ed6545fb02272f14d203429e56178869fc38c Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Sat, 12 Apr 2025 14:19:51 +0200 Subject: [PATCH 10/34] chore: fix lint violations Signed-off-by: Mark Sagi-Kazar --- .golangci.yaml | 9 ++++--- afero_test.go | 27 ++++++++++++++++--- basepath_test.go | 6 +++-- copyOnWriteFs.go | 9 +++++-- gcsfs/file.go | 5 +++- gcsfs/fs.go | 12 ++++++--- gcsfs/gcs.go | 12 +++++++-- gcsfs/gcs_mocks.go | 20 +++++++++++--- gcsfs/gcs_test.go | 37 +++++++++++++++++++++----- iofs.go | 9 +++++-- iofs_test.go | 39 ++++++++++++++++++++++----- ioutil_test.go | 1 + lstater.go | 2 +- mem/file.go | 22 ++++++++++++--- sftpfs/sftp_test.go | 5 +++- symlink_test.go | 7 ++++- tarfs/tarfs_test.go | 9 ++++++- unionFile.go | 5 ++-- util.go | 4 +-- util_test.go | 65 +++++++++++++++++++++++++++++++++++++-------- 20 files changed, 247 insertions(+), 58 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index fbba3201..4f359b81 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -7,14 +7,15 @@ linters: enable: - govet - ineffassign - # - misspell + - misspell - nolintlint # - revive + - staticcheck - unused disable: - errcheck - - staticcheck + # - staticcheck settings: misspell: @@ -31,9 +32,9 @@ formatters: enable: - gci - gofmt - # - gofumpt + - gofumpt - goimports - # - golines + - golines settings: gci: diff --git a/afero_test.go b/afero_test.go index 827c05c3..a6cb05c4 100644 --- a/afero_test.go +++ b/afero_test.go @@ -60,7 +60,9 @@ func TestRead0(t *testing.T) { for _, fs := range Fss { f := tmpFile(fs) defer f.Close() - f.WriteString("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.") + f.WriteString( + "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", + ) var b []byte // b := make([]byte, 0) @@ -103,7 +105,12 @@ func TestOpenFile(t *testing.T) { contents, _ := io.ReadAll(f) expectedContents := "initial|append" if string(contents) != expectedContents { - t.Errorf("%v: appending, expected '%v', got: '%v'", fs.Name(), expectedContents, string(contents)) + t.Errorf( + "%v: appending, expected '%v', got: '%v'", + fs.Name(), + expectedContents, + string(contents), + ) } f.Close() @@ -158,7 +165,11 @@ func TestCreate(t *testing.T) { continue } if string(buf) != secondContent { - t.Error(fs.Name(), "Content should be", "\""+secondContent+"\" but is \""+string(buf)+"\"") + t.Error( + fs.Name(), + "Content should be", + "\""+secondContent+"\" but is \""+string(buf)+"\"", + ) f.Close() continue } @@ -351,7 +362,15 @@ func TestSeek(t *testing.T) { // http://code.google.com/p/go/issues/detail?id=91 break } - t.Errorf("#%d: Seek(%v, %v) = %v, %v want %v, nil", i, tt.in, tt.whence, off, err, tt.out) + t.Errorf( + "#%d: Seek(%v, %v) = %v, %v want %v, nil", + i, + tt.in, + tt.whence, + off, + err, + tt.out, + ) } } } diff --git a/basepath_test.go b/basepath_test.go index 1c46abd2..1120210c 100644 --- a/basepath_test.go +++ b/basepath_test.go @@ -123,12 +123,14 @@ func TestNestedBasePaths(t *testing.T) { t.Errorf("Got error %s", err.Error()) } - if s.BaseFs == level3Fs { + switch s.BaseFs { + case level3Fs: pathToExist := filepath.Join(ds.Dir3, s.FileName) if _, err := level2Fs.Stat(pathToExist); err != nil { t.Errorf("Got error %s (path %s)", err.Error(), pathToExist) } - } else if s.BaseFs == level2Fs { + + case level2Fs: pathToExist := filepath.Join(ds.Dir2, ds.Dir3, s.FileName) if _, err := level1Fs.Stat(pathToExist); err != nil { t.Errorf("Got error %s (path %s)", err.Error(), pathToExist) diff --git a/copyOnWriteFs.go b/copyOnWriteFs.go index 184d6dd7..aba2879e 100644 --- a/copyOnWriteFs.go +++ b/copyOnWriteFs.go @@ -34,7 +34,8 @@ func (u *CopyOnWriteFs) isBaseFile(name string) (bool, error) { _, err := u.base.Stat(name) if err != nil { if oerr, ok := err.(*os.PathError); ok { - if oerr.Err == os.ErrNotExist || oerr.Err == syscall.ENOENT || oerr.Err == syscall.ENOTDIR { + if oerr.Err == os.ErrNotExist || oerr.Err == syscall.ENOENT || + oerr.Err == syscall.ENOTDIR { return false, nil } } @@ -237,7 +238,11 @@ func (u *CopyOnWriteFs) OpenFile(name string, flag int, perm os.FileMode) (File, return u.layer.OpenFile(name, flag, perm) } - return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOTDIR} // ...or os.ErrNotExist? + return nil, &os.PathError{ + Op: "open", + Path: name, + Err: syscall.ENOTDIR, + } // ...or os.ErrNotExist? } if b { return u.base.OpenFile(name, flag, perm) diff --git a/gcsfs/file.go b/gcsfs/file.go index 54178bc3..81a09377 100644 --- a/gcsfs/file.go +++ b/gcsfs/file.go @@ -108,7 +108,10 @@ func (o *GcsFile) Seek(newOffset int64, whence int) (int64, error) { if (whence == 0 && newOffset == o.fhOffset) || (whence == 1 && newOffset == 0) { return o.fhOffset, nil } - log.Printf("WARNING: Seek behavior triggered, highly inefficent. Offset before seek is at %d\n", o.fhOffset) + log.Printf( + "WARNING: Seek behavior triggered, highly inefficent. Offset before seek is at %d\n", + o.fhOffset, + ) // Fore the reader/writers to be reopened (at correct offset) err := o.Sync() diff --git a/gcsfs/fs.go b/gcsfs/fs.go index 914704f7..6e8660c6 100644 --- a/gcsfs/fs.go +++ b/gcsfs/fs.go @@ -158,7 +158,9 @@ func (fs *Fs) Create(name string) (*GcsFile, error) { } func (fs *Fs) Mkdir(name string, _ os.FileMode) error { - name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name)))) + name = fs.ensureNoLeadingSeparator( + fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name))), + ) if err := validateName(name); err != nil { return err } @@ -181,7 +183,9 @@ func (fs *Fs) Mkdir(name string, _ os.FileMode) error { } func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { - path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path)))) + path = fs.ensureNoLeadingSeparator( + fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path))), + ) if err := validateName(path); err != nil { return err } @@ -404,7 +408,9 @@ func (fs *Fs) Chmod(_ string, _ os.FileMode) error { } func (fs *Fs) Chtimes(_ string, _, _ time.Time) error { - return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly") + return errors.New( + "method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly", + ) } func (fs *Fs) Chown(_ string, _, _ int) error { diff --git a/gcsfs/gcs.go b/gcsfs/gcs.go index f9daa21f..eb09e897 100644 --- a/gcsfs/gcs.go +++ b/gcsfs/gcs.go @@ -48,7 +48,11 @@ func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (afero.Fs, error } // NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator. -func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (afero.Fs, error) { +func NewGcsFSWithSeparator( + ctx context.Context, + folderSeparator string, + opts ...option.ClientOption, +) (afero.Fs, error) { client, err := storage.NewClient(ctx, opts...) if err != nil { return nil, err @@ -65,7 +69,11 @@ func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (afero.Fs, } // NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator. -func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (afero.Fs, error) { +func NewGcsFSFromClientWithSeparator( + ctx context.Context, + client *storage.Client, + folderSeparator string, +) (afero.Fs, error) { c := stiface.AdaptClient(client) return &GcsFs{NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil diff --git a/gcsfs/gcs_mocks.go b/gcsfs/gcs_mocks.go index 9e64970a..adb5f0fe 100644 --- a/gcsfs/gcs_mocks.go +++ b/gcsfs/gcs_mocks.go @@ -73,7 +73,10 @@ func (o *objectMock) NewWriter(_ context.Context) stiface.Writer { return &writerMock{name: o.name, fs: o.fs} } -func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (stiface.Reader, error) { +func (o *objectMock) NewRangeReader( + _ context.Context, + offset, length int64, +) (stiface.Reader, error) { if o.name == "" { return nil, ErrEmptyObjectName } @@ -126,7 +129,11 @@ func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) { return nil, err } - res := &storage.ObjectAttrs{Name: normSeparators(o.name), Size: info.Size(), Updated: info.ModTime()} + res := &storage.ObjectAttrs{ + Name: normSeparators(o.name), + Size: info.Size(), + Updated: info.ModTime(), + } if info.IsDir() { // we have to mock it here, because of FileInfo logic @@ -240,7 +247,14 @@ func (it *objectItMock) Next() (*storage.ObjectAttrs, error) { if err != nil { return nil, err } - it.infos = append(it.infos, &storage.ObjectAttrs{Name: normSeparators(info.Name()), Size: info.Size(), Updated: info.ModTime()}) + it.infos = append( + it.infos, + &storage.ObjectAttrs{ + Name: normSeparators(info.Name()), + Size: info.Size(), + Updated: info.ModTime(), + }, + ) } else { var fInfos []os.FileInfo fInfos, err = it.dir.Readdir(0) diff --git a/gcsfs/gcs_test.go b/gcsfs/gcs_test.go index ba3d09cd..cbc2adf3 100644 --- a/gcsfs/gcs_test.go +++ b/gcsfs/gcs_test.go @@ -55,7 +55,10 @@ var dirs = []struct { name string children []string }{ - {"", []string{"sub", "testDir1", "testFile"}}, // in this case it will be prepended with bucket name + { + "", + []string{"sub", "testDir1", "testFile"}, + }, // in this case it will be prepended with bucket name {"sub", []string{"testDir2"}}, {"sub/testDir2", []string{"testFile"}}, {"testDir1", []string{"testFile"}}, @@ -365,7 +368,14 @@ func TestGcsSeek(t *testing.T) { } if n != s.offOut { - t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offIn, s.whence, n, s.offOut) + t.Errorf( + "%v: (off: %v, whence: %v): got %v, expected %v", + f.name, + s.offIn, + s.whence, + n, + s.offOut, + ) } } } @@ -693,7 +703,10 @@ func TestGcsGlob(t *testing.T) { prefixedEntries := [][]string{{}, {}} for _, entry := range s.entries { prefixedEntries[0] = append(prefixedEntries[0], filepath.Join(bucketName, entry)) - prefixedEntries[1] = append(prefixedEntries[1], string(os.PathSeparator)+filepath.Join(bucketName, entry)) + prefixedEntries[1] = append( + prefixedEntries[1], + string(os.PathSeparator)+filepath.Join(bucketName, entry), + ) } for i, prefixedGlob := range prefixedGlobs { @@ -775,7 +788,11 @@ func TestGcsMkdirAll(t *testing.T) { t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a")) } if info.Mode() != os.ModeDir|0o755 { - t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a"), info.Mode()) + t.Errorf( + "%s: wrong permissions, expected drwxr-xr-x, got %s", + filepath.Join(bucketName, "a"), + info.Mode(), + ) } info, err = gcsAfs.Stat(filepath.Join(bucketName, "a/b")) if err != nil { @@ -785,7 +802,11 @@ func TestGcsMkdirAll(t *testing.T) { t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a/b")) } if info.Mode() != os.ModeDir|0o755 { - t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a/b"), info.Mode()) + t.Errorf( + "%s: wrong permissions, expected drwxr-xr-x, got %s", + filepath.Join(bucketName, "a/b"), + info.Mode(), + ) } info, err = gcsAfs.Stat(dirName) if err != nil { @@ -800,7 +821,11 @@ func TestGcsMkdirAll(t *testing.T) { err = gcsAfs.RemoveAll(filepath.Join(bucketName, "a")) if err != nil { - t.Fatalf("failed to remove the folder %s with error: %s", filepath.Join(bucketName, "a"), err) + t.Fatalf( + "failed to remove the folder %s with error: %s", + filepath.Join(bucketName, "a"), + err, + ) } }) } diff --git a/iofs.go b/iofs.go index b13155ca..57ba5673 100644 --- a/iofs.go +++ b/iofs.go @@ -137,7 +137,7 @@ type readDirFile struct { var _ fs.ReadDirFile = readDirFile{} func (r readDirFile) ReadDir(n int) ([]fs.DirEntry, error) { - items, err := r.File.Readdir(n) + items, err := r.Readdir(n) if err != nil { return nil, err } @@ -161,7 +161,12 @@ var _ Fs = FromIOFS{} func (f FromIOFS) Create(name string) (File, error) { return nil, notImplemented("create", name) } -func (f FromIOFS) Mkdir(name string, perm os.FileMode) error { return notImplemented("mkdir", name) } +func (f FromIOFS) Mkdir( + name string, + perm os.FileMode, +) error { + return notImplemented("mkdir", name) +} func (f FromIOFS) MkdirAll(path string, perm os.FileMode) error { return notImplemented("mkdirall", path) diff --git a/iofs_test.go b/iofs_test.go index 0f126f0d..9af55001 100644 --- a/iofs_test.go +++ b/iofs_test.go @@ -291,7 +291,11 @@ func TestFromIOFS(t *testing.T) { } if lenFile := len(fsys["dir1/dir2/hello.txt"].Data); int64(lenFile) != stat.Size() { - t.Errorf("dir1/dir2/hello.txt stat told invalid size: expected %d, got %d", lenFile, stat.Size()) + t.Errorf( + "dir1/dir2/hello.txt stat told invalid size: expected %d, got %d", + lenFile, + stat.Size(), + ) return } }) @@ -427,21 +431,40 @@ func TestFromIOFS_File(t *testing.T) { } if len(expectedItems) != len(items) { - t.Errorf("Items count mismatch, expected %d, got %d", len(expectedItems), len(items)) + t.Errorf( + "Items count mismatch, expected %d, got %d", + len(expectedItems), + len(items), + ) return } for i, item := range items { if item.Name() != expectedItems[i].Name { - t.Errorf("Item %d: expected name %s, got %s", i, expectedItems[i].Name, item.Name()) + t.Errorf( + "Item %d: expected name %s, got %s", + i, + expectedItems[i].Name, + item.Name(), + ) } if item.IsDir() != expectedItems[i].IsDir { - t.Errorf("Item %d: expected IsDir %t, got %t", i, expectedItems[i].IsDir, item.IsDir()) + t.Errorf( + "Item %d: expected IsDir %t, got %t", + i, + expectedItems[i].IsDir, + item.IsDir(), + ) } if item.Size() != expectedItems[i].Size { - t.Errorf("Item %d: expected IsDir %d, got %d", i, expectedItems[i].Size, item.Size()) + t.Errorf( + "Item %d: expected IsDir %d, got %d", + i, + expectedItems[i].Size, + item.Size(), + ) } } }) @@ -471,7 +494,11 @@ func TestFromIOFS_File(t *testing.T) { expectedItems := []string{"dir1", "dir2", "test.txt"} if len(expectedItems) != len(items) { - t.Errorf("Items count mismatch, expected %d, got %d", len(expectedItems), len(items)) + t.Errorf( + "Items count mismatch, expected %d, got %d", + len(expectedItems), + len(items), + ) return } diff --git a/ioutil_test.go b/ioutil_test.go index 004c66ba..edeaf453 100644 --- a/ioutil_test.go +++ b/ioutil_test.go @@ -152,6 +152,7 @@ func TestTempFile(t *testing.T) { pattern: "foo-*.bar", }, want: func(t *testing.T, base string) { + //nolint: staticcheck if !(strings.HasPrefix(base, "foo-") || strings.HasPrefix(base, "bar")) || len(base) <= len("foo-*.bar") { t.Errorf("TempFile() file = %v, invalid file name", base) diff --git a/lstater.go b/lstater.go index 89c1bfc0..360edf93 100644 --- a/lstater.go +++ b/lstater.go @@ -21,7 +21,7 @@ import ( // filesystems saying so. // It will call Lstat if the filesystem iself is, or it delegates to, the os filesystem. // Else it will call Stat. -// In addtion to the FileInfo, it will return a boolean telling whether Lstat was called or not. +// In addition to the FileInfo, it will return a boolean telling whether Lstat was called or not. type Lstater interface { LstatIfPossible(name string) (os.FileInfo, bool, error) } diff --git a/mem/file.go b/mem/file.go index 62fe4498..c77fcd40 100644 --- a/mem/file.go +++ b/mem/file.go @@ -150,7 +150,11 @@ func (f *File) Sync() error { func (f *File) Readdir(count int) (res []os.FileInfo, err error) { if !f.fileData.dir { - return nil, &os.PathError{Op: "readdir", Path: f.fileData.name, Err: errors.New("not a dir")} + return nil, &os.PathError{ + Op: "readdir", + Path: f.fileData.name, + Err: errors.New("not a dir"), + } } var outLength int64 @@ -236,7 +240,11 @@ func (f *File) Truncate(size int64) error { return ErrFileClosed } if f.readOnly { - return &os.PathError{Op: "truncate", Path: f.fileData.name, Err: errors.New("file handle is read only")} + return &os.PathError{ + Op: "truncate", + Path: f.fileData.name, + Err: errors.New("file handle is read only"), + } } if size < 0 { return ErrOutOfRange @@ -273,7 +281,11 @@ func (f *File) Write(b []byte) (n int, err error) { return 0, ErrFileClosed } if f.readOnly { - return 0, &os.PathError{Op: "write", Path: f.fileData.name, Err: errors.New("file handle is read only")} + return 0, &os.PathError{ + Op: "write", + Path: f.fileData.name, + Err: errors.New("file handle is read only"), + } } n = len(b) cur := atomic.LoadInt64(&f.at) @@ -285,7 +297,9 @@ func (f *File) Write(b []byte) (n int, err error) { tail = f.fileData.data[n+int(cur):] } if diff > 0 { - f.fileData.data = append(f.fileData.data, append(bytes.Repeat([]byte{0o0}, int(diff)), b...)...) + f.fileData.data = append( + f.fileData.data, + append(bytes.Repeat([]byte{0o0}, int(diff)), b...)...) f.fileData.data = append(f.fileData.data, tail...) } else { f.fileData.data = append(f.fileData.data[:cur], b...) diff --git a/sftpfs/sftp_test.go b/sftpfs/sftp_test.go index 505807bb..620d0356 100644 --- a/sftpfs/sftp_test.go +++ b/sftpfs/sftp_test.go @@ -223,7 +223,10 @@ func MakeSSHKeyPair(bits int, pubKeyPath, privateKeyPath string) error { return err } - privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)} + privateKeyPEM := &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + } if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil { return err } diff --git a/symlink_test.go b/symlink_test.go index 77dd742f..aa5b63b0 100644 --- a/symlink_test.go +++ b/symlink_test.go @@ -79,7 +79,12 @@ func TestSymlinkIfPossible(t *testing.T) { testLink(osFs, osPath, filepath.Join(workDir, "os/link.txt"), nil) testLink(overlayFs1, osPath, filepath.Join(workDir, "overlay/link1.txt"), ¬Supported) testLink(overlayFs2, pathFileMem, filepath.Join(workDir, "overlay2/link2.txt"), nil) - testLink(overlayFsMemOnly, pathFileMem, filepath.Join(memWorkDir, "overlay3/link.txt"), ¬Supported) + testLink( + overlayFsMemOnly, + pathFileMem, + filepath.Join(memWorkDir, "overlay3/link.txt"), + ¬Supported, + ) testLink(basePathFs, "afero.txt", "basepath/link.txt", nil) testLink(basePathFsMem, pathFileMem, "link/file.txt", ¬Supported) testLink(roFs, osPath, filepath.Join(workDir, "ro/link.txt"), ¬Supported) diff --git a/tarfs/tarfs_test.go b/tarfs/tarfs_test.go index 228ed9b4..6310a116 100644 --- a/tarfs/tarfs_test.go +++ b/tarfs/tarfs_test.go @@ -185,7 +185,14 @@ func TestSeek(t *testing.T) { } if n != s.offout { - t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offin, s.whence, n, s.offout) + t.Errorf( + "%v: (off: %v, whence: %v): got %v, expected %v", + f.name, + s.offin, + s.whence, + n, + s.offout, + ) } } diff --git a/unionFile.go b/unionFile.go index 62dd6c93..2e2253f5 100644 --- a/unionFile.go +++ b/unionFile.go @@ -92,7 +92,8 @@ func (f *UnionFile) Seek(o int64, w int) (pos int64, err error) { func (f *UnionFile) Write(s []byte) (n int, err error) { if f.Layer != nil { n, err = f.Layer.Write(s) - if err == nil && f.Base != nil { // hmm, do we have fixed size files where a write may hit the EOF mark? + if err == nil && + f.Base != nil { // hmm, do we have fixed size files where a write may hit the EOF mark? _, err = f.Base.Write(s) } return n, err @@ -157,7 +158,7 @@ var defaultUnionMergeDirsFn = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, err // return a single view of the overlayed directories. // At the end of the directory view, the error is io.EOF if c > 0. func (f *UnionFile) Readdir(c int) (ofi []os.FileInfo, err error) { - var merge DirsMerger = f.Merger + merge := f.Merger if merge == nil { merge = defaultUnionMergeDirsFn } diff --git a/util.go b/util.go index 9e4cba27..23176883 100644 --- a/util.go +++ b/util.go @@ -113,11 +113,11 @@ func GetTempDir(fs Fs, subPath string) string { if subPath != "" { // preserve windows backslash :-( if FilePathSeparator == "\\" { - subPath = strings.Replace(subPath, "\\", "____", -1) + subPath = strings.ReplaceAll(subPath, "\\", "____") } dir = dir + UnicodeSanitize((subPath)) if FilePathSeparator == "\\" { - dir = strings.Replace(dir, "____", "\\", -1) + dir = strings.ReplaceAll(dir, "____", "\\") } if exists, _ := Exists(fs, dir); exists { diff --git a/util_test.go b/util_test.go index 13ec23b1..a8e7212f 100644 --- a/util_test.go +++ b/util_test.go @@ -102,7 +102,7 @@ func TestIsEmpty(t *testing.T) { nonEmptyNonZeroLengthFilesDirectory, _ := createTempDirWithNonZeroLengthFiles() defer deleteTempDir(nonEmptyNonZeroLengthFilesDirectory) nonExistentFile := os.TempDir() + "/this-file-does-not-exist.txt" - nonExistentDir := os.TempDir() + "/this/direcotry/does/not/exist/" + nonExistentDir := os.TempDir() + "/this/directory/does/not/exist/" fileDoesNotExist := fmt.Errorf("%q path does not exist", nonExistentFile) dirDoesNotExist := fmt.Errorf("%q path does not exist", nonExistentDir) @@ -125,11 +125,24 @@ func TestIsEmpty(t *testing.T) { for i, d := range data { exists, err := IsEmpty(testFS, d.input) if d.expectedResult != exists { - t.Errorf("Test %d %q failed exists. Expected result %t got %t", i, d.input, d.expectedResult, exists) + t.Errorf( + "Test %d %q failed exists. Expected result %t got %t", + i, + d.input, + d.expectedResult, + exists, + ) } if d.expectedErr != nil { if d.expectedErr.Error() != err.Error() { - t.Errorf("Test %d failed with err. Expected %q(%#v) got %q(%#v)", i, d.expectedErr, d.expectedErr, err, err) + t.Errorf( + "Test %d failed with err. Expected %q(%#v) got %q(%#v)", + i, + d.expectedErr, + d.expectedErr, + err, + err, + ) } } else { if d.expectedErr != err { @@ -267,7 +280,7 @@ func TestExists(t *testing.T) { emptyDirectory, _ := createEmptyTempDir() defer deleteTempDir(emptyDirectory) nonExistentFile := os.TempDir() + "/this-file-does-not-exist.txt" - nonExistentDir := os.TempDir() + "/this/direcotry/does/not/exist/" + nonExistentDir := os.TempDir() + "/this/directory/does/not/exist/" type test struct { input string @@ -320,7 +333,12 @@ func TestSafeWriteToDisk(t *testing.T) { e := SafeWriteReader(testFS, d.filename, reader) if d.expectedErr != nil { if d.expectedErr.Error() != e.Error() { - t.Errorf("Test %d failed. Expected error %q but got %q", i, d.expectedErr.Error(), e.Error()) + t.Errorf( + "Test %d failed. Expected error %q but got %q", + i, + d.expectedErr.Error(), + e.Error(), + ) } } else { if d.expectedErr != e { @@ -359,14 +377,24 @@ func TestWriteToDisk(t *testing.T) { for i, d := range data { e := WriteReader(testFS, d.filename, reader) if d.expectedErr != e { - t.Errorf("Test %d failed. WriteToDisk Error Expected %q but got %q", i, d.expectedErr, e) + t.Errorf( + "Test %d failed. WriteToDisk Error Expected %q but got %q", + i, + d.expectedErr, + e, + ) } contents, e := ReadFile(testFS, d.filename) if e != nil { t.Errorf("Test %d failed. Could not read file %s. Reason: %s\n", i, d.filename, e) } if randomString != string(contents) { - t.Errorf("Test %d failed. Expected contents %q but got %q", i, randomString, string(contents)) + t.Errorf( + "Test %d failed. Expected contents %q but got %q", + i, + randomString, + string(contents), + ) } reader.Seek(0, 0) } @@ -384,7 +412,10 @@ func TestGetTempDir(t *testing.T) { }{ {"", dir}, {testDir + " Foo bar ", dir + testDir + " Foo bar " + FilePathSeparator}, - {testDir + "Foo.Bar/foo_Bar-Foo", dir + testDir + "Foo.Bar/foo_Bar-Foo" + FilePathSeparator}, + { + testDir + "Foo.Bar/foo_Bar-Foo", + dir + testDir + "Foo.Bar/foo_Bar-Foo" + FilePathSeparator, + }, {testDir + "fOO,bar:foo%bAR", dir + testDir + "fOObarfoo%bAR" + FilePathSeparator}, {testDir + "FOo/BaR.html", dir + testDir + "FOo/BaR.html" + FilePathSeparator}, {testDir + "трям/трям", dir + testDir + "трям/трям" + FilePathSeparator}, @@ -431,9 +462,21 @@ func TestFullBaseFsPath(t *testing.T) { ExpectedPath string } specs := []spec{ - {BaseFs: level3Fs, FileName: "f.txt", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, ds.Dir3, "f.txt")}, - {BaseFs: level3Fs, FileName: "", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, ds.Dir3, "")}, - {BaseFs: level2Fs, FileName: "f.txt", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, "f.txt")}, + { + BaseFs: level3Fs, + FileName: "f.txt", + ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, ds.Dir3, "f.txt"), + }, + { + BaseFs: level3Fs, + FileName: "", + ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, ds.Dir3, ""), + }, + { + BaseFs: level2Fs, + FileName: "f.txt", + ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, "f.txt"), + }, {BaseFs: level2Fs, FileName: "", ExpectedPath: filepath.Join(ds.Dir1, ds.Dir2, "")}, {BaseFs: level1Fs, FileName: "f.txt", ExpectedPath: filepath.Join(ds.Dir1, "f.txt")}, {BaseFs: level1Fs, FileName: "", ExpectedPath: filepath.Join(ds.Dir1, "")}, From 90b8ab916e1ef9ec14be32c80010445b4b1eac26 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Sat, 12 Apr 2025 14:21:09 +0200 Subject: [PATCH 11/34] ci: fix scorecard branch Signed-off-by: Mark Sagi-Kazar --- .github/workflows/analysis-scorecard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml index 5fce75fb..6f686cc0 100644 --- a/.github/workflows/analysis-scorecard.yaml +++ b/.github/workflows/analysis-scorecard.yaml @@ -3,7 +3,7 @@ name: OpenSSF Scorecard on: branch_protection_rule: push: - branches: [main] + branches: [master] schedule: - cron: "30 0 * * 5" From f5e0ff55d403e817fcd5fa215c6a32b6f97e4a0c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Apr 2025 15:40:13 +0000 Subject: [PATCH 12/34] Bump actions/setup-go from 5.3.0 to 5.4.0 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5.3.0 to 5.4.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/f111f3307d8850f501ac008e886eec1fd1932a34...0aaccfd150d50ccaeb58ebd88d36e91967a5f35b) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a9e7123b..c23e5fc2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: go-version: ${{ matrix.go }} @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: go-version: "1.24" From 100a14e0c515e513c7b4faf4a97a6787c3199c86 Mon Sep 17 00:00:00 2001 From: messikiller Date: Mon, 14 Apr 2025 16:26:58 +0800 Subject: [PATCH 13/34] replace ossfs as third party link --- README.md | 4 + go.mod | 14 +- go.sum | 16 - ossfs/file.go | 288 ---------------- ossfs/file_info.go | 17 - ossfs/file_test.go | 296 ----------------- ossfs/fs.go | 188 ----------- ossfs/fs_test.go | 462 -------------------------- ossfs/fs_utils.go | 36 -- ossfs/init.go | 14 - ossfs/internal/mocks/FileInfo.go | 139 -------- ossfs/internal/mocks/ObjectManager.go | 293 ---------------- ossfs/internal/utils/contract.go | 21 -- ossfs/internal/utils/init.go | 6 - ossfs/internal/utils/oss.go | 210 ------------ 15 files changed, 5 insertions(+), 1999 deletions(-) delete mode 100644 ossfs/file.go delete mode 100644 ossfs/file_info.go delete mode 100644 ossfs/file_test.go delete mode 100644 ossfs/fs.go delete mode 100644 ossfs/fs_test.go delete mode 100644 ossfs/fs_utils.go delete mode 100644 ossfs/init.go delete mode 100644 ossfs/internal/mocks/FileInfo.go delete mode 100644 ossfs/internal/mocks/ObjectManager.go delete mode 100644 ossfs/internal/utils/contract.go delete mode 100644 ossfs/internal/utils/init.go delete mode 100644 ossfs/internal/utils/oss.go diff --git a/README.md b/README.md index 86f15455..b67c2059 100644 --- a/README.md +++ b/README.md @@ -399,6 +399,10 @@ implement: * SSH * S3 +## Third-party library + +- Alibaba Cloud OSS: [messikiller/afero-oss](https://github.com/messikiller/afero-oss) + # About the project ## What's in the name diff --git a/go.mod b/go.mod index fa24da40..101c2865 100644 --- a/go.mod +++ b/go.mod @@ -2,16 +2,4 @@ module github.com/spf13/afero go 1.23.0 -require ( - github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1 - github.com/stretchr/testify v1.10.0 - golang.org/x/text v0.23.0 -) - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/objx v0.5.2 // indirect - golang.org/x/time v0.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) +require golang.org/x/text v0.23.0 diff --git a/go.sum b/go.sum index 1862dd3d..d00bb390 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,2 @@ -github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1 h1:sOhpJdR/+lbQniznp3cYSfwQlXbVkT0ccuiZScBrI6Y= -github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= -golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ossfs/file.go b/ossfs/file.go deleted file mode 100644 index 630beff8..00000000 --- a/ossfs/file.go +++ /dev/null @@ -1,288 +0,0 @@ -package ossfs - -import ( - "errors" - "io" - "os" - "strconv" - "syscall" - - "github.com/spf13/afero" -) - -type File struct { - name string - fs *Fs - openFlag int - offset int64 - fi os.FileInfo - dirty bool - closed bool - isDir bool - preloaded bool - preloadedFd afero.File -} - -func NewOssFile(name string, flag int, fs *Fs) (*File, error) { - return &File{ - name: fs.normFileName(name), - fs: fs, - openFlag: flag, - offset: 0, - dirty: false, - closed: false, - isDir: fs.isDir(fs.normFileName(name)), - preloaded: false, - preloadedFd: nil, - }, nil -} - -func (f *File) preload() error { - pfs := f.fs.preloadFs - if _, err := pfs.Stat(f.name); err == nil { - if e := pfs.Remove(f.name); e != nil { - return e - } - } - pfd, err := f.fs.preloadFs.Create(f.name) - if err != nil { - return err - } - - r, clean, e := f.fs.manager.GetObject(f.fs.ctx, f.fs.bucketName, f.name) - if e != nil { - return e - } - defer clean() - - if _, err := io.Copy(pfd, r); err != nil { - return err - } - - if _, err := pfd.Seek(f.offset, io.SeekStart); err != nil { - return err - } - - f.preloadedFd = pfd - f.preloaded = true - return nil -} - -func (f *File) freshFileInfo() error { - fi, err := f.fs.Stat(f.name) - if err != nil { - return err - } - f.fi = fi - f.dirty = false - return nil -} - -func (f *File) getFileInfo() (os.FileInfo, error) { - if f.dirty { - if err := f.freshFileInfo(); err != nil { - return nil, err - } - } - return f.fi, nil -} - -func (f *File) isReadable() bool { - return !f.closed && (f.openFlag == os.O_RDONLY || f.openFlag == os.O_RDWR) -} - -func (f *File) isWriteable() bool { - return !f.closed && (f.openFlag == os.O_WRONLY || f.openFlag == os.O_RDWR) -} - -func (f *File) isAppendOnly() bool { - return f.isWriteable() && f.openFlag&os.O_APPEND != 0 -} - -func (f *File) Read(p []byte) (int, error) { - if !f.isReadable() || f.isDir { - return 0, syscall.EPERM - } - n, err := f.ReadAt(p, f.offset) - if err != nil { - return 0, err - } - f.offset += int64(n) - return n, err -} - -func (f *File) ReadAt(p []byte, off int64) (int, error) { - if !f.isReadable() || f.isDir { - return 0, syscall.EPERM - } - reader, cleanUp, err := f.fs.manager.GetObjectPart(f.fs.ctx, f.fs.bucketName, f.name, off, off+int64(len(p))) - if err != nil { - return 0, err - } - defer cleanUp() - return reader.Read(p) -} - -func (f *File) Seek(offset int64, whence int) (int64, error) { - if (!f.isReadable() && !f.isWriteable()) || f.isDir { - return 0, syscall.EPERM - } - fi, err := f.getFileInfo() - if err != nil { - return 0, err - } - max := fi.Size() - var newOffset int64 - switch whence { - case io.SeekCurrent: - newOffset = f.offset + offset - case io.SeekStart: - newOffset = offset - case io.SeekEnd: - newOffset = max + offset - default: - return 0, errors.New("Invalid whence value: " + strconv.Itoa(whence)) - } - if newOffset < 0 || newOffset > max { - return 0, afero.ErrOutOfRange - } - f.offset = newOffset - return f.offset, nil -} - -func (f *File) doAppend(p []byte) (int, error) { - if !f.isWriteable() { - return 0, syscall.EPERM - } - fi, err := f.getFileInfo() - if err != nil { - return 0, err - } - return f.doWriteAt(p, fi.Size()) -} - -func (f *File) Write(p []byte) (int, error) { - if !f.isWriteable() { - return 0, syscall.EPERM - } - if f.isAppendOnly() { - return f.doAppend(p) - } - n, e := f.doWriteAt(p, f.offset) - if e != nil { - return 0, e - } - f.offset += int64(n) - return n, e -} - -func (f *File) doWriteAt(p []byte, off int64) (int, error) { - if f.isDir { - return 0, syscall.EPERM - } - - if !f.preloaded { - if err := f.preload(); err != nil { - return 0, err - } - } - - n, e := f.preloadedFd.WriteAt(p, off) - f.dirty = true - if f.fs.autoSync { - f.Sync() - } - return n, e -} - -func (f *File) WriteAt(p []byte, off int64) (int, error) { - if !f.isWriteable() || f.isAppendOnly() { - return 0, syscall.EPERM - } - return f.doWriteAt(p, off) -} - -func (f *File) Close() error { - f.Sync() - f.closed = true - delete(f.fs.openedFiles, f.name) - if f.preloaded { - err := f.fs.preloadFs.Remove(f.name) - if err != nil { - return err - } - err = f.preloadedFd.Close() - if err != nil { - return err - } - f.preloadedFd = nil - f.preloaded = false - } - return nil -} - -func (f *File) Name() string { - return f.name -} - -func (f *File) Readdir(count int) ([]os.FileInfo, error) { - if !f.isReadable() { - return nil, syscall.EPERM - } - - fis, err := f.fs.manager.ListObjects(f.fs.ctx, f.fs.bucketName, f.fs.ensureAsDir(f.name), count) - return fis, err -} - -func (f *File) Readdirnames(n int) ([]string, error) { - if !f.isReadable() { - return nil, syscall.EPERM - } - - fis, err := f.Readdir(n) - if err != nil { - return nil, err - } - var fNames []string - for _, fi := range fis { - fNames = append(fNames, fi.Name()) - } - - return fNames, nil -} - -func (f *File) Stat() (os.FileInfo, error) { - if f.dirty { - err := f.freshFileInfo() - if err != nil { - return nil, err - } - } - return f.fi, nil -} - -func (f *File) Sync() error { - if f.preloaded { - if _, err := f.fs.manager.PutObject(f.fs.ctx, f.fs.bucketName, f.name, f.preloadedFd); err != nil { - return err - } - } - if f.dirty { - if err := f.freshFileInfo(); err != nil { - return err - } - } - return nil -} - -func (f *File) Truncate(size int64) error { - if !f.isWriteable() || f.isDir { - return syscall.EPERM - } - _, err := f.WriteAt([]byte(""), 0) - return err -} - -func (f *File) WriteString(s string) (int, error) { - return f.Write([]byte(s)) -} diff --git a/ossfs/file_info.go b/ossfs/file_info.go deleted file mode 100644 index ea31d2da..00000000 --- a/ossfs/file_info.go +++ /dev/null @@ -1,17 +0,0 @@ -package ossfs - -import ( - "time" - - "github.com/spf13/afero/ossfs/internal/utils" -) - -type FileInfo struct { - *utils.OssObjectMeta -} - -func NewFileInfo(name string, size int64, updatedAt time.Time) *FileInfo { - return &FileInfo{ - OssObjectMeta: utils.NewOssObjectMeta(name, size, updatedAt), - } -} diff --git a/ossfs/file_test.go b/ossfs/file_test.go deleted file mode 100644 index fba21bd2..00000000 --- a/ossfs/file_test.go +++ /dev/null @@ -1,296 +0,0 @@ -package ossfs - -import ( - "io" - "os" - "strings" - "syscall" - "testing" - "time" - - "github.com/spf13/afero" - "github.com/spf13/afero/ossfs/internal/mocks" - "github.com/spf13/afero/ossfs/internal/utils" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func getMockedFs() *Fs { - fs := NewOssFs("test-ak", "test-sk", "test-region", "test-bucket") - fs.manager = &mocks.ObjectManager{} - return fs -} - -func getMockedFile(name string, flag int, fs *Fs) *File { - f, _ := NewOssFile(name, flag, fs) - return f -} - -func TestNewOssFile(t *testing.T) { - t.Run("create new file with read flag", func(t *testing.T) { - fs := &Fs{} - file, err := NewOssFile("testfile", os.O_RDONLY, fs) - assert.NoError(t, err) - assert.Equal(t, "testfile", file.name) - assert.Equal(t, os.O_RDONLY, file.openFlag) - assert.Equal(t, fs, file.fs) - assert.False(t, file.dirty) - assert.False(t, file.closed) - assert.False(t, file.isDir) - assert.False(t, file.preloaded) - assert.Nil(t, file.preloadedFd) - }) - - t.Run("create new file with write flag", func(t *testing.T) { - fs := &Fs{} - file, err := NewOssFile("testfile", os.O_WRONLY, fs) - assert.NoError(t, err) - assert.Equal(t, "testfile", file.name) - assert.Equal(t, os.O_WRONLY, file.openFlag) - assert.Equal(t, fs, file.fs) - assert.False(t, file.dirty) - assert.False(t, file.closed) - assert.False(t, file.isDir) - assert.False(t, file.preloaded) - assert.Nil(t, file.preloadedFd) - }) - - t.Run("create new directory", func(t *testing.T) { - fs := &Fs{} - file, err := NewOssFile("testdir/", os.O_RDONLY, fs) - assert.NoError(t, err) - assert.Equal(t, "testdir/", file.name) - assert.Equal(t, os.O_RDONLY, file.openFlag) - assert.Equal(t, fs, file.fs) - assert.False(t, file.dirty) - assert.False(t, file.closed) - assert.True(t, file.isDir) - assert.False(t, file.preloaded) - assert.Nil(t, file.preloadedFd) - }) - - t.Run("normalize file name", func(t *testing.T) { - fs := &Fs{} - file, err := NewOssFile("/path/testfile", os.O_RDONLY, fs) - assert.NoError(t, err) - assert.Equal(t, "path/testfile", file.name) - }) -} - -func TestRead(t *testing.T) { - t.Run("Read with unreadable flag return error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_WRONLY, fs) - - p := make([]byte, 0) - _, e := f.Read(p) - - assert.Error(t, e) - assert.NotNil(t, e) - }) - - t.Run("Read on directory return error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testdir", os.O_RDONLY, fs) - f.isDir = true - - p := make([]byte, 10) - _, e := f.Read(p) - - assert.Error(t, e) - assert.Equal(t, syscall.EPERM, e) - }) - - t.Run("Read on closed file return error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDONLY, fs) - f.closed = true - - p := make([]byte, 10) - _, e := f.Read(p) - - assert.Error(t, e) - assert.Equal(t, syscall.EPERM, e) - }) - - t.Run("Successful read updates offset", func(t *testing.T) { - fs := getMockedFs() - var cu utils.CleanUp = func() {} - mockManager := fs.manager.(*mocks.ObjectManager) - mockManager.On("GetObjectPart", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - // Return(&mockReadCloser{data: []byte("testdata")}, cu, nil) - Return(strings.NewReader("testdata"), cu, nil) - - f := getMockedFile("testfile", os.O_RDONLY, fs) - p := make([]byte, 8) - n, err := f.Read(p) - - assert.NoError(t, err) - assert.Equal(t, 8, n) - assert.Equal(t, int64(8), f.offset) - }) - - t.Run("ReadAt error propagates", func(t *testing.T) { - fs := getMockedFs() - mockManager := fs.manager.(*mocks.ObjectManager) - mockManager.On("GetObjectPart", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(nil, nil, syscall.EIO) - - f := getMockedFile("testfile", os.O_RDONLY, fs) - p := make([]byte, 8) - _, err := f.Read(p) - - assert.Error(t, err) - assert.Equal(t, syscall.EIO, err) - }) -} - -func TestReadAt(t *testing.T) { - t.Run("ReadAt with unreadable flag return error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_WRONLY, fs) - - p := make([]byte, 0) - _, e := f.ReadAt(p, 0) - - assert.Error(t, e) - assert.NotNil(t, e) - }) - - t.Run("ReadAt on dir return error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("/path/to/dir/", os.O_WRONLY, fs) - - p := make([]byte, 0) - _, e := f.ReadAt(p, 0) - - assert.Error(t, e) - assert.NotNil(t, e) - }) - - t.Run("ReadAt success", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDONLY, fs) - - p := make([]byte, 4) - - var cu utils.CleanUp = func() {} - off := int64(5) - m := &mocks.ObjectManager{} - m. - On("GetObjectPart", f.fs.ctx, f.fs.bucketName, f.name, off, off+int64(len(p))). - Return(strings.NewReader("test result"), cu, nil) - fs.manager = m - - n, e := f.ReadAt(p, off) - - assert.Nil(t, e) - assert.Equal(t, 4, n) - assert.Equal(t, "test", string(p)) - }) -} - -func TestSeek(t *testing.T) { - t.Run("Seek on unreadable/unwritable file returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_WRONLY, fs) - f.closed = true - - _, err := f.Seek(0, io.SeekStart) - assert.Error(t, err) - assert.Equal(t, syscall.EPERM, err) - }) - - t.Run("Seek on directory returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testdir", os.O_RDONLY, fs) - f.isDir = true - - _, err := f.Seek(0, io.SeekStart) - assert.Error(t, err) - assert.Equal(t, syscall.EPERM, err) - }) - - t.Run("SeekStart sets correct offset", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - - offset, err := f.Seek(10, io.SeekStart) - assert.NoError(t, err) - assert.Equal(t, int64(10), offset) - assert.Equal(t, int64(10), f.offset) - }) - - t.Run("SeekCurrent adjusts offset correctly", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - f.offset = 5 - - offset, err := f.Seek(5, io.SeekCurrent) - assert.NoError(t, err) - assert.Equal(t, int64(10), offset) - assert.Equal(t, int64(10), f.offset) - }) - - t.Run("SeekEnd adjusts offset correctly", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - - offset, err := f.Seek(-10, io.SeekEnd) - assert.NoError(t, err) - assert.Equal(t, int64(90), offset) - assert.Equal(t, int64(90), f.offset) - }) - - t.Run("Seek beyond file size returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - - _, err := f.Seek(101, io.SeekStart) - assert.Error(t, err) - assert.Equal(t, afero.ErrOutOfRange, err) - }) - - t.Run("Seek negative offset returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - - _, err := f.Seek(-1, io.SeekStart) - assert.Error(t, err) - assert.Equal(t, afero.ErrOutOfRange, err) - }) - - t.Run("Seek with invalid whence returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - - _, err := f.Seek(0, 3) - assert.Error(t, err) - }) -} - -func TestWrite(t *testing.T) { - t.Run("Write unwritable file returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDONLY, fs) - _, e := f.Write([]byte("test input string")) - - assert.Error(t, e) - assert.NotNil(t, e) - }) - - t.Run("Write dir returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("/path/to/test_dir/", os.O_WRONLY, fs) - _, e := f.Write([]byte("test input string")) - - assert.Error(t, e) - assert.NotNil(t, e) - }) -} diff --git a/ossfs/fs.go b/ossfs/fs.go deleted file mode 100644 index 05ef2ffd..00000000 --- a/ossfs/fs.go +++ /dev/null @@ -1,188 +0,0 @@ -package ossfs - -import ( - "context" - "errors" - "os" - "strings" - "time" - - "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" - "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials" - "github.com/spf13/afero" - "github.com/spf13/afero/ossfs/internal/utils" -) - -const ( - defaultFileMode = 0o755 - defaultFileFlag = os.O_RDWR -) - -type Fs struct { - manager utils.ObjectManager - bucketName string - separator string - autoSync bool - openedFiles map[string]afero.File - preloadFs afero.Fs - ctx context.Context -} - -func NewOssFs(accessKeyId, accessKeySecret, region, bucket string) *Fs { - ossCfg := oss.LoadDefaultConfig(). - WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyId, accessKeySecret)). - WithRegion(region) - - return &Fs{ - manager: &utils.OssObjectManager{ - Client: oss.NewClient(ossCfg), - }, - bucketName: bucket, - separator: "/", - autoSync: true, - openedFiles: make(map[string]afero.File), - preloadFs: afero.NewMemMapFs(), - ctx: context.Background(), - } -} - -func (fs *Fs) WithContext(ctx context.Context) *Fs { - fs.ctx = ctx - return fs -} - -// Create creates a new empty file and open it, return the open file and error -// if any happens. -func (fs *Fs) Create(name string) (afero.File, error) { - n := fs.normFileName(name) - r := strings.NewReader("") - if _, err := fs.manager.PutObject(fs.ctx, fs.bucketName, n, r); err != nil { - return nil, err - } - return NewOssFile(n, defaultFileFlag, fs) -} - -// Mkdir creates a directory in the filesystem, return an error if any -// happens. -func (fs *Fs) Mkdir(name string, perm os.FileMode) error { - return fs.MkdirAll(fs.ensureAsDir(name), perm) -} - -// MkdirAll creates a directory path and all parents that does not exist -// yet. -func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { - dirName := fs.ensureAsDir(path) - r := strings.NewReader("") - _, err := fs.manager.PutObject(fs.ctx, fs.bucketName, dirName, r) - return err -} - -// Open opens a file, returning it or an error, if any happens. -func (fs *Fs) Open(name string) (afero.File, error) { - return fs.OpenFile(name, defaultFileFlag, defaultFileMode) -} - -// OpenFile opens a file using the given flags and the given mode. -func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { - name = fs.normFileName(name) - file, found := fs.openedFiles[name] - if found && file.(*File).openFlag == flag { - return file, nil - } - - f, err := NewOssFile(name, flag, fs) - if err != nil { - return nil, err - } - - existed := false - existed, err = fs.manager.IsObjectExist(fs.ctx, fs.bucketName, name) - if err != nil { - return nil, err - } - - if !existed && f.openFlag&os.O_CREATE == 0 { - return nil, afero.ErrFileNotFound - } - - if !existed && f.openFlag*os.O_CREATE != 0 { - if _, err := fs.Create(f.name); err != nil { - return nil, err - } - } - - if f.openFlag&os.O_TRUNC != 0 { - _, err := f.fs.manager.PutObject(fs.ctx, fs.bucketName, f.name, strings.NewReader("")) - if err != nil { - return nil, err - } - } - - fs.openedFiles[name] = f - - return f, nil -} - -// Remove removes a file identified by name, returning an error, if any -// happens. -func (fs *Fs) Remove(name string) error { - return fs.manager.DeleteObject(fs.ctx, fs.bucketName, fs.normFileName(name)) -} - -// RemoveAll removes a directory path and any children it contains. It -// does not fail if the path does not exist (return nil). -func (fs *Fs) RemoveAll(path string) error { - dir := fs.ensureAsDir(path) - fis, err := fs.manager.ListAllObjects(fs.ctx, fs.bucketName, dir) - if err != nil { - return err - } - for _, fi := range fis { - err = fs.manager.DeleteObject(fs.ctx, fs.bucketName, fi.Name()) - if err != nil { - return err - } - } - return nil -} - -// Rename renames a file. -func (fs *Fs) Rename(oldname, newname string) error { - err := fs.manager.CopyObject(fs.ctx, fs.bucketName, oldname, newname) - if err != nil { - return err - } - err = fs.manager.DeleteObject(fs.ctx, fs.bucketName, oldname) - return err -} - -// Stat returns a FileInfo describing the named file, or an error, if any -// happens. -func (fs *Fs) Stat(name string) (os.FileInfo, error) { - fi, err := fs.manager.GetObjectMeta(fs.ctx, fs.bucketName, fs.normFileName(name)) - if err != nil { - return nil, err - } - - return fi, err -} - -// The name of this FileSystem -func (fs *Fs) Name() string { - return "OssFs" -} - -// Chmod changes the mode of the named file to mode. -func (fs *Fs) Chmod(name string, mode os.FileMode) error { - return errors.New("OSS: method Chmod is not implemented") -} - -// Chown changes the uid and gid of the named file. -func (fs *Fs) Chown(name string, uid, gid int) error { - return errors.New("OSS: method Chown is not implemented") -} - -// Chtimes changes the access and modification times of the named file -func (fs *Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { - return errors.New("OSS: method Chtimes is not implemented") -} diff --git a/ossfs/fs_test.go b/ossfs/fs_test.go deleted file mode 100644 index 8d09ecf8..00000000 --- a/ossfs/fs_test.go +++ /dev/null @@ -1,462 +0,0 @@ -package ossfs - -import ( - "context" - "os" - "strings" - "testing" - "time" - - "github.com/spf13/afero" - "github.com/spf13/afero/ossfs/internal/mocks" - "github.com/stretchr/testify/assert" -) - -func TestNewOssFs(t *testing.T) { - tests := []struct { - name string - accessKeyId string - accessKeySecret string - region string - bucket string - expected *Fs - }{ - { - name: "valid credentials", - accessKeyId: "testKeyId", - accessKeySecret: "testKeySecret", - region: "test-region", - bucket: "test-bucket", - expected: &Fs{ - bucketName: "test-bucket", - separator: "/", - autoSync: true, - openedFiles: make(map[string]afero.File), - preloadFs: afero.NewMemMapFs(), - ctx: context.Background(), - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := NewOssFs(tt.accessKeyId, tt.accessKeySecret, tt.region, tt.bucket) - assert.NotNil(t, got.manager) - assert.Equal(t, tt.expected.bucketName, got.bucketName) - assert.Equal(t, tt.expected.separator, got.separator) - assert.Equal(t, tt.expected.autoSync, got.autoSync) - assert.NotNil(t, got.openedFiles) - assert.NotNil(t, got.preloadFs) - assert.NotNil(t, got.ctx) - }) - } -} - -func TestFsWithContext(t *testing.T) { - type bgMeta string - tests := []struct { - name string - fs *Fs - ctx context.Context - expected *Fs - }{ - { - name: "set new context", - fs: &Fs{ - ctx: context.Background(), - }, - ctx: context.WithValue(context.Background(), bgMeta("testKey"), bgMeta("testValue")), - expected: &Fs{ - ctx: context.WithValue(context.Background(), bgMeta("testKey"), bgMeta("testValue")), - }, - }, - { - name: "set nil context", - fs: &Fs{ - ctx: context.Background(), - }, - ctx: nil, - expected: &Fs{ - ctx: nil, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := tt.fs.WithContext(tt.ctx) - assert.Equal(t, tt.expected.ctx, got.ctx) - assert.Equal(t, tt.fs, got) - }) - } -} - -func TestFsCreate(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("create simple success", func(t *testing.T) { - m.On("PutObject", fs.ctx, bucket, "test.txt", strings.NewReader("")).Return(true, nil).Once() - file, err := fs.Create("test.txt") - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - m.AssertExpectations(t) - }) - - t.Run("create prefixed file path success", func(t *testing.T) { - m. - On("PutObject", fs.ctx, bucket, "path/to/test.txt", strings.NewReader("")). - Return(true, nil). - Once() - file, err := fs.Create("/path/to/test.txt") - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - m.AssertExpectations(t) - }) - - t.Run("create dir path success", func(t *testing.T) { - m. - On("PutObject", ctx, bucket, "path/to/test_dir/", strings.NewReader("")). - Return(true, nil). - Once() - file, err := fs.Create("/path/to/test_dir/") - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - assert.Equal(t, "path/to/test_dir/", file.Name()) - m.AssertExpectations(t) - }) - - t.Run("create failure", func(t *testing.T) { - m.On("PutObject", fs.ctx, bucket, "test2.txt", strings.NewReader("")).Return(false, afero.ErrFileNotFound).Once() - _, err := fs.Create("test2.txt") - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) -} - -func TestFsMkdirAll(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("MkDirAll simple success", func(t *testing.T) { - m. - On("PutObject", fs.ctx, bucket, "path/to/test_dir/", strings.NewReader("")). - Return(true, nil). - Once() - err := fs.MkdirAll("/path/to/test_dir/", defaultFileMode) - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("MkDirAll failure", func(t *testing.T) { - m. - On("PutObject", fs.ctx, bucket, "path/to/test_dir/", strings.NewReader("")). - Return(false, afero.ErrFileClosed). - Once() - err := fs.MkdirAll("/path/to/test_dir/", defaultFileMode) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileClosed) - m.AssertExpectations(t) - }) -} - -func TestFsOpenFile(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - openedFiles: make(map[string]afero.File), - } - - t.Run("open existing file success", func(t *testing.T) { - m.On("IsObjectExist", ctx, bucket, "test.txt").Return(true, nil).Once() - file, err := fs.OpenFile("test.txt", os.O_RDONLY, 0644) - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - m.AssertExpectations(t) - }) - - t.Run("open non-existing file with create flag success", func(t *testing.T) { - m.On("IsObjectExist", ctx, bucket, "new.txt").Return(false, nil).Once() - m.On("PutObject", ctx, bucket, "new.txt", strings.NewReader("")).Return(true, nil).Once() - file, err := fs.OpenFile("new.txt", os.O_CREATE|os.O_RDWR, 0644) - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - m.AssertExpectations(t) - }) - - t.Run("open file with truncate flag success", func(t *testing.T) { - m.On("IsObjectExist", ctx, bucket, "trunc.txt").Return(true, nil).Once() - m.On("PutObject", ctx, bucket, "trunc.txt", strings.NewReader("")).Return(true, nil).Once() - file, err := fs.OpenFile("trunc.txt", os.O_TRUNC|os.O_RDWR, 0644) - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - m.AssertExpectations(t) - }) - - t.Run("open existing file from cache", func(t *testing.T) { - cachedFile := &File{name: "cached.txt", openFlag: os.O_RDONLY} - fs.openedFiles["cached.txt"] = cachedFile - file, err := fs.OpenFile("cached.txt", os.O_RDONLY, 0644) - assert.Nil(t, err) - assert.Equal(t, cachedFile, file) - assert.Implements(t, (*afero.File)(nil), file) - }) - - t.Run("open non-existing file without create flag fails", func(t *testing.T) { - m.On("IsObjectExist", ctx, bucket, "nonexist.txt").Return(false, nil).Once() - _, err := fs.OpenFile("nonexist.txt", os.O_RDONLY, 0644) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) - - t.Run("open file with check exist error fails", func(t *testing.T) { - m.On("IsObjectExist", ctx, bucket, "error.txt").Return(false, afero.ErrFileNotFound).Once() - _, err := fs.OpenFile("error.txt", os.O_RDONLY, 0644) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) -} - -func TestFsRemove(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("remove file success", func(t *testing.T) { - m.On("DeleteObject", fs.ctx, bucket, "test.txt").Return(nil).Once() - err := fs.Remove("test.txt") - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("remove prefixed file success", func(t *testing.T) { - m.On("DeleteObject", fs.ctx, bucket, "path/to/test.txt").Return(nil).Once() - err := fs.Remove("/path/to/test.txt") - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("remove non-existent file", func(t *testing.T) { - m.On("DeleteObject", fs.ctx, bucket, "nonexistent.txt").Return(afero.ErrFileNotFound).Once() - err := fs.Remove("nonexistent.txt") - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) -} - -func TestFsRemoveAll(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("remove non-empty directory", func(t *testing.T) { - dirPath := "path/to/dir/" - files := []os.FileInfo{ - NewFileInfo("path/to/dir/file1.txt", 100, time.Now()), - NewFileInfo("path/to/dir/file2.txt", 200, time.Now()), - NewFileInfo("path/to/dir/subdir/", 0, time.Now()), - } - - m.On("ListAllObjects", ctx, bucket, dirPath).Return(files, nil).Once() - m.On("DeleteObject", ctx, bucket, "path/to/dir/file1.txt").Return(nil).Once() - m.On("DeleteObject", ctx, bucket, "path/to/dir/file2.txt").Return(nil).Once() - m.On("DeleteObject", ctx, bucket, "path/to/dir/subdir/").Return(nil).Once() - - err := fs.RemoveAll(dirPath) - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("remove empty directory", func(t *testing.T) { - dirPath := "empty/dir/" - m.On("ListAllObjects", ctx, bucket, dirPath).Return([]os.FileInfo{}, nil).Once() - - err := fs.RemoveAll(dirPath) - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("remove non-existent path", func(t *testing.T) { - nonExistentPath := "nonexistent/path/" - m.On("ListAllObjects", ctx, bucket, nonExistentPath).Return([]os.FileInfo{}, nil).Once() - - err := fs.RemoveAll(nonExistentPath) - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("list objects failure", func(t *testing.T) { - dirPath := "path/to/dir/" - m.On("ListAllObjects", ctx, bucket, dirPath).Return(nil, afero.ErrFileNotFound).Once() - - err := fs.RemoveAll(dirPath) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) - - t.Run("delete object failure", func(t *testing.T) { - dirPath := "path/to/dir/" - files := []os.FileInfo{ - NewFileInfo("path/to/dir/file1.txt", 0, time.Now()), - } - - m.On("ListAllObjects", ctx, bucket, dirPath).Return(files, nil).Once() - m.On("DeleteObject", ctx, bucket, "path/to/dir/file1.txt").Return(afero.ErrFileNotFound).Once() - - err := fs.RemoveAll(dirPath) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) -} - -func TestFsRename(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("successful rename", func(t *testing.T) { - oldname := "old/file.txt" - newname := "new/file.txt" - - m.On("CopyObject", ctx, bucket, oldname, newname).Return(nil).Once() - m.On("DeleteObject", ctx, bucket, oldname).Return(nil).Once() - - err := fs.Rename(oldname, newname) - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("copy failure", func(t *testing.T) { - oldname := "old/file.txt" - newname := "new/file.txt" - - m.On("CopyObject", ctx, bucket, oldname, newname).Return(afero.ErrFileNotFound).Once() - - err := fs.Rename(oldname, newname) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) - - t.Run("delete failure after successful copy", func(t *testing.T) { - oldname := "old/file.txt" - newname := "new/file.txt" - - m.On("CopyObject", ctx, bucket, oldname, newname).Return(nil).Once() - m.On("DeleteObject", ctx, bucket, oldname).Return(afero.ErrFileNotFound).Once() - - err := fs.Rename(oldname, newname) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) -} - -func TestFsStat(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("stat file success", func(t *testing.T) { - expectedInfo := &mocks.FileInfo{} - m.On("GetObjectMeta", fs.ctx, bucket, "test.txt").Return(expectedInfo, nil).Once() - info, err := fs.Stat("test.txt") - assert.Nil(t, err) - assert.Equal(t, expectedInfo, info) - m.AssertExpectations(t) - }) - - t.Run("stat prefixed file path success", func(t *testing.T) { - expectedInfo := &mocks.FileInfo{} - m.On("GetObjectMeta", fs.ctx, bucket, "path/to/test.txt").Return(expectedInfo, nil).Once() - info, err := fs.Stat("/path/to/test.txt") - assert.Nil(t, err) - assert.Equal(t, expectedInfo, info) - m.AssertExpectations(t) - }) - - t.Run("stat dir path success", func(t *testing.T) { - expectedInfo := &mocks.FileInfo{} - m.On("GetObjectMeta", fs.ctx, bucket, "path/to/dir/").Return(expectedInfo, nil).Once() - info, err := fs.Stat("/path/to/dir/") - assert.Nil(t, err) - assert.Equal(t, expectedInfo, info) - m.AssertExpectations(t) - }) - - t.Run("stat non-existent file", func(t *testing.T) { - m.On("GetObjectMeta", fs.ctx, bucket, "nonexistent.txt").Return(nil, os.ErrNotExist).Once() - _, err := fs.Stat("nonexistent.txt") - assert.NotNil(t, err) - assert.ErrorIs(t, err, os.ErrNotExist) - m.AssertExpectations(t) - }) -} - -func TestFsName(t *testing.T) { - fs := &Fs{} - name := fs.Name() - assert.Equal(t, "OssFs", name) -} diff --git a/ossfs/fs_utils.go b/ossfs/fs_utils.go deleted file mode 100644 index 2a14609c..00000000 --- a/ossfs/fs_utils.go +++ /dev/null @@ -1,36 +0,0 @@ -package ossfs - -import ( - "strings" -) - -func (fs *Fs) isDir(s string) bool { - sep := fs.separator - if fs.separator == "" { - sep = "/" - } - return strings.HasSuffix(s, sep) -} - -func (fs *Fs) ensureAsDir(s string) string { - sep := fs.separator - if fs.separator == "" { - sep = "/" - } - s = fs.normFileName(s) - if !strings.HasSuffix(s, sep) { - s = s + sep - } - return s -} - -func (fs *Fs) normFileName(s string) string { - sep := fs.separator - if fs.separator == "" { - sep = "/" - } - s = strings.TrimLeft(s, "/\\") - s = strings.Replace(s, "\\", sep, -1) - s = strings.Replace(s, "/", sep, -1) - return s -} diff --git a/ossfs/init.go b/ossfs/init.go deleted file mode 100644 index 3a8ae873..00000000 --- a/ossfs/init.go +++ /dev/null @@ -1,14 +0,0 @@ -package ossfs - -import ( - "os" - - "github.com/spf13/afero" -) - -func init() { - // Ensure oss.Fs implements afero.Fs interface - var _ afero.Fs = (*Fs)(nil) - var _ afero.File = (*File)(nil) - var _ os.FileInfo = (*FileInfo)(nil) -} diff --git a/ossfs/internal/mocks/FileInfo.go b/ossfs/internal/mocks/FileInfo.go deleted file mode 100644 index 64328f2b..00000000 --- a/ossfs/internal/mocks/FileInfo.go +++ /dev/null @@ -1,139 +0,0 @@ -// Code generated by mockery v2.53.3. DO NOT EDIT. - -package mocks - -import ( - "os" - time "time" - - mock "github.com/stretchr/testify/mock" -) - -// FileInfo is an autogenerated mock type for the FileInfo type -type FileInfo struct { - mock.Mock -} - -// IsDir provides a mock function with no fields -func (_m *FileInfo) IsDir() bool { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for IsDir") - } - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// ModTime provides a mock function with no fields -func (_m *FileInfo) ModTime() time.Time { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for ModTime") - } - - var r0 time.Time - if rf, ok := ret.Get(0).(func() time.Time); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(time.Time) - } - - return r0 -} - -// Mode provides a mock function with no fields -func (_m *FileInfo) Mode() os.FileMode { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Mode") - } - - var r0 os.FileMode - if rf, ok := ret.Get(0).(func() os.FileMode); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(os.FileMode) - } - - return r0 -} - -// Name provides a mock function with no fields -func (_m *FileInfo) Name() string { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Name") - } - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -// Size provides a mock function with no fields -func (_m *FileInfo) Size() int64 { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Size") - } - - var r0 int64 - if rf, ok := ret.Get(0).(func() int64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int64) - } - - return r0 -} - -// Sys provides a mock function with no fields -func (_m *FileInfo) Sys() interface{} { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Sys") - } - - var r0 interface{} - if rf, ok := ret.Get(0).(func() interface{}); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(interface{}) - } - } - - return r0 -} - -// NewFileInfo creates a new instance of FileInfo. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewFileInfo(t interface { - mock.TestingT - Cleanup(func()) -}) *FileInfo { - mock := &FileInfo{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/ossfs/internal/mocks/ObjectManager.go b/ossfs/internal/mocks/ObjectManager.go deleted file mode 100644 index bb3e087b..00000000 --- a/ossfs/internal/mocks/ObjectManager.go +++ /dev/null @@ -1,293 +0,0 @@ -// Code generated by mockery v2.53.3. DO NOT EDIT. - -package mocks - -import ( - context "context" - fs "io/fs" - - io "io" - - mock "github.com/stretchr/testify/mock" - - utils "github.com/spf13/afero/ossfs/internal/utils" -) - -// ObjectManager is an autogenerated mock type for the ObjectManager type -type ObjectManager struct { - mock.Mock -} - -// CopyObject provides a mock function with given fields: ctx, bucket, srcName, targetName -func (_m *ObjectManager) CopyObject(ctx context.Context, bucket string, srcName string, targetName string) error { - ret := _m.Called(ctx, bucket, srcName, targetName) - - if len(ret) == 0 { - panic("no return value specified for CopyObject") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, string) error); ok { - r0 = rf(ctx, bucket, srcName, targetName) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// DeleteObject provides a mock function with given fields: ctx, bucket, name -func (_m *ObjectManager) DeleteObject(ctx context.Context, bucket string, name string) error { - ret := _m.Called(ctx, bucket, name) - - if len(ret) == 0 { - panic("no return value specified for DeleteObject") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { - r0 = rf(ctx, bucket, name) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetObject provides a mock function with given fields: ctx, bucket, name -func (_m *ObjectManager) GetObject(ctx context.Context, bucket string, name string) (io.Reader, utils.CleanUp, error) { - ret := _m.Called(ctx, bucket, name) - - if len(ret) == 0 { - panic("no return value specified for GetObject") - } - - var r0 io.Reader - var r1 utils.CleanUp - var r2 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) (io.Reader, utils.CleanUp, error)); ok { - return rf(ctx, bucket, name) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) io.Reader); ok { - r0 = rf(ctx, bucket, name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(io.Reader) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) utils.CleanUp); ok { - r1 = rf(ctx, bucket, name) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(utils.CleanUp) - } - } - - if rf, ok := ret.Get(2).(func(context.Context, string, string) error); ok { - r2 = rf(ctx, bucket, name) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// GetObjectMeta provides a mock function with given fields: ctx, bucket, name -func (_m *ObjectManager) GetObjectMeta(ctx context.Context, bucket string, name string) (fs.FileInfo, error) { - ret := _m.Called(ctx, bucket, name) - - if len(ret) == 0 { - panic("no return value specified for GetObjectMeta") - } - - var r0 fs.FileInfo - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) (fs.FileInfo, error)); ok { - return rf(ctx, bucket, name) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) fs.FileInfo); ok { - r0 = rf(ctx, bucket, name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(fs.FileInfo) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { - r1 = rf(ctx, bucket, name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetObjectPart provides a mock function with given fields: ctx, bucket, name, start, end -func (_m *ObjectManager) GetObjectPart(ctx context.Context, bucket string, name string, start int64, end int64) (io.Reader, utils.CleanUp, error) { - ret := _m.Called(ctx, bucket, name, start, end) - - if len(ret) == 0 { - panic("no return value specified for GetObjectPart") - } - - var r0 io.Reader - var r1 utils.CleanUp - var r2 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64) (io.Reader, utils.CleanUp, error)); ok { - return rf(ctx, bucket, name, start, end) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64) io.Reader); ok { - r0 = rf(ctx, bucket, name, start, end) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(io.Reader) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string, int64, int64) utils.CleanUp); ok { - r1 = rf(ctx, bucket, name, start, end) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(utils.CleanUp) - } - } - - if rf, ok := ret.Get(2).(func(context.Context, string, string, int64, int64) error); ok { - r2 = rf(ctx, bucket, name, start, end) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// IsObjectExist provides a mock function with given fields: ctx, bucket, name -func (_m *ObjectManager) IsObjectExist(ctx context.Context, bucket string, name string) (bool, error) { - ret := _m.Called(ctx, bucket, name) - - if len(ret) == 0 { - panic("no return value specified for IsObjectExist") - } - - var r0 bool - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) (bool, error)); ok { - return rf(ctx, bucket, name) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) bool); ok { - r0 = rf(ctx, bucket, name) - } else { - r0 = ret.Get(0).(bool) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { - r1 = rf(ctx, bucket, name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListAllObjects provides a mock function with given fields: ctx, bucket, prefix -func (_m *ObjectManager) ListAllObjects(ctx context.Context, bucket string, prefix string) ([]fs.FileInfo, error) { - ret := _m.Called(ctx, bucket, prefix) - - if len(ret) == 0 { - panic("no return value specified for ListAllObjects") - } - - var r0 []fs.FileInfo - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]fs.FileInfo, error)); ok { - return rf(ctx, bucket, prefix) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) []fs.FileInfo); ok { - r0 = rf(ctx, bucket, prefix) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]fs.FileInfo) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { - r1 = rf(ctx, bucket, prefix) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListObjects provides a mock function with given fields: ctx, bucket, prefix, count -func (_m *ObjectManager) ListObjects(ctx context.Context, bucket string, prefix string, count int) ([]fs.FileInfo, error) { - ret := _m.Called(ctx, bucket, prefix, count) - - if len(ret) == 0 { - panic("no return value specified for ListObjects") - } - - var r0 []fs.FileInfo - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, int) ([]fs.FileInfo, error)); ok { - return rf(ctx, bucket, prefix, count) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string, int) []fs.FileInfo); ok { - r0 = rf(ctx, bucket, prefix, count) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]fs.FileInfo) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string, int) error); ok { - r1 = rf(ctx, bucket, prefix, count) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// PutObject provides a mock function with given fields: ctx, bucket, name, reader -func (_m *ObjectManager) PutObject(ctx context.Context, bucket string, name string, reader io.Reader) (bool, error) { - ret := _m.Called(ctx, bucket, name, reader) - - if len(ret) == 0 { - panic("no return value specified for PutObject") - } - - var r0 bool - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, io.Reader) (bool, error)); ok { - return rf(ctx, bucket, name, reader) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string, io.Reader) bool); ok { - r0 = rf(ctx, bucket, name, reader) - } else { - r0 = ret.Get(0).(bool) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string, io.Reader) error); ok { - r1 = rf(ctx, bucket, name, reader) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewObjectManager creates a new instance of ObjectManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewObjectManager(t interface { - mock.TestingT - Cleanup(func()) -}) *ObjectManager { - mock := &ObjectManager{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/ossfs/internal/utils/contract.go b/ossfs/internal/utils/contract.go deleted file mode 100644 index 9c458ec0..00000000 --- a/ossfs/internal/utils/contract.go +++ /dev/null @@ -1,21 +0,0 @@ -package utils - -import ( - "context" - "io" - "os" -) - -type CleanUp func() - -type ObjectManager interface { - GetObject(ctx context.Context, bucket, name string) (io.Reader, CleanUp, error) - GetObjectPart(ctx context.Context, bucket, name string, start, end int64) (io.Reader, CleanUp, error) - DeleteObject(ctx context.Context, bucket, name string) error - IsObjectExist(ctx context.Context, bucket, name string) (bool, error) - PutObject(ctx context.Context, bucket, name string, reader io.Reader) (bool, error) - CopyObject(ctx context.Context, bucket, srcName, targetName string) error - GetObjectMeta(ctx context.Context, bucket, name string) (os.FileInfo, error) - ListObjects(ctx context.Context, bucket, prefix string, count int) ([]os.FileInfo, error) - ListAllObjects(ctx context.Context, bucket, prefix string) ([]os.FileInfo, error) -} diff --git a/ossfs/internal/utils/init.go b/ossfs/internal/utils/init.go deleted file mode 100644 index 72709b88..00000000 --- a/ossfs/internal/utils/init.go +++ /dev/null @@ -1,6 +0,0 @@ -package utils - -func init() { - // Ensure OssObjectManager implements ObjectManager interface - var _ ObjectManager = (*OssObjectManager)(nil) -} diff --git a/ossfs/internal/utils/oss.go b/ossfs/internal/utils/oss.go deleted file mode 100644 index 6014d71d..00000000 --- a/ossfs/internal/utils/oss.go +++ /dev/null @@ -1,210 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "io" - "io/fs" - "os" - "strings" - "time" - - "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" - "github.com/spf13/afero" -) - -var ossDirSeparator string = "/" -var ossDefaultFileMode fs.FileMode = 0o755 - -type OssObjectManager struct { - ObjectManager - Client *oss.Client -} - -func (m *OssObjectManager) GetObject(ctx context.Context, bucket, name string) (io.Reader, CleanUp, error) { - req := &oss.GetObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(name), - } - res, err := m.Client.GetObject(ctx, req) - cleanUp := func() { - res.Body.Close() - } - return res.Body, cleanUp, err -} - -func (m *OssObjectManager) GetObjectPart(ctx context.Context, bucket, name string, start, end int64) (io.Reader, CleanUp, error) { - if start > end { - return nil, nil, afero.ErrOutOfRange - } - req := &oss.GetObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(name), - Range: oss.Ptr(fmt.Sprintf("bytes=%v-%v", start, end)), - RangeBehavior: oss.Ptr("standard"), - } - res, err := m.Client.GetObject(ctx, req) - cleanUp := func() { - res.Body.Close() - } - return res.Body, cleanUp, err -} - -func (m *OssObjectManager) DeleteObject(ctx context.Context, bucket, name string) error { - req := &oss.DeleteObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(name), - } - _, err := m.Client.DeleteObject(ctx, req) - return err -} - -func (m *OssObjectManager) IsObjectExist(ctx context.Context, bucket, name string) (bool, error) { - return m.Client.IsObjectExist(ctx, bucket, name) -} - -func (m *OssObjectManager) PutObject(ctx context.Context, bucket, name string, reader io.Reader) (bool, error) { - req := &oss.PutObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(name), - Body: reader, - } - _, err := m.Client.PutObject(ctx, req) - if err != nil { - return false, err - } - return true, nil -} - -func (m *OssObjectManager) CopyObject(ctx context.Context, bucket, srcName, targetName string) error { - req := &oss.CopyObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(srcName), - SourceKey: oss.Ptr(targetName), - SourceBucket: oss.Ptr(bucket), - StorageClass: oss.StorageClassStandard, - } - _, err := m.Client.CopyObject(ctx, req) - return err -} - -func (m *OssObjectManager) GetObjectMeta(ctx context.Context, bucket, name string) (os.FileInfo, error) { - req := &oss.HeadObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(name), - } - - res, err := m.Client.HeadObject(ctx, req) - if err != nil { - return nil, err - } - return &OssObjectMeta{ - name: name, - size: res.ContentLength, - lastModifiedAt: *res.LastModified, - }, nil -} - -func (m *OssObjectManager) ListObjects(ctx context.Context, bucket, prefix string, count int) ([]os.FileInfo, error) { - req := &oss.ListObjectsV2Request{ - Bucket: oss.Ptr(bucket), - Delimiter: oss.Ptr(ossDirSeparator), - Prefix: oss.Ptr(prefix), - } - p := m.Client.NewListObjectsV2Paginator(req) - - s := make([]os.FileInfo, 0) - - var i int - -loop: - for p.HasNext() { - page, err := p.NextPage(ctx) - if err != nil { - return nil, err - } - - for _, obj := range page.Contents { - i++ - if i == count { - break loop - } - s = append(s, &OssObjectMeta{ - name: oss.ToString(obj.Key), - size: obj.Size, - lastModifiedAt: oss.ToTime(obj.LastModified), - }) - } - } - - return s, nil -} - -func (m *OssObjectManager) ListAllObjects(ctx context.Context, bucket, prefix string) ([]os.FileInfo, error) { - req := &oss.ListObjectsV2Request{ - Bucket: oss.Ptr(bucket), - Prefix: oss.Ptr(prefix), - } - p := m.Client.NewListObjectsV2Paginator(req) - - s := make([]os.FileInfo, 0) - - for p.HasNext() { - page, err := p.NextPage(ctx) - if err != nil { - return nil, err - } - - for _, obj := range page.Contents { - s = append(s, &OssObjectMeta{ - name: oss.ToString(obj.Key), - size: obj.Size, - lastModifiedAt: oss.ToTime(obj.LastModified), - }) - } - } - - return s, nil -} - -type OssObjectMeta struct { - os.FileInfo - name string - size int64 - lastModifiedAt time.Time -} - -func NewOssObjectMeta(name string, size int64, updatedAt time.Time) *OssObjectMeta { - return &OssObjectMeta{ - name: name, - size: size, - lastModifiedAt: updatedAt, - } -} - -func (objMeta *OssObjectMeta) isDir() bool { - return strings.HasSuffix(objMeta.name, ossDirSeparator) -} - -func (objMeta *OssObjectMeta) ModTime() time.Time { - return objMeta.lastModifiedAt -} - -func (objMeta *OssObjectMeta) Mode() fs.FileMode { - if objMeta.isDir() { - return ossDefaultFileMode | fs.ModeDir - } - return ossDefaultFileMode -} - -func (objMeta *OssObjectMeta) Name() string { - return objMeta.name -} - -func (objMeta *OssObjectMeta) Size() int64 { - return objMeta.size -} - -func (objMeta *OssObjectMeta) Sys() any { - return nil -} From 264143d8d74cfabff43ae23d0a79421041c8136c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 20:29:22 +0000 Subject: [PATCH 14/34] build(deps): bump actions/dependency-review-action from 4.5.0 to 4.6.0 Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 4.5.0 to 4.6.0. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/3b139cfc5fae8b618d3eae3675e383bb1769c019...ce3cf9537a52e8119d91fd484ab5b8a807627bf8) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-version: 4.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a9e7123b..2e4f5aa2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -64,4 +64,4 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Dependency Review - uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0 + uses: actions/dependency-review-action@ce3cf9537a52e8119d91fd484ab5b8a807627bf8 # v4.6.0 From c50eb1d89641f6094accd75b238d7ff2dff3780d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 20:29:57 +0000 Subject: [PATCH 15/34] build(deps): bump github/codeql-action from 2.13.4 to 3.28.15 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.13.4 to 3.28.15. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/cdcdbb579706841c47f7063dda365e292e5cad7a...45775bd8235c68ba998cffa5171334d58593da47) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.28.15 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/analysis-scorecard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml index 6f686cc0..4a5c8444 100644 --- a/.github/workflows/analysis-scorecard.yaml +++ b/.github/workflows/analysis-scorecard.yaml @@ -42,6 +42,6 @@ jobs: retention-days: 5 - name: Upload results to GitHub Security tab - uses: github/codeql-action/upload-sarif@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4 + uses: github/codeql-action/upload-sarif@45775bd8235c68ba998cffa5171334d58593da47 # v3.28.15 with: sarif_file: results.sarif From dc288a4607abe488caa7b4b61a7c574322a216c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Apr 2025 20:23:06 +0000 Subject: [PATCH 16/34] build(deps): bump github/codeql-action from 3.28.15 to 3.28.16 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.28.15 to 3.28.16. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/45775bd8235c68ba998cffa5171334d58593da47...28deaeda66b76a05916b6923827895f2b14ab387) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.28.16 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/analysis-scorecard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml index 4a5c8444..eda201f8 100644 --- a/.github/workflows/analysis-scorecard.yaml +++ b/.github/workflows/analysis-scorecard.yaml @@ -42,6 +42,6 @@ jobs: retention-days: 5 - name: Upload results to GitHub Security tab - uses: github/codeql-action/upload-sarif@45775bd8235c68ba998cffa5171334d58593da47 # v3.28.15 + uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16 with: sarif_file: results.sarif From ecfd29bd6b330e100692de90278971c906f40d0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 May 2025 20:40:18 +0000 Subject: [PATCH 17/34] build(deps): bump github/codeql-action from 3.28.16 to 3.28.17 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.28.16 to 3.28.17. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/28deaeda66b76a05916b6923827895f2b14ab387...60168efe1c415ce0f5521ea06d5c2062adbeed1b) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.28.17 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/analysis-scorecard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml index eda201f8..4394336b 100644 --- a/.github/workflows/analysis-scorecard.yaml +++ b/.github/workflows/analysis-scorecard.yaml @@ -42,6 +42,6 @@ jobs: retention-days: 5 - name: Upload results to GitHub Security tab - uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16 + uses: github/codeql-action/upload-sarif@60168efe1c415ce0f5521ea06d5c2062adbeed1b # v3.28.17 with: sarif_file: results.sarif From 7d032558714c9674970e3e8b5e44708033511a38 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Wed, 7 May 2025 01:56:04 +0300 Subject: [PATCH 18/34] chore: update x/test Signed-off-by: Mark Sagi-Kazar --- gcsfs/go.mod | 4 ++-- gcsfs/go.sum | 8 ++++---- go.mod | 2 +- go.sum | 4 ++-- sftpfs/go.mod | 2 +- sftpfs/go.sum | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/gcsfs/go.mod b/gcsfs/go.mod index 64cb4b76..d076adc5 100644 --- a/gcsfs/go.mod +++ b/gcsfs/go.mod @@ -45,9 +45,9 @@ require ( go.opentelemetry.io/otel/trace v1.34.0 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/net v0.37.0 // indirect - golang.org/x/sync v0.12.0 // indirect + golang.org/x/sync v0.14.0 // indirect golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/text v0.25.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect diff --git a/gcsfs/go.sum b/gcsfs/go.sum index b74bb42c..d3217703 100644 --- a/gcsfs/go.sum +++ b/gcsfs/go.sum @@ -95,12 +95,12 @@ golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= google.golang.org/api v0.226.0 h1:9A29y1XUD+YRXfnHkO66KggxHBZWg9LsTGqm7TkUvtQ= diff --git a/go.mod b/go.mod index 101c2865..3259b2cd 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,4 @@ module github.com/spf13/afero go 1.23.0 -require golang.org/x/text v0.23.0 +require golang.org/x/text v0.25.0 diff --git a/go.sum b/go.sum index d00bb390..3470e4e3 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,2 @@ -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= diff --git a/sftpfs/go.mod b/sftpfs/go.mod index ee2d5469..19dcfc2e 100644 --- a/sftpfs/go.mod +++ b/sftpfs/go.mod @@ -13,5 +13,5 @@ require ( require ( github.com/kr/fs v0.1.0 // indirect golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/text v0.25.0 // indirect ) diff --git a/sftpfs/go.sum b/sftpfs/go.sum index 04e303ce..6f8e2a67 100644 --- a/sftpfs/go.sum +++ b/sftpfs/go.sum @@ -75,8 +75,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= From 7b878d40362bc1fb8c45d85cf3615c6c66ed8ff3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 20:13:04 +0000 Subject: [PATCH 19/34] build(deps): bump actions/setup-go from 5.4.0 to 5.5.0 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5.4.0 to 5.5.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/0aaccfd150d50ccaeb58ebd88d36e91967a5f35b...d35c59abb061a4a6fb18e82ac0862c26744d6ab5) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: 5.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4f12ce86..1a5e1321 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Go - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: go-version: ${{ matrix.go }} @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Go - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: go-version: "1.24" From 02d222034279acefec1bd533bb8a0791d005c749 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 May 2025 20:25:56 +0000 Subject: [PATCH 20/34] build(deps): bump actions/dependency-review-action from 4.6.0 to 4.7.1 Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 4.6.0 to 4.7.1. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/ce3cf9537a52e8119d91fd484ab5b8a807627bf8...da24556b548a50705dd671f47852072ea4c105d9) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-version: 4.7.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4f12ce86..ed6d9c2b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -64,4 +64,4 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Dependency Review - uses: actions/dependency-review-action@ce3cf9537a52e8119d91fd484ab5b8a807627bf8 # v4.6.0 + uses: actions/dependency-review-action@da24556b548a50705dd671f47852072ea4c105d9 # v4.7.1 From f8b64d249443ab200c5eb3e911fc99698e162e4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 May 2025 20:13:11 +0000 Subject: [PATCH 21/34] build(deps): bump github/codeql-action from 3.28.17 to 3.28.18 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.28.17 to 3.28.18. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/60168efe1c415ce0f5521ea06d5c2062adbeed1b...ff0a06e83cb2de871e5a09832bc6a81e7276941f) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.28.18 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/analysis-scorecard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml index 4394336b..422ce63d 100644 --- a/.github/workflows/analysis-scorecard.yaml +++ b/.github/workflows/analysis-scorecard.yaml @@ -42,6 +42,6 @@ jobs: retention-days: 5 - name: Upload results to GitHub Security tab - uses: github/codeql-action/upload-sarif@60168efe1c415ce0f5521ea06d5c2062adbeed1b # v3.28.17 + uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 with: sarif_file: results.sarif From 4bec3cd1a82ed580cb8c2686fded34b92a663880 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 May 2025 20:29:54 +0000 Subject: [PATCH 22/34] build(deps): bump ossf/scorecard-action from 2.4.1 to 2.4.2 Bumps [ossf/scorecard-action](https://github.com/ossf/scorecard-action) from 2.4.1 to 2.4.2. - [Release notes](https://github.com/ossf/scorecard-action/releases) - [Changelog](https://github.com/ossf/scorecard-action/blob/main/RELEASE.md) - [Commits](https://github.com/ossf/scorecard-action/compare/f49aabe0b5af0936a0987cfb85d86b75731b0186...05b42c624433fc40578a4040d5cf5e36ddca8cde) --- updated-dependencies: - dependency-name: ossf/scorecard-action dependency-version: 2.4.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/analysis-scorecard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml index 422ce63d..3bf05b0b 100644 --- a/.github/workflows/analysis-scorecard.yaml +++ b/.github/workflows/analysis-scorecard.yaml @@ -28,7 +28,7 @@ jobs: persist-credentials: false - name: Run analysis - uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 with: results_file: results.sarif results_format: sarif From 13f8aa3c845c389e2849f70e7961b57943488fad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 21:09:37 +0000 Subject: [PATCH 23/34] build(deps): bump github/codeql-action from 3.28.18 to 3.29.4 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.28.18 to 3.29.4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/ff0a06e83cb2de871e5a09832bc6a81e7276941f...4e828ff8d448a8a6e532957b1811f387a63867e8) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.29.4 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/analysis-scorecard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml index 3bf05b0b..39508255 100644 --- a/.github/workflows/analysis-scorecard.yaml +++ b/.github/workflows/analysis-scorecard.yaml @@ -42,6 +42,6 @@ jobs: retention-days: 5 - name: Upload results to GitHub Security tab - uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 + uses: github/codeql-action/upload-sarif@4e828ff8d448a8a6e532957b1811f387a63867e8 # v3.29.4 with: sarif_file: results.sarif From c448d4915fd021670a243a36d571bc75ce2f3bf9 Mon Sep 17 00:00:00 2001 From: Mark Rosemaker <48681726+MarkRosemaker@users.noreply.github.com> Date: Sun, 3 Aug 2025 18:34:48 +0200 Subject: [PATCH 24/34] fix spelling --- basepath_test.go | 6 +++--- lstater.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/basepath_test.go b/basepath_test.go index 1120210c..828bb6a8 100644 --- a/basepath_test.go +++ b/basepath_test.go @@ -80,10 +80,10 @@ func TestRealPath(t *testing.T) { t.Errorf("Got error %s", err) } - excpected := filepath.Join(baseDir, anotherDir) + expected := filepath.Join(baseDir, anotherDir) - if surrealPath != excpected { - t.Errorf("Expected \n%s got \n%s", excpected, surrealPath) + if surrealPath != expected { + t.Errorf("Expected \n%s got \n%s", expected, surrealPath) } } } diff --git a/lstater.go b/lstater.go index 360edf93..2dcbdb1f 100644 --- a/lstater.go +++ b/lstater.go @@ -19,7 +19,7 @@ import ( // Lstater is an optional interface in Afero. It is only implemented by the // filesystems saying so. -// It will call Lstat if the filesystem iself is, or it delegates to, the os filesystem. +// It will call Lstat if the filesystem itself is, or it delegates to, the os filesystem. // Else it will call Stat. // In addition to the FileInfo, it will return a boolean telling whether Lstat was called or not. type Lstater interface { From 0254e8dcdcf60fd74a934678c47076f9806a253a Mon Sep 17 00:00:00 2001 From: Steve Francia Date: Mon, 4 Aug 2025 17:16:20 -0400 Subject: [PATCH 25/34] docs: update README to enhance clarity and structure, improve feature descriptions --- README.md | 681 ++++++++++++++++++++++++++---------------------------- 1 file changed, 334 insertions(+), 347 deletions(-) diff --git a/README.md b/README.md index f42c3887..5c1c17cb 100644 --- a/README.md +++ b/README.md @@ -1,482 +1,469 @@ -![afero logo-sm](https://cloud.githubusercontent.com/assets/173412/11490338/d50e16dc-97a5-11e5-8b12-019a300d0fcb.png) +# Afero: The Universal Filesystem Abstraction for Go -A FileSystem Abstraction System for Go +[![GoDoc](https://godoc.org/github.com/spf13/afero?status.svg)](https://godoc.org/github.com/spf13/afero) +[![Build Status](https://github.com/spf13/afero/actions/workflows/test.yml/badge.svg)](https://github.com/spf13/afero/actions/workflows/test.yml) +[![Go Report Card](https://goreportcard.com/badge/github.com/spf13/afero)](https://goreportcard.com/report/github.com/spf13/afero) -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/spf13/afero/ci.yaml?style=flat-square)](https://github.com/spf13/afero/actions/workflows/ci.yaml) -[![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/mod/github.com/spf13/afero) -![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/spf13/afero?style=flat-square&color=61CFDD) -[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/spf13/afero/badge?style=flat-square)](https://deps.dev/go/github.com%252Fspf13%252Fafero) +Afero is a powerful and extensible filesystem abstraction system for Go. It provides a single, unified API for interacting with diverse filesystems—including the local disk, memory, archives, and network storage. -# Overview +Afero acts as a drop-in replacement for the standard `os` package, enabling you to write modular code that is agnostic to the underlying storage, dramatically simplifies testing, and allows for sophisticated architectural patterns through filesystem composition. -Afero is a filesystem framework providing a simple, uniform and universal API -interacting with any filesystem, as an abstraction layer providing interfaces, -types and methods. Afero has an exceptionally clean interface and simple design -without needless constructors or initialization methods. +## Why Afero? -Afero is also a library providing a base set of interoperable backend -filesystems that make it easy to work with, while retaining all the power -and benefit of the os and ioutil packages. +Afero elevates filesystem interaction beyond simple file reading and writing, offering solutions for testability, flexibility, and advanced architecture. -Afero provides significant improvements over using the os package alone, most -notably the ability to create mock and testing filesystems without relying on the disk. +🔑 **Key Features:** -It is suitable for use in any situation where you would consider using the OS -package as it provides an additional abstraction that makes it easy to use a -memory backed file system during testing. It also adds support for the http -filesystem for full interoperability. +* **Universal API:** Write your code once. Run it against the local OS, in-memory storage, ZIP/TAR archives, or remote systems (SFTP, GCS). +* **Ultimate Testability:** Utilize `MemMapFs`, a fully concurrent-safe, read/write in-memory filesystem. Write fast, isolated, and reliable unit tests without touching the physical disk or worrying about cleanup. +* **Powerful Composition:** Afero's hidden superpower. Layer filesystems on top of each other to create sophisticated behaviors: + * **Sandboxing:** Use `CopyOnWriteFs` to create temporary scratch spaces that isolate changes from the base filesystem. + * **Caching:** Use `CacheOnReadFs` to automatically layer a fast cache (like memory) over a slow backend (like a network drive). + * **Security Jails:** Use `BasePathFs` to restrict application access to a specific subdirectory (chroot). +* **`os` Package Compatibility:** Afero mirrors the functions in the standard `os` package, making adoption and refactoring seamless. +* **`io/fs` Compatibility:** Fully compatible with the Go standard library's `io/fs` interfaces. +## Installation -## Afero Features - -* A single consistent API for accessing a variety of filesystems -* Interoperation between a variety of file system types -* A set of interfaces to encourage and enforce interoperability between backends -* An atomic cross platform memory backed file system -* Support for compositional (union) file systems by combining multiple file systems acting as one -* Specialized backends which modify existing filesystems (Read Only, Regexp filtered) -* A set of utility functions ported from io, ioutil & hugo to be afero aware -* Wrapper for go 1.16 filesystem abstraction `io/fs.FS` - -# Using Afero - -Afero is easy to use and easier to adopt. - -A few different ways you could use Afero: - -* Use the interfaces alone to define your own file system. -* Wrapper for the OS packages. -* Define different filesystems for different parts of your application. -* Use Afero for mock filesystems while testing - -## Step 1: Install Afero - -First use go get to install the latest version of the library. - - $ go get github.com/spf13/afero +```bash +go get github.com/spf13/afero +``` -Next include Afero in your application. ```go import "github.com/spf13/afero" ``` -## Step 2: Declare a backend +## Quick Start: The Power of Abstraction + +The core of Afero is the `afero.Fs` interface. By designing your functions to accept this interface rather than calling `os.*` functions directly, your code instantly becomes more flexible and testable. + +### 1. Refactor Your Code + +Change functions that rely on the `os` package to accept `afero.Fs`. -First define a package variable and set it to a pointer to a filesystem. ```go -var AppFs = afero.NewMemMapFs() +// Before: Coupled to the OS and difficult to test +// func ProcessConfiguration(path string) error { +// data, err := os.ReadFile(path) +// ... +// } -or +import "github.com/spf13/afero" -var AppFs = afero.NewOsFs() +// After: Decoupled, flexible, and testable +func ProcessConfiguration(fs afero.Fs, path string) error { + // Use Afero utility functions which mirror os/ioutil + data, err := afero.ReadFile(fs, path) + // ... process the data + return err +} ``` -It is important to note that if you repeat the composite literal you -will be using a completely new and isolated filesystem. In the case of -OsFs it will still use the same underlying filesystem but will reduce -the ability to drop in other filesystems as desired. -## Step 3: Use it like you would the OS package +### 2. Usage in Production -Throughout your application use any function and method like you normally -would. +In your production environment, inject the `OsFs` backend, which wraps the standard operating system calls. -So if my application before had: -```go -os.Open("/tmp/foo") -``` -We would replace it with: ```go -AppFs.Open("/tmp/foo") +func main() { + // Use the real OS filesystem + AppFs := afero.NewOsFs() + ProcessConfiguration(AppFs, "/etc/myapp.conf") +} ``` -`AppFs` being the variable we defined above. +### 3. Usage in Testing +In your tests, inject `MemMapFs`. This provides a blazing-fast, isolated, in-memory filesystem that requires no disk I/O and no cleanup. -## List of all available functions - -File System Methods Available: -```go -Chmod(name string, mode os.FileMode) : error -Chown(name string, uid, gid int) : error -Chtimes(name string, atime time.Time, mtime time.Time) : error -Create(name string) : File, error -Mkdir(name string, perm os.FileMode) : error -MkdirAll(path string, perm os.FileMode) : error -Name() : string -Open(name string) : File, error -OpenFile(name string, flag int, perm os.FileMode) : File, error -Remove(name string) : error -RemoveAll(path string) : error -Rename(oldname, newname string) : error -Stat(name string) : os.FileInfo, error -``` -File Interfaces and Methods Available: ```go -io.Closer -io.Reader -io.ReaderAt -io.Seeker -io.Writer -io.WriterAt - -Name() : string -Readdir(count int) : []os.FileInfo, error -Readdirnames(n int) : []string, error -Stat() : os.FileInfo, error -Sync() : error -Truncate(size int64) : error -WriteString(s string) : ret int, err error +func TestProcessConfiguration(t *testing.T) { + // Use the in-memory filesystem + AppFs := afero.NewMemMapFs() + + // Pre-populate the memory filesystem for the test + configPath := "/test/config.json" + afero.WriteFile(AppFs, configPath, []byte(`{"feature": true}`), 0644) + + // Run the test entirely in memory + err := ProcessConfiguration(AppFs, configPath) + if err != nil { + t.Fatal(err) + } +} ``` -In some applications it may make sense to define a new package that -simply exports the file system variable for easy access from anywhere. -## Using Afero's utility functions +## Afero's Superpower: Composition -Afero provides a set of functions to make it easier to use the underlying file systems. -These functions have been primarily ported from io & ioutil with some developed for Hugo. +Afero's most unique feature is its ability to combine filesystems. This allows you to build complex behaviors out of simple components, keeping your application logic clean. -The afero utilities support all afero compatible backends. +### Example 1: Sandboxing with Copy-on-Write -The list of utilities includes: +Create a temporary environment where an application can "modify" system files without affecting the actual disk. ```go -DirExists(path string) (bool, error) -Exists(path string) (bool, error) -FileContainsBytes(filename string, subslice []byte) (bool, error) -GetTempDir(subPath string) string -IsDir(path string) (bool, error) -IsEmpty(path string) (bool, error) -ReadDir(dirname string) ([]os.FileInfo, error) -ReadFile(filename string) ([]byte, error) -SafeWriteReader(path string, r io.Reader) (err error) -TempDir(dir, prefix string) (name string, err error) -TempFile(dir, prefix string) (f File, err error) -Walk(root string, walkFn filepath.WalkFunc) error -WriteFile(filename string, data []byte, perm os.FileMode) error -WriteReader(path string, r io.Reader) (err error) -``` -For a complete list see [Afero's GoDoc](https://godoc.org/github.com/spf13/afero) +// 1. The base layer is the real OS, made read-only for safety. +baseFs := afero.NewReadOnlyFs(afero.NewOsFs()) -They are available under two different approaches to use. You can either call -them directly where the first parameter of each function will be the file -system, or you can declare a new `Afero`, a custom type used to bind these -functions as methods to a given filesystem. +// 2. The overlay layer is a temporary in-memory filesystem for changes. +overlayFs := afero.NewMemMapFs() -### Calling utilities directly +// 3. Combine them. Reads fall through to the base; writes only hit the overlay. +sandboxFs := afero.NewCopyOnWriteFs(baseFs, overlayFs) -```go -fs := new(afero.MemMapFs) -f, err := afero.TempFile(fs,"", "ioutil-test") +// The application can now "modify" /etc/hosts, but the changes are isolated in memory. +afero.WriteFile(sandboxFs, "/etc/hosts", []byte("127.0.0.1 sandboxed-app"), 0644) +// The real /etc/hosts on disk is untouched. ``` -### Calling via Afero +### Example 2: Caching a Slow Filesystem -```go -fs := afero.NewMemMapFs() -afs := &afero.Afero{Fs: fs} -f, err := afs.TempFile("", "ioutil-test") -``` +Improve performance by layering a fast cache (like memory) over a slow backend (like a network drive or cloud storage). -## Using Afero for Testing +```go +import "time" -There is a large benefit to using a mock filesystem for testing. It has a -completely blank state every time it is initialized and can be easily -reproducible regardless of OS. You could create files to your heart’s content -and the file access would be fast while also saving you from all the annoying -issues with deleting temporary files, Windows file locking, etc. The MemMapFs -backend is perfect for testing. +// Assume 'remoteFs' is a slow backend (e.g., SFTP or GCS) +var remoteFs afero.Fs -* Much faster than performing I/O operations on disk -* Avoid security issues and permissions -* Far more control. 'rm -rf /' with confidence -* Test setup is far more easier to do -* No test cleanup needed +// 'cacheFs' is a fast in-memory backend +cacheFs := afero.NewMemMapFs() -One way to accomplish this is to define a variable as mentioned above. -In your application this will be set to afero.NewOsFs() during testing you -can set it to afero.NewMemMapFs(). +// Create the caching layer. Cache items for 5 minutes upon first read. +cachedFs := afero.NewCacheOnReadFs(remoteFs, cacheFs, 5*time.Minute) -It wouldn't be uncommon to have each test initialize a blank slate memory -backend. To do this I would define my `appFS = afero.NewOsFs()` somewhere -appropriate in my application code. This approach ensures that Tests are order -independent, with no test relying on the state left by an earlier test. +// The first read is slow (fetches from remote, then caches) +data1, _ := afero.ReadFile(cachedFs, "data.json") -Then in my tests I would initialize a new MemMapFs for each test: -```go -func TestExist(t *testing.T) { - appFS := afero.NewMemMapFs() - // create test files and directories - appFS.MkdirAll("src/a", 0755) - afero.WriteFile(appFS, "src/a/b", []byte("file b"), 0644) - afero.WriteFile(appFS, "src/c", []byte("file c"), 0644) - name := "src/c" - _, err := appFS.Stat(name) - if os.IsNotExist(err) { - t.Errorf("file \"%s\" does not exist.\n", name) - } -} +// The second read is instant (serves from memory cache) +data2, _ := afero.ReadFile(cachedFs, "data.json") ``` -# Available Backends +### Example 3: Security Jails (chroot) + +Restrict an application component's access to a specific subdirectory. -## Operating System Native +```go +osFs := afero.NewOsFs() -### OsFs +// Create a filesystem rooted at /home/user/public +// The application cannot access anything above this directory. +jailedFs := afero.NewBasePathFs(osFs, "/home/user/public") -The first is simply a wrapper around the native OS calls. This makes it -very easy to use as all of the calls are the same as the existing OS -calls. It also makes it trivial to have your code use the OS during -operation and a mock filesystem during testing or as needed. +// To the application, this is reading "/" +// In reality, it's reading "/home/user/public/" +dirInfo, err := afero.ReadDir(jailedFs, "/") -```go -appfs := afero.NewOsFs() -appfs.MkdirAll("src/a", 0755) +// Attempts to access parent directories fail +_, err = jailedFs.Open("../secrets.txt") // Returns an error ``` -## Memory Backed Storage +## Real-World Use Cases -### MemMapFs +### Build Cloud-Agnostic Applications -Afero also provides a fully atomic memory backed filesystem perfect for use in -mocking and to speed up unnecessary disk io when persistence isn’t -necessary. It is fully concurrent and will work within go routines -safely. +Write applications that seamlessly work with different storage backends: ```go -mm := afero.NewMemMapFs() -mm.MkdirAll("src/a", 0755) -``` +type DocumentProcessor struct { + fs afero.Fs +} -#### InMemoryFile +func NewDocumentProcessor(fs afero.Fs) *DocumentProcessor { + return &DocumentProcessor{fs: fs} +} + +func (p *DocumentProcessor) Process(inputPath, outputPath string) error { + // This code works whether fs is local disk, cloud storage, or memory + content, err := afero.ReadFile(p.fs, inputPath) + if err != nil { + return err + } + + processed := processContent(content) + return afero.WriteFile(p.fs, outputPath, processed, 0644) +} -As part of MemMapFs, Afero also provides an atomic, fully concurrent memory -backed file implementation. This can be used in other memory backed file -systems with ease. Plans are to add a radix tree memory stored file -system using InMemoryFile. +// Use with local filesystem +processor := NewDocumentProcessor(afero.NewOsFs()) -## Network Interfaces +// Use with Google Cloud Storage +processor := NewDocumentProcessor(gcsFS) -### SftpFs +// Use with in-memory filesystem for testing +processor := NewDocumentProcessor(afero.NewMemMapFs()) +``` -Afero has experimental support for secure file transfer protocol (sftp). Which can -be used to perform file operations over a encrypted channel. +### Treating Archives as Filesystems -### GCSFs +Read files directly from `.zip` or `.tar` archives without unpacking them to disk first. -Afero has experimental support for Google Cloud Storage (GCS). You can either set the -`GOOGLE_APPLICATION_CREDENTIALS_JSON` env variable to your JSON credentials or use `opts` in -`NewGcsFS` to configure access to your GCS bucket. +```go +import ( + "archive/zip" + "github.com/spf13/afero/zipfs" +) -Some known limitations of the existing implementation: -* No Chmod support - The GCS ACL could probably be mapped to *nix style permissions but that would add another level of complexity and is ignored in this version. -* No Chtimes support - Could be simulated with attributes (gcs a/m-times are set implicitly) but that's is left for another version. -* Not thread safe - Also assumes all file operations are done through the same instance of the GcsFs. File operations between different GcsFs instances are not guaranteed to be consistent. +// Assume 'zipReader' is a *zip.Reader initialized from a file or memory +var zipReader *zip.Reader +// Create a read-only ZipFs +archiveFS := zipfs.New(zipReader) -## Filtering Backends +// Read a file from within the archive using the standard Afero API +content, err := afero.ReadFile(archiveFS, "/docs/readme.md") +``` -### BasePathFs +### Serving Any Filesystem over HTTP -The BasePathFs restricts all operations to a given path within an Fs. -The given file name to the operations on this Fs will be prepended with -the base path before calling the source Fs. +Use `HttpFs` to expose any Afero filesystem—even one created dynamically in memory—through a standard Go web server. ```go -bp := afero.NewBasePathFs(afero.NewOsFs(), "/base/path") -``` +import ( + "net/http" + "github.com/spf13/afero" +) -### ReadOnlyFs +func main() { + memFS := afero.NewMemMapFs() + afero.WriteFile(memFS, "index.html", []byte("

Hello from Memory!

"), 0644) -A thin wrapper around the source Fs providing a read only view. + // Wrap the memory filesystem to make it compatible with http.FileServer. + httpFS := afero.NewHttpFs(memFS) -```go -fs := afero.NewReadOnlyFs(afero.NewOsFs()) -_, err := fs.Create("/file.txt") -// err = syscall.EPERM + http.Handle("/", http.FileServer(httpFS.Dir("/"))) + http.ListenAndServe(":8080", nil) +} ``` -# RegexpFs +### Testing Made Simple -A filtered view on file names, any file NOT matching -the passed regexp will be treated as non-existing. -Files not matching the regexp provided will not be created. -Directories are not filtered. +One of Afero's greatest strengths is making filesystem-dependent code easily testable: ```go -fs := afero.NewRegexpFs(afero.NewMemMapFs(), regexp.MustCompile(`\.txt$`)) -_, err := fs.Create("/file.html") -// err = syscall.ENOENT -``` +func SaveUserData(fs afero.Fs, userID string, data []byte) error { + filename := fmt.Sprintf("users/%s.json", userID) + return afero.WriteFile(fs, filename, data, 0644) +} -### HttpFs +func TestSaveUserData(t *testing.T) { + // Create a clean, fast, in-memory filesystem for testing + testFS := afero.NewMemMapFs() + + userData := []byte(`{"name": "John", "email": "john@example.com"}`) + err := SaveUserData(testFS, "123", userData) + + if err != nil { + t.Fatalf("SaveUserData failed: %v", err) + } + + // Verify the file was saved correctly + saved, err := afero.ReadFile(testFS, "users/123.json") + if err != nil { + t.Fatalf("Failed to read saved file: %v", err) + } + + if string(saved) != string(userData) { + t.Errorf("Data mismatch: got %s, want %s", saved, userData) + } +} +``` -Afero provides an http compatible backend which can wrap any of the existing -backends. +**Benefits of testing with Afero:** +- ⚡ **Fast** - No disk I/O, tests run in memory +- 🔄 **Reliable** - Each test starts with a clean slate +- 🧹 **No cleanup** - Memory is automatically freed +- 🔒 **Safe** - Can't accidentally modify real files +- 🏃 **Parallel** - Tests can run concurrently without conflicts + +## Backend Reference + +| Type | Backend | Constructor | Description | Status | +| :--- | :--- | :--- | :--- | :--- | +| **Core** | **OsFs** | `afero.NewOsFs()` | Interacts with the real operating system filesystem. Use in production. | ✅ Official | +| | **MemMapFs** | `afero.NewMemMapFs()` | A fast, atomic, concurrent-safe, in-memory filesystem. Ideal for testing. | ✅ Official | +| **Composition** | **CopyOnWriteFs**| `afero.NewCopyOnWriteFs(base, overlay)` | A read-only base with a writable overlay. Ideal for sandboxing. | ✅ Official | +| | **CacheOnReadFs**| `afero.NewCacheOnReadFs(base, cache, ttl)` | Lazily caches files from a slow base into a fast layer on first read. | ✅ Official | +| | **BasePathFs** | `afero.NewBasePathFs(source, path)` | Restricts operations to a subdirectory (chroot/jail). | ✅ Official | +| | **ReadOnlyFs** | `afero.NewReadOnlyFs(source)` | Provides a read-only view, preventing any modifications. | ✅ Official | +| | **RegexpFs** | `afero.NewRegexpFs(source, regexp)` | Filters a filesystem, only showing files that match a regex. | ✅ Official | +| **Utility** | **HttpFs** | `afero.NewHttpFs(source)` | Wraps any Afero filesystem to be served via `http.FileServer`. | ✅ Official | +| **Archives** | **ZipFs** | `zipfs.New(zipReader)` | Read-only access to files within a ZIP archive. | ✅ Official | +| | **TarFs** | `tarfs.New(tarReader)` | Read-only access to files within a TAR archive. | ✅ Official | +| **Network** | **GcsFs** | `gcsfs.NewGcsFs(...)` | Google Cloud Storage backend. | ⚡ Experimental | +| | **SftpFs** | `sftpfs.New(...)` | SFTP backend. | ⚡ Experimental | +| **3rd Party Cloud** | **S3Fs** | [`fclairamb/afero-s3`](https://github.com/fclairamb/afero-s3) | Production-ready S3 backend built on official AWS SDK. | 🔹 3rd Party | +| | **MinioFs** | [`cpyun/afero-minio`](https://github.com/cpyun/afero-minio) | MinIO object storage backend with S3 compatibility. | 🔹 3rd Party | +| | **DriveFs** | [`fclairamb/afero-gdrive`](https://github.com/fclairamb/afero-gdrive) | Google Drive backend with streaming support. | 🔹 3rd Party | +| | **DropboxFs** | [`fclairamb/afero-dropbox`](https://github.com/fclairamb/afero-dropbox) | Dropbox backend with streaming support. | 🔹 3rd Party | +| **3rd Party Specialized** | **GitFs** | [`tobiash/go-gitfs`](https://github.com/tobiash/go-gitfs) | Git repository filesystem (read-only, Afero compatible). | 🔹 3rd Party | +| | **DockerFs** | [`unmango/aferox`](https://github.com/unmango/aferox) | Docker container filesystem access. | 🔹 3rd Party | +| | **GitHubFs** | [`unmango/aferox`](https://github.com/unmango/aferox) | GitHub repository and releases filesystem. | 🔹 3rd Party | +| | **FilterFs** | [`unmango/aferox`](https://github.com/unmango/aferox) | Filesystem filtering with predicates. | 🔹 3rd Party | +| | **IgnoreFs** | [`unmango/aferox`](https://github.com/unmango/aferox) | .gitignore-aware filtering filesystem. | 🔹 3rd Party | +| | **FUSEFs** | [`JakWai01/sile-fystem`](https://github.com/JakWai01/sile-fystem) | Generic FUSE implementation using any Afero backend. | 🔹 3rd Party | + +## Afero vs. `io/fs` (Go 1.16+) + +Go 1.16 introduced the `io/fs` package, which provides a standard abstraction for **read-only** filesystems. + +Afero complements `io/fs` by focusing on different needs: + +* **Use `io/fs` when:** You only need to read files and want to conform strictly to the standard library interfaces. +* **Use Afero when:** + * Your application needs to **create, write, modify, or delete** files. + * You need to test complex read/write interactions (e.g., renaming, concurrent writes). + * You need advanced compositional features (Copy-on-Write, Caching, etc.). + +Afero is fully compatible with `io/fs`. You can wrap any Afero filesystem to satisfy the `fs.FS` interface using `afero.NewIOFS`: -The Http package requires a slightly specific version of Open which -returns an http.File type. +```go +import "io/fs" -Afero provides an httpFs file system which satisfies this requirement. -Any Afero FileSystem can be used as an httpFs. +// Create an Afero filesystem (writable) +var myAferoFs afero.Fs = afero.NewMemMapFs() -```go -httpFs := afero.NewHttpFs() -fileserver := http.FileServer(httpFs.Dir()) -http.Handle("/", fileserver) +// Convert it to a standard library fs.FS (read-only view) +var myIoFs fs.FS = afero.NewIOFS(myAferoFs) ``` -## Composite Backends +## Third-Party Backends & Ecosystem -Afero provides the ability have two filesystems (or more) act as a single -file system. +The Afero community has developed numerous backends and tools that extend the library's capabilities. Below are curated, well-maintained options organized by maturity and reliability. -### CacheOnReadFs +### Featured Community Backends -The CacheOnReadFs will lazily make copies of any accessed files from the base -layer into the overlay. Subsequent reads will be pulled from the overlay -directly permitting the request is within the cache duration of when it was -created in the overlay. +These are mature, reliable backends that we can confidently recommend for production use: -If the base filesystem is writeable, any changes to files will be -done first to the base, then to the overlay layer. Write calls to open file -handles like `Write()` or `Truncate()` to the overlay first. +#### **Amazon S3** - [`fclairamb/afero-s3`](https://github.com/fclairamb/afero-s3) +Production-ready S3 backend built on the official AWS SDK for Go. -To writing files to the overlay only, you can use the overlay Fs directly (not -via the union Fs). +```go +import "github.com/fclairamb/afero-s3" -Cache files in the layer for the given time.Duration, a cache duration of 0 -means "forever" meaning the file will not be re-requested from the base ever. +s3fs := s3.NewFs(bucket, session) +``` -A read-only base will make the overlay also read-only but still copy files -from the base to the overlay when they're not present (or outdated) in the -caching layer. +#### **MinIO** - [`cpyun/afero-minio`](https://github.com/cpyun/afero-minio) +MinIO object storage backend providing S3-compatible object storage with deduplication and optimization features. ```go -base := afero.NewOsFs() -layer := afero.NewMemMapFs() -ufs := afero.NewCacheOnReadFs(base, layer, 100 * time.Second) -``` +import "github.com/cpyun/afero-minio" -### CopyOnWriteFs() +minioFs := miniofs.NewMinioFs(ctx, "minio://endpoint/bucket") +``` -The CopyOnWriteFs is a read only base file system with a potentially -writeable layer on top. +### Community & Specialized Backends -Read operations will first look in the overlay and if not found there, will -serve the file from the base. +#### Cloud Storage -Changes to the file system will only be made in the overlay. +- **Google Drive** - [`fclairamb/afero-gdrive`](https://github.com/fclairamb/afero-gdrive) + Streaming support; no write-seeking or POSIX permissions; no files listing cache -Any attempt to modify a file found only in the base will copy the file to the -overlay layer before modification (including opening a file with a writable -handle). +- **Dropbox** - [`fclairamb/afero-dropbox`](https://github.com/fclairamb/afero-dropbox) + Streaming support; no write-seeking or POSIX permissions -Removing and Renaming files present only in the base layer is not currently -permitted. If a file is present in the base layer and the overlay, only the -overlay will be removed/renamed. +#### Version Control Systems -```go - base := afero.NewOsFs() - roBase := afero.NewReadOnlyFs(base) - ufs := afero.NewCopyOnWriteFs(roBase, afero.NewMemMapFs()) +- **Git Repositories** - [`tobiash/go-gitfs`](https://github.com/tobiash/go-gitfs) + Read-only filesystem abstraction for Git repositories. Works with bare repositories and provides filesystem view of any git reference. Uses go-git for repository access. - fh, _ = ufs.Create("/home/test/file2.txt") - fh.WriteString("This is a test") - fh.Close() -``` +#### Container and Remote Systems -In this example all write operations will only occur in memory (MemMapFs) -leaving the base filesystem (OsFs) untouched. +- **Docker Containers** - [`unmango/aferox`](https://github.com/unmango/aferox) + Access Docker container filesystems as if they were local filesystems +- **GitHub API** - [`unmango/aferox`](https://github.com/unmango/aferox) + Turn GitHub repositories, releases, and assets into browsable filesystems -## Desired/possible backends +#### FUSE Integration -The following is a short list of possible backends we hope someone will -implement: +- **Generic FUSE** - [`JakWai01/sile-fystem`](https://github.com/JakWai01/sile-fystem) + Mount any Afero filesystem as a FUSE filesystem, allowing any Afero backend to be used as a real mounted filesystem -* SSH -* S3 +#### Specialized Filesystems -## Third-party library +- **FAT32 Support** - [`aligator/GoFAT`](https://github.com/aligator/GoFAT) + Pure Go FAT filesystem implementation (currently read-only) -- Alibaba Cloud OSS: [messikiller/afero-oss](https://github.com/messikiller/afero-oss) +### Interface Adapters & Utilities -# About the project +**Cross-Interface Compatibility:** +- [`jfontan/go-billy-desfacer`](https://github.com/jfontan/go-billy-desfacer) - Adapter between Afero and go-billy interfaces (for go-git compatibility) +- [`Maldris/go-billy-afero`](https://github.com/Maldris/go-billy-afero) - Alternative wrapper for using Afero with go-billy +- [`c4milo/afero2billy`](https://github.com/c4milo/afero2billy) - Another Afero to billy filesystem adapter -## What's in the name +**Working Directory Management:** +- [`carolynvs/aferox`](https://github.com/carolynvs/aferox) - Working directory-aware filesystem wrapper -Afero comes from the latin roots Ad-Facere. +**Advanced Filtering:** +- [`unmango/aferox`](https://github.com/unmango/aferox) includes multiple specialized filesystems: + - **FilterFs** - Predicate-based file filtering + - **IgnoreFs** - .gitignore-aware filtering + - **WriterFs** - Dump writes to io.Writer for debugging -**"Ad"** is a prefix meaning "to". +#### Developer Tools & Utilities -**"Facere"** is a form of the root "faciō" making "make or do". +**nhatthm Utility Suite** - Essential tools for Afero development: +- [`nhatthm/aferocopy`](https://github.com/nhatthm/aferocopy) - Copy files between any Afero filesystems +- [`nhatthm/aferomock`](https://github.com/nhatthm/aferomock) - Mocking toolkit for testing +- [`nhatthm/aferoassert`](https://github.com/nhatthm/aferoassert) - Assertion helpers for filesystem testing -The literal meaning of afero is "to make" or "to do" which seems very fitting -for a library that allows one to make files and directories and do things with them. +### Ecosystem Showcase -The English word that shares the same roots as Afero is "affair". Affair shares -the same concept but as a noun it means "something that is made or done" or "an -object of a particular type". +**Windows Virtual Drives** - [`balazsgrill/potatodrive`](https://github.com/balazsgrill/potatodrive) +Mount any Afero filesystem as a Windows drive letter. Brilliant demonstration of Afero's power! -It's also nice that unlike some of my other libraries (hugo, cobra, viper) it -Googles very well. +### Modern Asset Embedding (Go 1.16+) -## Release Notes +Instead of third-party tools, use Go's native `//go:embed` with Afero: -See the [Releases Page](https://github.com/spf13/afero/releases). +```go +import ( + "embed" + "github.com/spf13/afero" +) + +//go:embed assets/* +var assetsFS embed.FS + +func main() { + // Convert embedded files to Afero filesystem + fs := afero.FromIOFS(assetsFS) + + // Use like any other Afero filesystem + content, _ := afero.ReadFile(fs, "assets/config.json") +} +``` ## Contributing -1. Fork it +We welcome contributions! The project is mature, but we are actively looking for contributors to help implement and stabilize network/cloud backends. + +* 🔥 **Microsoft Azure Blob Storage** +* 🔒 **Modern Encryption Backend** - Built on secure, contemporary crypto (not legacy EncFS) +* 🐙 **Canonical go-git Adapter** - Unified solution for Git integration +* 📡 **SSH/SCP Backend** - Secure remote file operations +* Stabilization of existing experimental backends (GCS, SFTP) + +To contribute: +1. Fork the repository 2. Create your feature branch (`git checkout -b my-new-feature`) 3. Commit your changes (`git commit -am 'Add some feature'`) 4. Push to the branch (`git push origin my-new-feature`) -5. Create new Pull Request - -## Releasing - -As of version 1.14.0, Afero moved implementations with third-party libraries to -their own submodules. - -Releasing a new version now requires a few steps: - -``` -VERSION=X.Y.Z -git tag -a v$VERSION -m "Release $VERSION" -git push origin v$VERSION - -cd gcsfs -go get github.com/spf13/afero@v$VERSION -go mod tidy -git commit -am "Update afero to v$VERSION" -git tag -a gcsfs/v$VERSION -m "Release gcsfs $VERSION" -git push origin gcsfs/v$VERSION -cd .. - -cd sftpfs -go get github.com/spf13/afero@v$VERSION -go mod tidy -git commit -am "Update afero to v$VERSION" -git tag -a sftpfs/v$VERSION -m "Release sftpfs $VERSION" -git push origin sftpfs/v$VERSION -cd .. - -git push -``` +5. Create a new Pull Request -TODO: move these instructions to a Makefile or something +## 📄 License -## Contributors +Afero is released under the Apache 2.0 license. See [LICENSE.txt](https://github.com/spf13/afero/blob/master/LICENSE.txt) for details. -Names in no particular order: +## 🔗 Additional Resources -* [spf13](https://github.com/spf13) -* [jaqx0r](https://github.com/jaqx0r) -* [mbertschler](https://github.com/mbertschler) -* [xor-gate](https://github.com/xor-gate) +- [📖 Full API Documentation](https://pkg.go.dev/github.com/spf13/afero) +- [🎯 Examples Repository](https://github.com/spf13/afero/tree/master/examples) +- [📋 Release Notes](https://github.com/spf13/afero/releases) +- [❓ GitHub Discussions](https://github.com/spf13/afero/discussions) -## License +--- -Afero is released under the Apache 2.0 license. See -[LICENSE.txt](https://github.com/spf13/afero/blob/master/LICENSE.txt) +*Afero comes from the Latin roots Ad-Facere, meaning "to make" or "to do" - fitting for a library that empowers you to make and do amazing things with filesystems.* \ No newline at end of file From f40560b92cdee2b8da632209bd05b11ece21d5b3 Mon Sep 17 00:00:00 2001 From: Steve Francia Date: Mon, 4 Aug 2025 21:18:30 -0400 Subject: [PATCH 26/34] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5c1c17cb..179cf425 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # Afero: The Universal Filesystem Abstraction for Go [![GoDoc](https://godoc.org/github.com/spf13/afero?status.svg)](https://godoc.org/github.com/spf13/afero) -[![Build Status](https://github.com/spf13/afero/actions/workflows/test.yml/badge.svg)](https://github.com/spf13/afero/actions/workflows/test.yml) [![Go Report Card](https://goreportcard.com/badge/github.com/spf13/afero)](https://goreportcard.com/report/github.com/spf13/afero) +[📖 Full API Documentation](https://pkg.go.dev/github.com/spf13/afero) + Afero is a powerful and extensible filesystem abstraction system for Go. It provides a single, unified API for interacting with diverse filesystems—including the local disk, memory, archives, and network storage. @@ -466,4 +467,4 @@ Afero is released under the Apache 2.0 license. See [LICENSE.txt](https://github --- -*Afero comes from the Latin roots Ad-Facere, meaning "to make" or "to do" - fitting for a library that empowers you to make and do amazing things with filesystems.* \ No newline at end of file +*Afero comes from the Latin roots Ad-Facere, meaning "to make" or "to do" - fitting for a library that empowers you to make and do amazing things with filesystems.* From c7b2c2fcb7d7570723b99b9d316b50ac852f8bb4 Mon Sep 17 00:00:00 2001 From: Steve Francia Date: Mon, 4 Aug 2025 21:35:08 -0400 Subject: [PATCH 27/34] Update README.md with logo and badges --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 179cf425..ef67e9a7 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ -# Afero: The Universal Filesystem Abstraction for Go +afero logo-sm + -[![GoDoc](https://godoc.org/github.com/spf13/afero?status.svg)](https://godoc.org/github.com/spf13/afero) +[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/spf13/afero/ci.yaml?branch=master&style=flat-square)](https://github.com/spf13/afero/actions?query=workflow%3ACI) +[![GoDoc](https://pkg.go.dev/badge/mod/github.com/spf13/afero)](https://pkg.go.dev/mod/github.com/spf13/afero) [![Go Report Card](https://goreportcard.com/badge/github.com/spf13/afero)](https://goreportcard.com/report/github.com/spf13/afero) -[📖 Full API Documentation](https://pkg.go.dev/github.com/spf13/afero) +![Go Version](https://img.shields.io/badge/go%20version-%3E=1.23-61CFDD.svg?style=flat-square") +# Afero: The Universal Filesystem Abstraction for Go + Afero is a powerful and extensible filesystem abstraction system for Go. It provides a single, unified API for interacting with diverse filesystems—including the local disk, memory, archives, and network storage. Afero acts as a drop-in replacement for the standard `os` package, enabling you to write modular code that is agnostic to the underlying storage, dramatically simplifies testing, and allows for sophisticated architectural patterns through filesystem composition. From 008852502ed342837053831e451bc0649e26725c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:39:40 +0000 Subject: [PATCH 28/34] build(deps): bump github/codeql-action from 3.29.4 to 3.29.7 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.29.4 to 3.29.7. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/4e828ff8d448a8a6e532957b1811f387a63867e8...51f77329afa6477de8c49fc9c7046c15b9a4e79d) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.29.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/analysis-scorecard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml index 39508255..dd58b169 100644 --- a/.github/workflows/analysis-scorecard.yaml +++ b/.github/workflows/analysis-scorecard.yaml @@ -42,6 +42,6 @@ jobs: retention-days: 5 - name: Upload results to GitHub Security tab - uses: github/codeql-action/upload-sarif@4e828ff8d448a8a6e532957b1811f387a63867e8 # v3.29.4 + uses: github/codeql-action/upload-sarif@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.7 with: sarif_file: results.sarif From fd689a8e681ec626d678686fd90fd715639580c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Aug 2025 08:04:50 +0000 Subject: [PATCH 29/34] build(deps): bump actions/dependency-review-action from 4.7.1 to 4.7.3 Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 4.7.1 to 4.7.3. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/da24556b548a50705dd671f47852072ea4c105d9...595b5aeba73380359d98a5e087f648dbb0edce1b) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-version: 4.7.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6eb7d351..7f4486fd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -64,4 +64,4 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Dependency Review - uses: actions/dependency-review-action@da24556b548a50705dd671f47852072ea4c105d9 # v4.7.1 + uses: actions/dependency-review-action@595b5aeba73380359d98a5e087f648dbb0edce1b # v4.7.3 From 4b022993a0059b67d4aaeaa2cecd0cad84f938ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 20:23:57 +0000 Subject: [PATCH 30/34] build(deps): bump actions/setup-go from 5.5.0 to 6.0.0 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5.5.0 to 6.0.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/d35c59abb061a4a6fb18e82ac0862c26744d6ab5...44694675825211faa026b3c33043df3e48a5fa00) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6eb7d351..41d65f0d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Go - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 with: go-version: ${{ matrix.go }} @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Go - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 with: go-version: "1.24" From 9596fe84b763fd2e18267ac455a4cedba2655194 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 20:12:58 +0000 Subject: [PATCH 31/34] build(deps): bump github/codeql-action from 3.29.7 to 3.30.1 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.29.7 to 3.30.1. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/51f77329afa6477de8c49fc9c7046c15b9a4e79d...f1f6e5f6af878fb37288ce1c627459e94dbf7d01) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.30.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/analysis-scorecard.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml index dd58b169..35faafe8 100644 --- a/.github/workflows/analysis-scorecard.yaml +++ b/.github/workflows/analysis-scorecard.yaml @@ -42,6 +42,6 @@ jobs: retention-days: 5 - name: Upload results to GitHub Security tab - uses: github/codeql-action/upload-sarif@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.7 + uses: github/codeql-action/upload-sarif@f1f6e5f6af878fb37288ce1c627459e94dbf7d01 # v3.30.1 with: sarif_file: results.sarif From 41206fdfdacaad1dffaad870ded6f497ae1b803a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 16:14:36 +0000 Subject: [PATCH 32/34] build(deps): bump actions/checkout from 4.2.2 to 5.0.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.2 to 5.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/11bd71901bbe5b1630ceea73d27597364c9af683...08c6903cd8c0fde910a37f88322edcfb5dd907a8) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 5.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/analysis-scorecard.yaml | 2 +- .github/workflows/ci.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/analysis-scorecard.yaml b/.github/workflows/analysis-scorecard.yaml index 35faafe8..d3a90de8 100644 --- a/.github/workflows/analysis-scorecard.yaml +++ b/.github/workflows/analysis-scorecard.yaml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: persist-credentials: false diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ee914c6a..922f573f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Set up Go uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 @@ -42,7 +42,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Set up Go uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 @@ -61,7 +61,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Dependency Review uses: actions/dependency-review-action@595b5aeba73380359d98a5e087f648dbb0edce1b # v4.7.3 From c245c4fc3df2e427d681479553a625c5ef0e1eb8 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Mon, 8 Sep 2025 18:20:02 +0200 Subject: [PATCH 33/34] ci: update ci Signed-off-by: Mark Sagi-Kazar --- .github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 922f573f..7e8b149a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - go: ["1.23", "1.24"] + go: [stable, oldstable, "1.23", "1.24", "1.25"] steps: - name: Checkout repository @@ -47,12 +47,12 @@ jobs: - name: Set up Go uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 with: - go-version: "1.24" + go-version: "1.25" - name: Lint - uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd # v7.0.0 + uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0 with: - version: v2.0.2 + version: v2.4.0 dependency-review: name: Dependency review From f5f4f7bd6427212efca35481b3b8c749bbf06243 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Mon, 8 Sep 2025 18:21:11 +0200 Subject: [PATCH 34/34] chore: update deps Signed-off-by: Mark Sagi-Kazar --- gcsfs/go.mod | 4 ++-- gcsfs/go.sum | 8 ++++---- go.mod | 2 +- go.sum | 4 ++-- sftpfs/go.mod | 2 +- sftpfs/go.sum | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/gcsfs/go.mod b/gcsfs/go.mod index d076adc5..722b1b4c 100644 --- a/gcsfs/go.mod +++ b/gcsfs/go.mod @@ -45,9 +45,9 @@ require ( go.opentelemetry.io/otel/trace v1.34.0 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/net v0.37.0 // indirect - golang.org/x/sync v0.14.0 // indirect + golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/text v0.28.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect diff --git a/gcsfs/go.sum b/gcsfs/go.sum index d3217703..7037fb6d 100644 --- a/gcsfs/go.sum +++ b/gcsfs/go.sum @@ -95,12 +95,12 @@ golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= google.golang.org/api v0.226.0 h1:9A29y1XUD+YRXfnHkO66KggxHBZWg9LsTGqm7TkUvtQ= diff --git a/go.mod b/go.mod index 3259b2cd..f4ad62e7 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,4 @@ module github.com/spf13/afero go 1.23.0 -require golang.org/x/text v0.25.0 +require golang.org/x/text v0.28.0 diff --git a/go.sum b/go.sum index 3470e4e3..433ec670 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,2 @@ -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= diff --git a/sftpfs/go.mod b/sftpfs/go.mod index 19dcfc2e..720fb871 100644 --- a/sftpfs/go.mod +++ b/sftpfs/go.mod @@ -13,5 +13,5 @@ require ( require ( github.com/kr/fs v0.1.0 // indirect golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/text v0.28.0 // indirect ) diff --git a/sftpfs/go.sum b/sftpfs/go.sum index 6f8e2a67..0abc7d5e 100644 --- a/sftpfs/go.sum +++ b/sftpfs/go.sum @@ -75,8 +75,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=