Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Fix: Don't bust cache so aggressively when compiling test sources #1110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Fix: Don't bust cache so aggressively when compiling test sources #1110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 8 additions & 9 deletions build/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,13 +458,12 @@ func NewSession(options *Options) (*Session, error) {
}
s.xctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags)
s.buildCache = cache.BuildCache{
GOOS: s.xctx.GOOS(),
GOARCH: "js",
GOROOT: options.GOROOT,
GOPATH: options.GOPATH,
BuildTags: options.BuildTags,
Minify: options.Minify,
TestedPackage: options.TestedPackage,
Copy link
Author

@regenvanwalbeek-wf regenvanwalbeek-wf Mar 22, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One thing that's not clear to me is if we should share a cache between gopherjs test and gopherjs build.

For example, let's say foo depends on shared, and you run gopherjs test foo/.... This will create a cache for shared. If you run a subsequent gopherjs build, is it okay to reuse the cache for shared?

I expect this would be fine as long as the rest of the BuildCache args are the same.

GOOS: s.xctx.GOOS(),
GOARCH: "js",
GOROOT: options.GOROOT,
GOPATH: options.GOPATH,
BuildTags: options.BuildTags,
Minify: options.Minify,
}
s.Types = make(map[string]*types.Package)
if options.Watch {
Expand Down Expand Up @@ -608,7 +607,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
}

if !s.options.NoCache {
archive := s.buildCache.LoadArchive(pkg.ImportPath)
archive := s.buildCache.LoadArchive(pkg.ImportPath, s.options.TestedPackage)
if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) {
if err := archive.RegisterTypes(s.Types); err != nil {
panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err))
Expand Down Expand Up @@ -649,7 +648,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
fmt.Println(pkg.ImportPath)
}

s.buildCache.StoreArchive(archive)
s.buildCache.StoreArchive(archive, s.options.TestedPackage)
s.UpToDateArchives[pkg.ImportPath] = archive

return archive, nil
Expand Down
24 changes: 12 additions & 12 deletions build/cache/cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func Clear() error {
// the cache. For example, any artifacts that were cached for a minified build
// must not be reused for a non-minified build. GopherJS version change also
// invalidates the cache. It is callers responsibility to ensure that artifacts
// passed the the StoreArchive function were generated with the same build
// passed to the StoreArchive function were generated with the same build
// parameters as the cache is configured.
//
// There is no upper limit for the total cache size. It can be cleared
Expand All @@ -77,11 +77,6 @@ type BuildCache struct {
GOPATH string
BuildTags []string
Minify bool
// When building for tests, import path of the package being tested. The
// package under test is built with *_test.go sources included, and since it
// may be imported by other packages in the binary we can't reuse the "normal"
// cache.
TestedPackage string
}

func (bc BuildCache) String() string {
Expand All @@ -90,11 +85,11 @@ func (bc BuildCache) String() string {

// StoreArchive compiled archive in the cache. Any error inside this method
// will cause the cache not to be persisted.
func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
func (bc *BuildCache) StoreArchive(a *compiler.Archive, testedPackage string) {
if bc == nil {
return // Caching is disabled.
}
path := cachedPath(bc.archiveKey(a.ImportPath))
path := cachedPath(bc.archiveKey(a.ImportPath, testedPackage))
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
log.Warningf("Failed to create build cache directory: %v", err)
return
Expand Down Expand Up @@ -125,11 +120,11 @@ func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
//
// The returned archive would have been built with the same configuration as
// the build cache was.
func (bc *BuildCache) LoadArchive(importPath string) *compiler.Archive {
func (bc *BuildCache) LoadArchive(importPath string, testedPackage string) *compiler.Archive {
if bc == nil {
return nil // Caching is disabled.
}
path := cachedPath(bc.archiveKey(importPath))
path := cachedPath(bc.archiveKey(importPath, testedPackage))
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
Expand All @@ -156,6 +151,11 @@ func (bc *BuildCache) commonKey() string {
}

// archiveKey returns a full cache key for a package's compiled archive.
func (bc *BuildCache) archiveKey(importPath string) string {
return path.Join("archive", bc.commonKey(), importPath)
func (bc *BuildCache) archiveKey(importPath string, testedPackage string) string {
// When building for tests, testedPackage is import path of the package being tested.
// testedPackage is built with *_test.go sources included, and since it
// may be imported by other packages in the binary we can't reuse the "normal"
// cache.
compiledWithTests := testedPackage == importPath
return path.Join("archive", bc.commonKey(), importPath, fmt.Sprintf("compiledWithTests: %v", compiledWithTests))
}
56 changes: 49 additions & 7 deletions build/cache/cache_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,20 @@ func TestStore(t *testing.T) {
}

bc := BuildCache{}
if got := bc.LoadArchive(want.ImportPath); got != nil {
if got := bc.LoadArchive(want.ImportPath, ""); got != nil {
t.Errorf("Got: %s was found in the cache. Want: empty cache.", got.ImportPath)
}
bc.StoreArchive(want)
got := bc.LoadArchive(want.ImportPath)
bc.StoreArchive(want, "")
got := bc.LoadArchive(want.ImportPath, "")
if got == nil {
t.Errorf("Got: %s wan not found in the cache. Want: archive is can be loaded after store.", want.ImportPath)
t.Errorf("Got: %s was not found in the cache. Want: archive can be loaded after store.", want.ImportPath)
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("Loaded archive is different from stored (-want,+got):\n%s", diff)
}

// Make sure the package names are a part of the cache key.
if got := bc.LoadArchive("fake/other"); got != nil {
if got := bc.LoadArchive("fake/other", ""); got != nil {
t.Errorf("Got: fake/other was found in cache: %#v. Want: nil for packages that weren't cached.", got)
}
}
Expand Down Expand Up @@ -61,15 +61,57 @@ func TestInvalidation(t *testing.T) {

for _, test := range tests {
a := &compiler.Archive{ImportPath: "package/fake"}
test.cache1.StoreArchive(a)
test.cache1.StoreArchive(a, "")

if got := test.cache2.LoadArchive(a.ImportPath); got != nil {
if got := test.cache2.LoadArchive(a.ImportPath, ""); got != nil {
t.Logf("-cache1,+cache2:\n%s", cmp.Diff(test.cache1, test.cache2))
t.Errorf("Got: %v loaded from cache. Want: build parameter change invalidates cache.", got)
}
}
}

func TestDoNotReuseCacheWhenLoadingPackageForTest(t *testing.T) {
cacheForTest(t)

sharedPkgArchive := &compiler.Archive{
ImportPath: "fake/sharedTestPkg",
}

bc := BuildCache{}
if got := bc.LoadArchive(sharedPkgArchive.ImportPath, "fake/pkg1"); got != nil {
t.Errorf("Got: %s was found in the cache. Want: empty cache.", got.ImportPath)
}
bc.StoreArchive(sharedPkgArchive, "fake/package_1")

// sharedPkgArchive has not been stored with *_test.go sources, so it cannot be reused
if got := bc.LoadArchive(sharedPkgArchive.ImportPath, sharedPkgArchive.ImportPath); got != nil {
t.Errorf("Got: %s was found in the cache. Want: empty cache.", got.ImportPath)
}
}

func TestDifferentSourcesCanShareSameArchive(t *testing.T) {
cacheForTest(t)

sharedPkgArchive := &compiler.Archive{
ImportPath: "fake/sharedTestPkg",
}

bc := BuildCache{}
if got := bc.LoadArchive(sharedPkgArchive.ImportPath, "fake/pkg1"); got != nil {
t.Errorf("Got: %s was found in the cache. Want: empty cache.", got.ImportPath)
}
bc.StoreArchive(sharedPkgArchive, "fake/package_1")

// sharedPkgArchive has been stored without *_test.go sources, so it can be reused
if got := bc.LoadArchive(sharedPkgArchive.ImportPath, "fake/pkg2"); got == nil {
t.Errorf("Got: %s was not found in the cache. Want: archive can be loaded after store.", sharedPkgArchive.ImportPath)
}

if got := bc.LoadArchive(sharedPkgArchive.ImportPath, ""); got == nil {
t.Errorf("Got: %s was not found in the cache. Want: archive can be loaded after store.", sharedPkgArchive.ImportPath)
}
}

func cacheForTest(t *testing.T) {
t.Helper()
originalRoot := cacheRoot
Expand Down