Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5c8416e

Browse files
nevkontaktenevkontakte
committed
Build sequence and caching overhaul.
The biggest change in this commit is a rewrite of build cache machinery. Previously GopherJS used to store its build cache under $GOPATH/pkg/$GOOS_$GOARCH, and after Go Modules introduction some parts of it were stored under os.UserCacheDir(). Starting from this commit, *all* build cache is located under os.UserCacheDir(), in a manner similar to the modern Go tool. The cache is keyed by a set of build options (such as minification, compiler version, etc.) to ensure that incompatible archives aren't picked up (see gopherjs#440 for example). This change doesn't solve *all* possible cache-related issues (for example, it still relies on timestamps to invalidate the cache, see gopherjs#805), but should eliminate a large class of confusing failure modes. The second important change is to the build order. Previously GopherJS could initiate build of an imported dependency while building the package that imported it (typically for an augmented stdlib package). Now the build is stricter, all imports are built upfront, preventing weird cyclic builds that could happened under the previous system. Lastly, package source modification time has been refactored in a way that doesn't require modification of Package.SrcModTime after the package was loaded. The old side-effect-based behavior was difficult to get right and debug.
1 parent 15ed2e8 commit 5c8416e

File tree

13 files changed

+470
-231
lines changed

13 files changed

+470
-231
lines changed

‎build/build.go‎

Lines changed: 148 additions & 143 deletions
Large diffs are not rendered by default.

‎build/cache.go‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎build/cache/cache.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
// Package cache solves one of the hardest computer science problems in
2+
// application to GopherJS compiler outputs.
3+
package cache
4+
5+
import (
6+
"crypto/sha256"
7+
"fmt"
8+
"go/build"
9+
"os"
10+
"path"
11+
"path/filepath"
12+
13+
"github.com/gopherjs/gopherjs/compiler"
14+
)
15+
16+
// cacheRoot is the base path for GopherJS's own build cache.
17+
//
18+
// It serves a similar function to the Go build cache, but is a lot more
19+
// simplistic and therefore not compatible with Go. We use this cache directory
20+
// to store build artifacts for packages loaded from a module, for which PkgObj
21+
// provided by go/build points inside the module source tree, which can cause
22+
// inconvenience with version control, etc.
23+
var cacheRoot = func() string {
24+
path, err := os.UserCacheDir()
25+
if err == nil {
26+
return filepath.Join(path, "gopherjs", "build_cache")
27+
}
28+
29+
return filepath.Join(build.Default.GOPATH, "pkg", "gopherjs_build_cache")
30+
}()
31+
32+
// cachedPath returns a location inside the build cache for a given set of key
33+
// strings. The set of keys must uniquely identify cacheable object. Prefer
34+
// using more specific functions to ensure key consistency.
35+
func cachedPath(keys ...string) string {
36+
key := path.Join(keys...)
37+
if key == "" {
38+
panic("CachedPath() must not be used with an empty string")
39+
}
40+
sum := fmt.Sprintf("%x", sha256.Sum256([]byte(key)))
41+
return filepath.Join(cacheRoot, sum[0:2], sum)
42+
}
43+
44+
// Clear the cache. This will remove *all* cached artifacts from *all* build
45+
// configurations.
46+
func Clear() error {
47+
return os.RemoveAll(cacheRoot)
48+
}
49+
50+
// BuildCache manages build artifacts that are cached for incremental builds.
51+
//
52+
// Cache is designed to be non-durable: any store and load errors are swallowed
53+
// and simply lead to a cache miss. The caller must be able to handle cache
54+
// misses. Nil pointer to BuildCache is valid and simply disables caching.
55+
//
56+
// BuildCache struct fields represent build parameters which change invalidates
57+
// the cache. For example, any artifacts that were cached for a minified build
58+
// must not be reused for a non-minified build. GopherJS version change also
59+
// invalidates the cache. It is callers responsibility to ensure that artifacts
60+
// passed the the StoreArchive function were generated with the same build
61+
// parameters as the cache is configured.
62+
//
63+
// There is no upper limit for the total cache size. It can be cleared
64+
// programmatically via the Clear() function, or the user can just delete the
65+
// directory if it grows too big.
66+
//
67+
// TODO(nevkontakte): changes in the input sources or dependencies doesn't
68+
// currently invalidate the cache. This is handled at the higher level by
69+
// checking cached archive timestamp against loaded package modification time.
70+
//
71+
// TODO(nevkontakte): this cache could benefit from checksum integrity checks.
72+
type BuildCache struct {
73+
GOOS string
74+
GOARCH string
75+
GOROOT string
76+
GOPATH string
77+
BuildTags []string
78+
Minify bool
79+
}
80+
81+
func (bc BuildCache) String() string {
82+
return fmt.Sprintf("%#v", bc)
83+
}
84+
85+
// StoreArchive compiled archive in the cache. Any error inside this method
86+
// will cause the cache not to be persisted.
87+
func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
88+
if bc == nil {
89+
return // Caching is disabled.
90+
}
91+
path := cachedPath(bc.archiveKey(a.ImportPath))
92+
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
93+
return
94+
}
95+
f, err := os.Create(path)
96+
if err != nil {
97+
return
98+
}
99+
defer f.Close()
100+
if err := compiler.WriteArchive(a, f); err != nil {
101+
// Make sure we don't leave a half-written archive behind.
102+
os.Remove(path)
103+
}
104+
}
105+
106+
// LoadArchive returns a previously cached archive of the given package or nil
107+
// if it wasn't previously stored.
108+
//
109+
// The returned archive would have been built with the same configuration as
110+
// the build cache was.
111+
func (bc *BuildCache) LoadArchive(importPath string) *compiler.Archive {
112+
if bc == nil {
113+
return nil // Caching is disabled.
114+
}
115+
path := cachedPath(bc.archiveKey(importPath))
116+
f, err := os.Open(path)
117+
if err != nil {
118+
return nil // Cache miss.
119+
}
120+
defer f.Close()
121+
a, err := compiler.ReadArchive(importPath, f)
122+
if err != nil {
123+
return nil // Invalid/corrupted archive, cache miss.
124+
}
125+
return a
126+
}
127+
128+
// commonKey returns a part of the cache key common for all artifacts generated
129+
// under a given BuildCache configuration.
130+
func (bc *BuildCache) commonKey() string {
131+
return fmt.Sprintf("%#v + %v", *bc, compiler.Version)
132+
}
133+
134+
// archiveKey returns a full cache key for a package's compiled archive.
135+
func (bc *BuildCache) archiveKey(importPath string) string {
136+
return path.Join("archive", bc.commonKey(), importPath)
137+
}

‎build/cache/cache_test.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package cache
2+
3+
import (
4+
"testing"
5+
6+
"github.com/google/go-cmp/cmp"
7+
"github.com/gopherjs/gopherjs/compiler"
8+
)
9+
10+
func TestStore(t *testing.T) {
11+
cacheForTest(t)
12+
13+
want := &compiler.Archive{
14+
ImportPath: "fake/package",
15+
Imports: []string{"fake/dep"},
16+
}
17+
18+
bc := BuildCache{}
19+
if got := bc.LoadArchive(want.ImportPath); got != nil {
20+
t.Errorf("Got: %s was found in the cache. Want: empty cache.", got.ImportPath)
21+
}
22+
bc.StoreArchive(want)
23+
got := bc.LoadArchive(want.ImportPath)
24+
if got == nil {
25+
t.Errorf("Got: %s wan not found in the cache. Want: archive is can be loaded after store.", want.ImportPath)
26+
}
27+
if diff := cmp.Diff(want, got); diff != "" {
28+
t.Errorf("Loaded archive is different from stored (-want,+got):\n%s", diff)
29+
}
30+
31+
// Make sure the package names are a part of the cache key.
32+
if got := bc.LoadArchive("fake/other"); got != nil {
33+
t.Errorf("Got: fake/other was found in cache: %#v. Want: nil for packages that weren't cached.", got)
34+
}
35+
}
36+
37+
func TestInvalidation(t *testing.T) {
38+
cacheForTest(t)
39+
40+
tests := []struct {
41+
cache1 BuildCache
42+
cache2 BuildCache
43+
}{
44+
{
45+
cache1: BuildCache{Minify: true},
46+
cache2: BuildCache{Minify: false},
47+
}, {
48+
cache1: BuildCache{GOOS: "dos"},
49+
cache2: BuildCache{GOOS: "amiga"},
50+
}, {
51+
cache1: BuildCache{GOARCH: "m68k"},
52+
cache2: BuildCache{GOARCH: "mos6502"},
53+
}, {
54+
cache1: BuildCache{GOROOT: "here"},
55+
cache2: BuildCache{GOROOT: "there"},
56+
}, {
57+
cache1: BuildCache{GOPATH: "home"},
58+
cache2: BuildCache{GOPATH: "away"},
59+
},
60+
}
61+
62+
for _, test := range tests {
63+
a := &compiler.Archive{ImportPath: "package/fake"}
64+
test.cache1.StoreArchive(a)
65+
66+
if got := test.cache2.LoadArchive(a.ImportPath); got != nil {
67+
t.Logf("-cache1,+cache2:\n%s", cmp.Diff(test.cache1, test.cache2))
68+
t.Errorf("Got: %v loaded from cache. Want: build parameter change invalidates cache.", got)
69+
}
70+
}
71+
}
72+
73+
func cacheForTest(t *testing.T) {
74+
t.Helper()
75+
originalRoot := cacheRoot
76+
t.Cleanup(func() { cacheRoot = originalRoot })
77+
cacheRoot = t.TempDir()
78+
}

‎build/context.go‎

Lines changed: 11 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"go/build"
66
"go/token"
7+
"io/fs"
78
"net/http"
89
"os"
910
"os/exec"
@@ -36,8 +37,11 @@ type XContext interface {
3637
// simpleCtx is a wrapper around go/build.Context with support for GopherJS-specific
3738
// features.
3839
type simpleCtx struct {
39-
bctx build.Context
40-
isVirtual bool // Imported packages don't have a physical directory on disk.
40+
bctx build.Context
41+
// If not nil, used instead of os.Stat().
42+
stat func(path string) (fs.FileInfo, error)
43+
// Imported packages don't have a physical directory on disk.
44+
isVirtual bool
4145
}
4246

4347
// Import implements XContext.Import().
@@ -51,7 +55,6 @@ func (sc simpleCtx) Import(importPath string, srcDir string, mode build.ImportMo
5155
if err != nil {
5256
return nil, fmt.Errorf("failed to enumerate .inc.js files in %s: %w", pkg.Dir, err)
5357
}
54-
pkg.PkgObj = sc.rewritePkgObj(pkg.PkgObj)
5558
if !path.IsAbs(pkg.Dir) {
5659
pkg.Dir = mustAbs(pkg.Dir)
5760
}
@@ -61,12 +64,14 @@ func (sc simpleCtx) Import(importPath string, srcDir string, mode build.ImportMo
6164
return nil, &ImportCError{pkg.ImportPath}
6265
}
6366

64-
return &PackageData{
67+
pkgData := &PackageData{
6568
Package: pkg,
6669
IsVirtual: sc.isVirtual,
6770
JSFiles: jsFiles,
6871
bctx: &sc.bctx,
69-
}, nil
72+
}
73+
pkgData.SrcModTime = pkgModTime(pkgData, sc.stat)
74+
return pkgData, nil
7075
}
7176

7277
// Match implements XContext.Match.
@@ -252,33 +257,6 @@ func (sc simpleCtx) applyPostloadTweaks(pkg *build.Package) *build.Package {
252257
return pkg
253258
}
254259

255-
func (sc simpleCtx) rewritePkgObj(orig string) string {
256-
if orig == "" {
257-
return orig
258-
}
259-
260-
goroot := mustAbs(sc.bctx.GOROOT)
261-
gopath := mustAbs(sc.bctx.GOPATH)
262-
orig = mustAbs(orig)
263-
264-
if strings.HasPrefix(orig, filepath.Join(gopath, "pkg", "mod")) {
265-
// Go toolchain makes sources under GOPATH/pkg/mod readonly, so we can't
266-
// store our artifacts there.
267-
return cachedPath(orig)
268-
}
269-
270-
allowed := []string{goroot, gopath}
271-
for _, prefix := range allowed {
272-
if strings.HasPrefix(orig, prefix) {
273-
// Traditional GOPATH-style locations for build artifacts are ok to use.
274-
return orig
275-
}
276-
}
277-
278-
// Everything else also goes into the cache just in case.
279-
return cachedPath(orig)
280-
}
281-
282260
var defaultBuildTags = []string{
283261
"netgo", // See https://godoc.org/net#hdr-Name_Resolution.
284262
"purego", // See https://golang.org/issues/23172.
@@ -302,6 +280,7 @@ func embeddedCtx(embedded http.FileSystem, installSuffix string, buildTags []str
302280
ec.bctx.IsDir = fs.IsDir
303281
ec.bctx.ReadDir = fs.ReadDir
304282
ec.bctx.OpenFile = fs.OpenFile
283+
ec.stat = fs.Stat
305284
ec.isVirtual = true
306285
return ec
307286
}

0 commit comments

Comments
 (0)