Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 60cb860

Browse files
nevkontaktenevkontakte
committed
Improved build cache.
Previously GopherJS used to store its build cache under $GOPATH/pkg/$GOOS_$GOARCH, and after Go Modules introduction some parts of it were stored under os.UserCacheDir(). Starting from this commit, *all* build cache is located under os.UserCacheDir(), in a manner similar to the modern Go tool. The cache is keyed by a set of build options (such as minification, compiler version, etc.) to ensure that incompatible archives aren't picked up (see gopherjs#440 for example). This change doesn't solve *all* possible cache-related issues (for example, it still relies on timestamps to invalidate the cache, see gopherjs#805), but should eliminate a large class of confusing failure modes.
1 parent 15ed2e8 commit 60cb860

File tree

9 files changed

+374
-203
lines changed

9 files changed

+374
-203
lines changed

‎build/build.go‎

Lines changed: 124 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/shurcooL/httpfs/vfsutil"
3232
"golang.org/x/tools/go/buildutil"
3333

34+
"github.com/gopherjs/gopherjs/build/cache"
3435
_ "github.com/gopherjs/gopherjs/build/versionhack" // go/build release tags hack.
3536
)
3637

@@ -117,19 +118,6 @@ func importWithSrcDir(xctx XContext, path string, srcDir string, mode build.Impo
117118
return nil, err
118119
}
119120

120-
if pkg.IsCommand() {
121-
pkg.PkgObj = filepath.Join(pkg.BinDir, filepath.Base(pkg.ImportPath)+".js")
122-
}
123-
124-
if _, err := os.Stat(pkg.PkgObj); os.IsNotExist(err) && strings.HasPrefix(pkg.PkgObj, DefaultGOROOT) {
125-
// fall back to GOPATH
126-
firstGopathWorkspace := filepath.SplitList(build.Default.GOPATH)[0] // TODO: Need to check inside all GOPATH workspaces.
127-
gopathPkgObj := filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(DefaultGOROOT):])
128-
if _, err := os.Stat(gopathPkgObj); err == nil {
129-
pkg.PkgObj = gopathPkgObj
130-
}
131-
}
132-
133121
return pkg, nil
134122
}
135123

@@ -374,11 +362,18 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) {
374362
// GopherJS requires.
375363
type PackageData struct {
376364
*build.Package
377-
JSFiles []string
378-
IsTest bool // IsTest is true if the package is being built for running tests.
365+
JSFiles []string
366+
// IsTest is true if the package is being built for running tests.
367+
IsTest bool
379368
SrcModTime time.Time
380369
UpToDate bool
381-
IsVirtual bool // If true, the package does not have a corresponding physical directory on disk.
370+
// If true, the package does not have a corresponding physical directory on disk.
371+
IsVirtual bool
372+
// When importing a package present as a key in this map, will import the
373+
// package by the override package instead. This is primarily intended to
374+
// support importing "test" variants of packages from the test main and
375+
// "external test" packages.
376+
ImportOverrides map[string]string
382377

383378
bctx *build.Context // The original build context this package came from.
384379
}
@@ -395,7 +390,7 @@ func (p *PackageData) InternalBuildContext() *build.Context {
395390
func (p *PackageData) TestPackage() *PackageData {
396391
return &PackageData{
397392
Package: &build.Package{
398-
ImportPath: p.ImportPath,
393+
ImportPath: p.ImportPath + ".text",
399394
Dir: p.Dir,
400395
GoFiles: append(p.GoFiles, p.TestGoFiles...),
401396
Imports: append(p.Imports, p.TestImports...),
@@ -410,26 +405,51 @@ func (p *PackageData) TestPackage() *PackageData {
410405
func (p *PackageData) XTestPackage() *PackageData {
411406
return &PackageData{
412407
Package: &build.Package{
413-
ImportPath: p.ImportPath + "_test",
408+
ImportPath: p.ImportPath + ".xtest",
414409
Dir: p.Dir,
415410
GoFiles: p.XTestGoFiles,
416411
Imports: p.XTestImports,
417412
},
418413
IsTest: true,
419-
bctx: p.bctx,
414+
ImportOverrides: map[string]string{
415+
// Import the tested package variant built with _test.go sources included.
416+
p.ImportPath: p.ImportPath + ".test",
417+
},
418+
bctx: p.bctx,
419+
}
420+
}
421+
422+
// InstallPath returns the path where "gopherjs install" command should place the
423+
// generated output.
424+
func (p *PackageData) InstallPath() string {
425+
if p.IsCommand() {
426+
name := filepath.Base(p.ImportPath) + ".js"
427+
// For executable packages, mimic go tool behavior if possible.
428+
if gobin := os.Getenv("GOBIN"); gobin != "" {
429+
return filepath.Join(gobin, name)
430+
} else if gopath := os.Getenv("GOPATH"); gopath != "" {
431+
return filepath.Join(gopath, "bin", name)
432+
} else if home, err := os.UserHomeDir(); err == nil {
433+
return filepath.Join(home, "go", "bin", name)
434+
}
420435
}
436+
return p.PkgObj
421437
}
422438

423439
// Session manages internal state GopherJS requires to perform a build.
424440
//
425441
// This is the main interface to GopherJS build system. Session lifetime is
426442
// roughly equivalent to a single GopherJS tool invocation.
427443
type Session struct {
428-
options *Options
429-
xctx XContext
430-
Archives map[string]*compiler.Archive
431-
Types map[string]*types.Package
432-
Watcher *fsnotify.Watcher
444+
options *Options
445+
xctx XContext
446+
buildCache cache.BuildCache
447+
// Binary archives produced during the current session and assumed to be
448+
// up to date with input sources and dependencies. In the -w ("watch") mode
449+
// must be cleared upon entering watching.
450+
UpToDateArchives map[string]*compiler.Archive
451+
Types map[string]*types.Package
452+
Watcher *fsnotify.Watcher
433453
}
434454

435455
// NewSession creates a new GopherJS build session.
@@ -448,10 +468,18 @@ func NewSession(options *Options) (*Session, error) {
448468
}
449469

450470
s := &Session{
451-
options: options,
452-
Archives: make(map[string]*compiler.Archive),
471+
options: options,
472+
UpToDateArchives: make(map[string]*compiler.Archive),
453473
}
454474
s.xctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags)
475+
s.buildCache = cache.BuildCache{
476+
GOOS: s.xctx.GOOS(),
477+
GOARCH: "js",
478+
GOROOT: options.GOROOT,
479+
GOPATH: options.GOPATH,
480+
BuildTags: options.BuildTags,
481+
Minify: options.Minify,
482+
}
455483
s.Types = make(map[string]*types.Package)
456484
if options.Watch {
457485
if out, err := exec.Command("ulimit", "-n").Output(); err == nil {
@@ -493,10 +521,13 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath stri
493521
pkg := &PackageData{
494522
Package: &build.Package{
495523
Name: "main",
496-
ImportPath: "main",
524+
ImportPath: "command-line-arguments",
497525
Dir: packagePath,
498526
},
499-
bctx: &goCtx(s.InstallSuffix(), s.options.BuildTags).bctx,
527+
// This ephemeral package doesn't have a unique import path to be used as a
528+
// build cache key, so we never cache it.
529+
SrcModTime: time.Now().Add(time.Hour),
530+
bctx: &goCtx(s.InstallSuffix(), s.options.BuildTags).bctx,
500531
}
501532

502533
for _, file := range filenames {
@@ -511,7 +542,7 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath stri
511542
if err != nil {
512543
return err
513544
}
514-
if s.Types["main"].Name() != "main" {
545+
if s.Types["command-line-arguments"].Name() != "main" {
515546
return fmt.Errorf("cannot build/run non-main package")
516547
}
517548
return s.WriteCommandPackage(archive, pkgObj)
@@ -548,94 +579,69 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag
548579

549580
// BuildPackage compiles an already loaded package.
550581
func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
551-
if archive, ok := s.Archives[pkg.ImportPath]; ok {
582+
if archive, ok := s.UpToDateArchives[pkg.ImportPath]; ok {
552583
return archive, nil
553584
}
554585

555-
if pkg.PkgObj != "" {
556-
var fileInfo os.FileInfo
557-
gopherjsBinary, err := os.Executable()
586+
var fileInfo os.FileInfo
587+
gopherjsBinary, err := os.Executable()
588+
if err == nil {
589+
fileInfo, err = os.Stat(gopherjsBinary)
558590
if err == nil {
559-
fileInfo, err = os.Stat(gopherjsBinary)
560-
if err == nil {
561-
pkg.SrcModTime = fileInfo.ModTime()
562-
}
591+
pkg.SrcModTime = fileInfo.ModTime()
563592
}
593+
}
594+
if err != nil {
595+
os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
596+
pkg.SrcModTime = time.Now()
597+
}
598+
599+
for _, importedPkgPath := range pkg.Imports {
600+
if importedPkgPath == "unsafe" {
601+
continue
602+
}
603+
importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
564604
if err != nil {
565-
os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
566-
pkg.SrcModTime = time.Now()
605+
return nil, err
567606
}
568-
569-
for _, importedPkgPath := range pkg.Imports {
570-
if importedPkgPath == "unsafe" {
571-
continue
572-
}
573-
importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
574-
if err != nil {
575-
return nil, err
576-
}
577-
impModTime := importedPkg.SrcModTime
578-
if impModTime.After(pkg.SrcModTime) {
579-
pkg.SrcModTime = impModTime
580-
}
607+
impModTime := importedPkg.SrcModTime
608+
if impModTime.After(pkg.SrcModTime) {
609+
pkg.SrcModTime = impModTime
581610
}
611+
}
582612

583-
for _, name := range append(pkg.GoFiles, pkg.JSFiles...) {
584-
fileInfo, err := statFile(filepath.Join(pkg.Dir, name))
585-
if err != nil {
586-
return nil, err
587-
}
588-
if fileInfo.ModTime().After(pkg.SrcModTime) {
589-
pkg.SrcModTime = fileInfo.ModTime()
590-
}
613+
for _, name := range append(pkg.GoFiles, pkg.JSFiles...) {
614+
fileInfo, err := statFile(filepath.Join(pkg.Dir, name))
615+
if err != nil {
616+
return nil, err
591617
}
618+
if fileInfo.ModTime().After(pkg.SrcModTime) {
619+
pkg.SrcModTime = fileInfo.ModTime()
620+
}
621+
}
592622

593-
pkgObjFileInfo, err := os.Stat(pkg.PkgObj)
594-
if err == nil && !pkg.SrcModTime.After(pkgObjFileInfo.ModTime()) {
595-
// package object is up to date, load from disk if library
596-
pkg.UpToDate = true
597-
if pkg.IsCommand() {
598-
return nil, nil
599-
}
600-
601-
objFile, err := os.Open(pkg.PkgObj)
602-
if err != nil {
603-
return nil, err
604-
}
605-
defer objFile.Close()
606-
607-
archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types)
608-
if err != nil {
609-
return nil, err
610-
}
611-
612-
s.Archives[pkg.ImportPath] = archive
613-
return archive, err
623+
archive := s.buildCache.LoadArchive(pkg.ImportPath)
624+
if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) {
625+
if err := archive.RegisterTypes(s.Types); err != nil {
626+
panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err))
614627
}
628+
s.UpToDateArchives[pkg.ImportPath] = archive
629+
// Existing archive is up to date, no need to build it from scratch.
630+
return archive, nil
615631
}
616632

633+
// Existing archive is out of date or doesn't exist, let's build the package.
617634
fileSet := token.NewFileSet()
618635
files, err := parseAndAugment(s.xctx, pkg, pkg.IsTest, fileSet)
619636
if err != nil {
620637
return nil, err
621638
}
622639

623-
localImportPathCache := make(map[string]*compiler.Archive)
624640
importContext := &compiler.ImportContext{
625641
Packages: s.Types,
626-
Import: func(path string) (*compiler.Archive, error) {
627-
if archive, ok := localImportPathCache[path]; ok {
628-
return archive, nil
629-
}
630-
_, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir)
631-
if err != nil {
632-
return nil, err
633-
}
634-
localImportPathCache[path] = archive
635-
return archive, nil
636-
},
642+
Import: s.ImportResolverFor(pkg),
637643
}
638-
archive, err := compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
644+
archive, err = compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
639645
if err != nil {
640646
return nil, err
641647
}
@@ -654,40 +660,30 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
654660
fmt.Println(pkg.ImportPath)
655661
}
656662

657-
s.Archives[pkg.ImportPath] = archive
663+
s.buildCache.StoreArchive(archive)
664+
s.UpToDateArchives[pkg.ImportPath] = archive
658665

659-
if pkg.PkgObj == "" || pkg.IsCommand() {
660-
return archive, nil
661-
}
666+
return archive, nil
667+
}
662668

663-
if err := s.writeLibraryPackage(archive, pkg.PkgObj); err != nil {
664-
if strings.HasPrefix(pkg.PkgObj, s.options.GOROOT) {
665-
// fall back to first GOPATH workspace
666-
firstGopathWorkspace := filepath.SplitList(s.options.GOPATH)[0]
667-
if err := s.writeLibraryPackage(archive, filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(s.options.GOROOT):])); err != nil {
668-
return nil, err
669+
// ImportResolverFor returns a function which returns a compiled package archive
670+
// given an import path. Import paths are resolved relative to the |pkg|.
671+
func (s *Session) ImportResolverFor(pkg *PackageData) func(string) (*compiler.Archive, error) {
672+
return func(path string) (*compiler.Archive, error) {
673+
if pkg.ImportOverrides != nil {
674+
if override, ok := pkg.ImportOverrides[path]; ok {
675+
path = override
669676
}
677+
}
678+
if archive, ok := s.UpToDateArchives[path]; ok {
670679
return archive, nil
671680
}
672-
return nil, err
673-
}
674-
675-
return archive, nil
676-
}
677-
678-
// writeLibraryPackage writes a compiled package archive to disk at pkgObj path.
679-
func (s *Session) writeLibraryPackage(archive *compiler.Archive, pkgObj string) error {
680-
if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil {
681-
return err
682-
}
683-
684-
objFile, err := os.Create(pkgObj)
685-
if err != nil {
686-
return err
681+
_, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir)
682+
if err != nil {
683+
return nil, err
684+
}
685+
return archive, nil
687686
}
688-
defer objFile.Close()
689-
690-
return compiler.WriteArchive(archive, objFile)
691687
}
692688

693689
// WriteCommandPackage writes the final JavaScript output file at pkgObj path.
@@ -719,7 +715,7 @@ func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string)
719715
}
720716

721717
deps, err := compiler.ImportDependencies(archive, func(path string) (*compiler.Archive, error) {
722-
if archive, ok := s.Archives[path]; ok {
718+
if archive, ok := s.UpToDateArchives[path]; ok {
723719
return archive, nil
724720
}
725721
_, archive, err := s.buildImportPathWithSrcDir(path, "")
@@ -788,6 +784,11 @@ func hasGopathPrefix(file, gopath string) (hasGopathPrefix bool, prefixLen int)
788784
// WaitForChange watches file system events and returns if either when one of
789785
// the source files is modified.
790786
func (s *Session) WaitForChange() {
787+
// Will need to re-validate up-to-dateness of all archives, so flush them from
788+
// memory.
789+
s.UpToDateArchives = map[string]*compiler.Archive{}
790+
s.Types = map[string]*types.Package{}
791+
791792
s.options.PrintSuccess("watching for changes...\n")
792793
for {
793794
select {

0 commit comments

Comments
 (0)