Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1f08531

Browse files
nevkontaktenevkontakte
committed
Improved build cache.
Previously GopherJS used to store its build cache under $GOPATH/pkg/$GOOS_$GOARCH, and after Go Modules introduction some parts of it were stored under os.UserCacheDir(). Starting from this commit, *all* build cache is located under os.UserCacheDir(), in a manner similar to the modern Go tool. The cache is keyed by a set of build options (such as minification, compiler version, etc.) to ensure that incompatible archives aren't picked up (see gopherjs#440 for example). This change doesn't solve *all* possible cache-related issues (for example, it still relies on timestamps to invalidate the cache, see gopherjs#805), but should eliminate a large class of confusing failure modes.
1 parent 15ed2e8 commit 1f08531

File tree

9 files changed

+365
-200
lines changed

9 files changed

+365
-200
lines changed

‎build/build.go‎

Lines changed: 111 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/shurcooL/httpfs/vfsutil"
3232
"golang.org/x/tools/go/buildutil"
3333

34+
"github.com/gopherjs/gopherjs/build/cache"
3435
_ "github.com/gopherjs/gopherjs/build/versionhack" // go/build release tags hack.
3536
)
3637

@@ -117,19 +118,6 @@ func importWithSrcDir(xctx XContext, path string, srcDir string, mode build.Impo
117118
return nil, err
118119
}
119120

120-
if pkg.IsCommand() {
121-
pkg.PkgObj = filepath.Join(pkg.BinDir, filepath.Base(pkg.ImportPath)+".js")
122-
}
123-
124-
if _, err := os.Stat(pkg.PkgObj); os.IsNotExist(err) && strings.HasPrefix(pkg.PkgObj, DefaultGOROOT) {
125-
// fall back to GOPATH
126-
firstGopathWorkspace := filepath.SplitList(build.Default.GOPATH)[0] // TODO: Need to check inside all GOPATH workspaces.
127-
gopathPkgObj := filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(DefaultGOROOT):])
128-
if _, err := os.Stat(gopathPkgObj); err == nil {
129-
pkg.PkgObj = gopathPkgObj
130-
}
131-
}
132-
133121
return pkg, nil
134122
}
135123

@@ -352,6 +340,7 @@ type Options struct {
352340
Minify bool
353341
Color bool
354342
BuildTags []string
343+
TestedPackage string
355344
}
356345

357346
// PrintError message to the terminal.
@@ -374,11 +363,13 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) {
374363
// GopherJS requires.
375364
type PackageData struct {
376365
*build.Package
377-
JSFiles []string
378-
IsTest bool // IsTest is true if the package is being built for running tests.
366+
JSFiles []string
367+
// IsTest is true if the package is being built for running tests.
368+
IsTest bool
379369
SrcModTime time.Time
380370
UpToDate bool
381-
IsVirtual bool // If true, the package does not have a corresponding physical directory on disk.
371+
// If true, the package does not have a corresponding physical directory on disk.
372+
IsVirtual bool
382373

383374
bctx *build.Context // The original build context this package came from.
384375
}
@@ -420,16 +411,38 @@ func (p *PackageData) XTestPackage() *PackageData {
420411
}
421412
}
422413

414+
// InstallPath returns the path where "gopherjs install" command should place the
415+
// generated output.
416+
func (p *PackageData) InstallPath() string {
417+
if p.IsCommand() {
418+
name := filepath.Base(p.ImportPath) + ".js"
419+
// For executable packages, mimic go tool behavior if possible.
420+
if gobin := os.Getenv("GOBIN"); gobin != "" {
421+
return filepath.Join(gobin, name)
422+
} else if gopath := os.Getenv("GOPATH"); gopath != "" {
423+
return filepath.Join(gopath, "bin", name)
424+
} else if home, err := os.UserHomeDir(); err == nil {
425+
return filepath.Join(home, "go", "bin", name)
426+
}
427+
}
428+
return p.PkgObj
429+
}
430+
423431
// Session manages internal state GopherJS requires to perform a build.
424432
//
425433
// This is the main interface to GopherJS build system. Session lifetime is
426434
// roughly equivalent to a single GopherJS tool invocation.
427435
type Session struct {
428-
options *Options
429-
xctx XContext
430-
Archives map[string]*compiler.Archive
431-
Types map[string]*types.Package
432-
Watcher *fsnotify.Watcher
436+
options *Options
437+
xctx XContext
438+
buildCache cache.BuildCache
439+
440+
// Binary archives produced during the current session and assumed to be
441+
// up to date with input sources and dependencies. In the -w ("watch") mode
442+
// must be cleared upon entering watching.
443+
UpToDateArchives map[string]*compiler.Archive
444+
Types map[string]*types.Package
445+
Watcher *fsnotify.Watcher
433446
}
434447

435448
// NewSession creates a new GopherJS build session.
@@ -448,10 +461,19 @@ func NewSession(options *Options) (*Session, error) {
448461
}
449462

450463
s := &Session{
451-
options: options,
452-
Archives: make(map[string]*compiler.Archive),
464+
options: options,
465+
UpToDateArchives: make(map[string]*compiler.Archive),
453466
}
454467
s.xctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags)
468+
s.buildCache = cache.BuildCache{
469+
GOOS: s.xctx.GOOS(),
470+
GOARCH: "js",
471+
GOROOT: options.GOROOT,
472+
GOPATH: options.GOPATH,
473+
BuildTags: options.BuildTags,
474+
Minify: options.Minify,
475+
TestedPackage: options.TestedPackage,
476+
}
455477
s.Types = make(map[string]*types.Package)
456478
if options.Watch {
457479
if out, err := exec.Command("ulimit", "-n").Output(); err == nil {
@@ -493,10 +515,13 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath stri
493515
pkg := &PackageData{
494516
Package: &build.Package{
495517
Name: "main",
496-
ImportPath: "main",
518+
ImportPath: "command-line-arguments",
497519
Dir: packagePath,
498520
},
499-
bctx: &goCtx(s.InstallSuffix(), s.options.BuildTags).bctx,
521+
// This ephemeral package doesn't have a unique import path to be used as a
522+
// build cache key, so we never cache it.
523+
SrcModTime: time.Now().Add(time.Hour),
524+
bctx: &goCtx(s.InstallSuffix(), s.options.BuildTags).bctx,
500525
}
501526

502527
for _, file := range filenames {
@@ -511,7 +536,7 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath stri
511536
if err != nil {
512537
return err
513538
}
514-
if s.Types["main"].Name() != "main" {
539+
if s.Types["command-line-arguments"].Name() != "main" {
515540
return fmt.Errorf("cannot build/run non-main package")
516541
}
517542
return s.WriteCommandPackage(archive, pkgObj)
@@ -548,94 +573,70 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag
548573

549574
// BuildPackage compiles an already loaded package.
550575
func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
551-
if archive, ok := s.Archives[pkg.ImportPath]; ok {
576+
if archive, ok := s.UpToDateArchives[pkg.ImportPath]; ok {
552577
return archive, nil
553578
}
554579

555-
if pkg.PkgObj != "" {
556-
var fileInfo os.FileInfo
557-
gopherjsBinary, err := os.Executable()
580+
var fileInfo os.FileInfo
581+
gopherjsBinary, err := os.Executable()
582+
if err == nil {
583+
fileInfo, err = os.Stat(gopherjsBinary)
558584
if err == nil {
559-
fileInfo, err = os.Stat(gopherjsBinary)
560-
if err == nil {
561-
pkg.SrcModTime = fileInfo.ModTime()
562-
}
585+
pkg.SrcModTime = fileInfo.ModTime()
586+
}
587+
}
588+
if err != nil {
589+
os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
590+
pkg.SrcModTime = time.Now()
591+
}
592+
593+
for _, importedPkgPath := range pkg.Imports {
594+
if importedPkgPath == "unsafe" {
595+
continue
563596
}
597+
importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
564598
if err != nil {
565-
os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
566-
pkg.SrcModTime = time.Now()
599+
return nil, err
567600
}
568601

569-
for _, importedPkgPath := range pkg.Imports {
570-
if importedPkgPath == "unsafe" {
571-
continue
572-
}
573-
importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
574-
if err != nil {
575-
return nil, err
576-
}
577-
impModTime := importedPkg.SrcModTime
578-
if impModTime.After(pkg.SrcModTime) {
579-
pkg.SrcModTime = impModTime
580-
}
602+
impModTime := importedPkg.SrcModTime
603+
if impModTime.After(pkg.SrcModTime) {
604+
pkg.SrcModTime = impModTime
581605
}
606+
}
582607

583-
for _, name := range append(pkg.GoFiles, pkg.JSFiles...) {
584-
fileInfo, err := statFile(filepath.Join(pkg.Dir, name))
585-
if err != nil {
586-
return nil, err
587-
}
588-
if fileInfo.ModTime().After(pkg.SrcModTime) {
589-
pkg.SrcModTime = fileInfo.ModTime()
590-
}
608+
for _, name := range append(pkg.GoFiles, pkg.JSFiles...) {
609+
fileInfo, err := statFile(filepath.Join(pkg.Dir, name))
610+
if err != nil {
611+
return nil, err
591612
}
613+
if fileInfo.ModTime().After(pkg.SrcModTime) {
614+
pkg.SrcModTime = fileInfo.ModTime()
615+
}
616+
}
592617

593-
pkgObjFileInfo, err := os.Stat(pkg.PkgObj)
594-
if err == nil && !pkg.SrcModTime.After(pkgObjFileInfo.ModTime()) {
595-
// package object is up to date, load from disk if library
596-
pkg.UpToDate = true
597-
if pkg.IsCommand() {
598-
return nil, nil
599-
}
600-
601-
objFile, err := os.Open(pkg.PkgObj)
602-
if err != nil {
603-
return nil, err
604-
}
605-
defer objFile.Close()
606-
607-
archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types)
608-
if err != nil {
609-
return nil, err
610-
}
611-
612-
s.Archives[pkg.ImportPath] = archive
613-
return archive, err
618+
archive := s.buildCache.LoadArchive(pkg.ImportPath)
619+
if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) {
620+
if err := archive.RegisterTypes(s.Types); err != nil {
621+
panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err))
614622
}
623+
s.UpToDateArchives[pkg.ImportPath] = archive
624+
// Existing archive is up to date, no need to build it from scratch.
625+
return archive, nil
615626
}
616627

628+
// Existing archive is out of date or doesn't exist, let's build the package.
617629
fileSet := token.NewFileSet()
618630
files, err := parseAndAugment(s.xctx, pkg, pkg.IsTest, fileSet)
619631
if err != nil {
620632
return nil, err
621633
}
622634

623-
localImportPathCache := make(map[string]*compiler.Archive)
624635
importContext := &compiler.ImportContext{
625636
Packages: s.Types,
626-
Import: func(path string) (*compiler.Archive, error) {
627-
if archive, ok := localImportPathCache[path]; ok {
628-
return archive, nil
629-
}
630-
_, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir)
631-
if err != nil {
632-
return nil, err
633-
}
634-
localImportPathCache[path] = archive
635-
return archive, nil
636-
},
637+
Import: s.ImportResolverFor(pkg),
637638
}
638-
archive, err := compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
639+
archive, err = compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
639640
if err != nil {
640641
return nil, err
641642
}
@@ -654,40 +655,25 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
654655
fmt.Println(pkg.ImportPath)
655656
}
656657

657-
s.Archives[pkg.ImportPath] = archive
658-
659-
if pkg.PkgObj == "" || pkg.IsCommand() {
660-
return archive, nil
661-
}
662-
663-
if err := s.writeLibraryPackage(archive, pkg.PkgObj); err != nil {
664-
if strings.HasPrefix(pkg.PkgObj, s.options.GOROOT) {
665-
// fall back to first GOPATH workspace
666-
firstGopathWorkspace := filepath.SplitList(s.options.GOPATH)[0]
667-
if err := s.writeLibraryPackage(archive, filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(s.options.GOROOT):])); err != nil {
668-
return nil, err
669-
}
670-
return archive, nil
671-
}
672-
return nil, err
673-
}
658+
s.buildCache.StoreArchive(archive)
659+
s.UpToDateArchives[pkg.ImportPath] = archive
674660

675661
return archive, nil
676662
}
677663

678-
// writeLibraryPackage writes a compiled package archive to disk at pkgObj path.
679-
func (s *Session) writeLibraryPackage(archive *compiler.Archive, pkgObj string) error {
680-
if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil {
681-
return err
682-
}
683-
684-
objFile, err := os.Create(pkgObj)
685-
if err != nil {
686-
return err
664+
// ImportResolverFor returns a function which returns a compiled package archive
665+
// given an import path. Import paths are resolved relative to the |pkg|.
666+
func (s *Session) ImportResolverFor(pkg *PackageData) func(string) (*compiler.Archive, error) {
667+
return func(path string) (*compiler.Archive, error) {
668+
if archive, ok := s.UpToDateArchives[path]; ok {
669+
return archive, nil
670+
}
671+
_, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir)
672+
if err != nil {
673+
return nil, err
674+
}
675+
return archive, nil
687676
}
688-
defer objFile.Close()
689-
690-
return compiler.WriteArchive(archive, objFile)
691677
}
692678

693679
// WriteCommandPackage writes the final JavaScript output file at pkgObj path.
@@ -719,7 +705,7 @@ func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string)
719705
}
720706

721707
deps, err := compiler.ImportDependencies(archive, func(path string) (*compiler.Archive, error) {
722-
if archive, ok := s.Archives[path]; ok {
708+
if archive, ok := s.UpToDateArchives[path]; ok {
723709
return archive, nil
724710
}
725711
_, archive, err := s.buildImportPathWithSrcDir(path, "")
@@ -788,6 +774,11 @@ func hasGopathPrefix(file, gopath string) (hasGopathPrefix bool, prefixLen int)
788774
// WaitForChange watches file system events and returns if either when one of
789775
// the source files is modified.
790776
func (s *Session) WaitForChange() {
777+
// Will need to re-validate up-to-dateness of all archives, so flush them from
778+
// memory.
779+
s.UpToDateArchives = map[string]*compiler.Archive{}
780+
s.Types = map[string]*types.Package{}
781+
791782
s.options.PrintSuccess("watching for changes...\n")
792783
for {
793784
select {

‎build/cache.go‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

0 commit comments

Comments
 (0)