Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit fdc611d

Browse files
nevkontaktenevkontakte
committed
Build sequence and caching overhaul.
The biggest change in this commit is a rewrite of build cache machinery. Previously GopherJS used to store its build cache under $GOPATH/pkg/$GOOS_$GOARCH, and after Go Modules introduction some parts of it were stored under os.UserCacheDir(). Starting from this commit, *all* build cache is located under os.UserCacheDir(), in a manner similar to the modern Go tool. The cache is keyed by a set of build options (such as minification, compiler version, etc.) to ensure that incompatible archives aren't picked up (see gopherjs#440 for example). This change doesn't solve *all* possible cache-related issues (for example, it still relies on timestamps to invalidate the cache, see gopherjs#805), but should eliminate a large class of confusing failure modes. The second important change is to the build order. Previously GopherJS could initiate build of an imported dependency while building the package that imported it (typically for an augmented stdlib package). Now the build is stricter, all imports are built upfront, preventing weird cyclic builds that could happened under the previous system. Lastly, package source modification time has been refactored in a way that doesn't require modification of Package.SrcModTime after the package was loaded. The old side-effect-based behavior was difficult to get right and debug.
1 parent 15ed2e8 commit fdc611d

File tree

12 files changed

+413
-219
lines changed

12 files changed

+413
-219
lines changed

‎build/build.go‎

Lines changed: 105 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ import (
2828
"github.com/gopherjs/gopherjs/compiler/gopherjspkg"
2929
"github.com/gopherjs/gopherjs/compiler/natives"
3030
"github.com/neelance/sourcemap"
31-
"github.com/shurcooL/httpfs/vfsutil"
3231
"golang.org/x/tools/go/buildutil"
3332

33+
"github.com/gopherjs/gopherjs/build/cache"
3434
_ "github.com/gopherjs/gopherjs/build/versionhack" // go/build release tags hack.
3535
)
3636

@@ -71,20 +71,6 @@ func NewBuildContext(installSuffix string, buildTags []string) XContext {
7171
}
7272
}
7373

74-
// statFile returns an os.FileInfo describing the named file.
75-
// For files in "$GOROOT/src/github.com/gopherjs/gopherjs" directory,
76-
// gopherjspkg.FS is consulted first.
77-
func statFile(path string) (os.FileInfo, error) {
78-
gopherjsRoot := filepath.Join(DefaultGOROOT, "src", "github.com", "gopherjs", "gopherjs")
79-
if strings.HasPrefix(path, gopherjsRoot+string(filepath.Separator)) {
80-
path = filepath.ToSlash(path[len(gopherjsRoot):])
81-
if fi, err := vfsutil.Stat(gopherjspkg.FS, path); err == nil {
82-
return fi, nil
83-
}
84-
}
85-
return os.Stat(path)
86-
}
87-
8874
// Import returns details about the Go package named by the import path. If the
8975
// path is a local import path naming a package that can be imported using
9076
// a standard import path, the returned package will set p.ImportPath to
@@ -117,19 +103,6 @@ func importWithSrcDir(xctx XContext, path string, srcDir string, mode build.Impo
117103
return nil, err
118104
}
119105

120-
if pkg.IsCommand() {
121-
pkg.PkgObj = filepath.Join(pkg.BinDir, filepath.Base(pkg.ImportPath)+".js")
122-
}
123-
124-
if _, err := os.Stat(pkg.PkgObj); os.IsNotExist(err) && strings.HasPrefix(pkg.PkgObj, DefaultGOROOT) {
125-
// fall back to GOPATH
126-
firstGopathWorkspace := filepath.SplitList(build.Default.GOPATH)[0] // TODO: Need to check inside all GOPATH workspaces.
127-
gopathPkgObj := filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(DefaultGOROOT):])
128-
if _, err := os.Stat(gopathPkgObj); err == nil {
129-
pkg.PkgObj = gopathPkgObj
130-
}
131-
}
132-
133106
return pkg, nil
134107
}
135108

@@ -375,8 +348,8 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) {
375348
type PackageData struct {
376349
*build.Package
377350
JSFiles []string
378-
IsTest bool // IsTest is true if the package is being built for running tests.
379-
SrcModTime time.Time
351+
IsTest bool // IsTest is true if the package is being built for running tests.
352+
SrcModTime time.Time // Most recent source modification time, excluding deps.
380353
UpToDate bool
381354
IsVirtual bool // If true, the package does not have a corresponding physical directory on disk.
382355

@@ -420,16 +393,42 @@ func (p *PackageData) XTestPackage() *PackageData {
420393
}
421394
}
422395

396+
// Sources returns a slice of buildable Go and JS source files in the package.
397+
func (p *PackageData) Sources() []string {
398+
return append(append([]string{}, p.GoFiles...), p.JSFiles...)
399+
}
400+
401+
// InstallPath returns the path where "gopherjs install" command should place the
402+
// generated output.
403+
func (p *PackageData) InstallPath() string {
404+
if p.IsCommand() {
405+
name := filepath.Base(p.ImportPath) + ".js"
406+
// For executable packages, mimic go tool behavior if possible.
407+
if gobin := os.Getenv("GOBIN"); gobin != "" {
408+
return filepath.Join(gobin, name)
409+
} else if gopath := os.Getenv("GOPATH"); gopath != "" {
410+
return filepath.Join(gopath, "bin", name)
411+
} else if home, err := os.UserHomeDir(); err == nil {
412+
return filepath.Join(home, "go", "bin", name)
413+
}
414+
}
415+
return p.PkgObj
416+
}
417+
423418
// Session manages internal state GopherJS requires to perform a build.
424419
//
425420
// This is the main interface to GopherJS build system. Session lifetime is
426421
// roughly equivalent to a single GopherJS tool invocation.
427422
type Session struct {
428-
options *Options
429-
xctx XContext
430-
Archives map[string]*compiler.Archive
431-
Types map[string]*types.Package
432-
Watcher *fsnotify.Watcher
423+
options *Options
424+
xctx XContext
425+
buildCache cache.BuildCache
426+
// Binary archives produced during the current session and assumed to be
427+
// up to date with input sources and dependencies. In the -w ("watch") mode
428+
// must be cleared upon entering watching.
429+
UpToDateArchives map[string]*compiler.Archive
430+
Types map[string]*types.Package
431+
Watcher *fsnotify.Watcher
433432
}
434433

435434
// NewSession creates a new GopherJS build session.
@@ -448,10 +447,18 @@ func NewSession(options *Options) (*Session, error) {
448447
}
449448

450449
s := &Session{
451-
options: options,
452-
Archives: make(map[string]*compiler.Archive),
450+
options: options,
451+
UpToDateArchives: make(map[string]*compiler.Archive),
453452
}
454453
s.xctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags)
454+
s.buildCache = cache.BuildCache{
455+
GOOS: s.xctx.GOOS(),
456+
GOARCH: "js",
457+
GOROOT: options.GOROOT,
458+
GOPATH: options.GOPATH,
459+
BuildTags: options.BuildTags,
460+
Minify: options.Minify,
461+
}
455462
s.Types = make(map[string]*types.Package)
456463
if options.Watch {
457464
if out, err := exec.Command("ulimit", "-n").Output(); err == nil {
@@ -466,6 +473,12 @@ func NewSession(options *Options) (*Session, error) {
466473
return nil, err
467474
}
468475
}
476+
// FIXME(nevkontakte): This is a temporary hack to make sure we account for
477+
// dependencies introduced by standard library overlays. Find a better solution
478+
// before upstreaming.
479+
s.BuildImportPath("github.com/gopherjs/gopherjs/nosync")
480+
s.BuildImportPath("github.com/gopherjs/gopherjs/js")
481+
469482
return s, nil
470483
}
471484

@@ -548,70 +561,60 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag
548561

549562
// BuildPackage compiles an already loaded package.
550563
func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
551-
if archive, ok := s.Archives[pkg.ImportPath]; ok {
552-
return archive, nil
564+
if a, ok := s.UpToDateArchives[pkg.ImportPath]; ok {
565+
return a, nil
553566
}
567+
modTime := pkg.SrcModTime // Package and its dependencies modification time.
554568

555-
if pkg.PkgObj != "" {
556-
var fileInfo os.FileInfo
557-
gopherjsBinary, err := os.Executable()
558-
if err == nil {
559-
fileInfo, err = os.Stat(gopherjsBinary)
560-
if err == nil {
561-
pkg.SrcModTime = fileInfo.ModTime()
562-
}
569+
// Recursively build all dependencies first.
570+
imports := []string{}
571+
for _, impPath := range pkg.Imports {
572+
imports = append(imports, impPath)
573+
}
574+
575+
// FIXME(nevkontakte): This is a temporary hack to make sure we account for
576+
// dependencies introduced by standard library overlays. Find a better solution
577+
// before upstreaming.
578+
nativesContext := embeddedCtx(&withPrefix{fs: natives.FS, prefix: DefaultGOROOT}, "", nil)
579+
if pkg.ImportPath == "syscall" {
580+
// Special handling for the syscall package, which uses OS native
581+
// GOOS/GOARCH pair. This will no longer be necessary after
582+
// https://github.com/gopherjs/gopherjs/issues/693.
583+
nativesContext.bctx.GOARCH = build.Default.GOARCH
584+
nativesContext.bctx.BuildTags = append(nativesContext.bctx.BuildTags, "js")
585+
}
586+
if nativesPkg, err := nativesContext.Import(pkg.ImportPath, "", 0); err == nil {
587+
for _, impPath := range nativesPkg.Imports {
588+
imports = append(imports, impPath)
589+
}
590+
}
591+
592+
for _, importedPkgPath := range imports {
593+
if importedPkgPath == "unsafe" {
594+
continue
563595
}
596+
_, importedArchive, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
564597
if err != nil {
565-
os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
566-
pkg.SrcModTime = time.Now()
598+
return nil, err
567599
}
568-
569-
for _, importedPkgPath := range pkg.Imports {
570-
if importedPkgPath == "unsafe" {
571-
continue
572-
}
573-
importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
574-
if err != nil {
575-
return nil, err
576-
}
577-
impModTime := importedPkg.SrcModTime
578-
if impModTime.After(pkg.SrcModTime) {
579-
pkg.SrcModTime = impModTime
580-
}
600+
if importedArchive.BuildTime.After(pkg.SrcModTime) {
601+
modTime = importedArchive.BuildTime
581602
}
603+
}
582604

583-
for _, name := range append(pkg.GoFiles, pkg.JSFiles...) {
584-
fileInfo, err := statFile(filepath.Join(pkg.Dir, name))
585-
if err != nil {
586-
return nil, err
587-
}
588-
if fileInfo.ModTime().After(pkg.SrcModTime) {
589-
pkg.SrcModTime = fileInfo.ModTime()
590-
}
605+
archive := s.buildCache.LoadArchive(pkg.ImportPath)
606+
if archive != nil && !modTime.After(archive.BuildTime) {
607+
if err := archive.RegisterTypes(s.Types); err != nil {
608+
panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err))
591609
}
610+
s.UpToDateArchives[pkg.ImportPath] = archive
611+
// Existing archive is up to date, no need to build it from scratch.
612+
return archive, nil
613+
}
592614

593-
pkgObjFileInfo, err := os.Stat(pkg.PkgObj)
594-
if err == nil && !pkg.SrcModTime.After(pkgObjFileInfo.ModTime()) {
595-
// package object is up to date, load from disk if library
596-
pkg.UpToDate = true
597-
if pkg.IsCommand() {
598-
return nil, nil
599-
}
600-
601-
objFile, err := os.Open(pkg.PkgObj)
602-
if err != nil {
603-
return nil, err
604-
}
605-
defer objFile.Close()
606-
607-
archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types)
608-
if err != nil {
609-
return nil, err
610-
}
611-
612-
s.Archives[pkg.ImportPath] = archive
613-
return archive, err
614-
}
615+
// Existing archive is out of date or doesn't exist, let's build the package.
616+
if s.options.Verbose {
617+
fmt.Println(pkg.ImportPath)
615618
}
616619

617620
fileSet := token.NewFileSet()
@@ -620,22 +623,16 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
620623
return nil, err
621624
}
622625

623-
localImportPathCache := make(map[string]*compiler.Archive)
624626
importContext := &compiler.ImportContext{
625627
Packages: s.Types,
626628
Import: func(path string) (*compiler.Archive, error) {
627-
if archive, ok := localImportPathCache[path]; ok {
629+
if archive, ok := s.UpToDateArchives[path]; ok {
628630
return archive, nil
629631
}
630-
_, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir)
631-
if err != nil {
632-
return nil, err
633-
}
634-
localImportPathCache[path] = archive
635-
return archive, nil
632+
panic(fmt.Errorf("attempted to import package %q while building package %q, which was not among dependencies", path, pkg.ImportPath))
636633
},
637634
}
638-
archive, err := compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
635+
archive, err = compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
639636
if err != nil {
640637
return nil, err
641638
}
@@ -650,46 +647,11 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
650647
archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...)
651648
}
652649

653-
if s.options.Verbose {
654-
fmt.Println(pkg.ImportPath)
655-
}
656-
657-
s.Archives[pkg.ImportPath] = archive
658-
659-
if pkg.PkgObj == "" || pkg.IsCommand() {
660-
return archive, nil
661-
}
662-
663-
if err := s.writeLibraryPackage(archive, pkg.PkgObj); err != nil {
664-
if strings.HasPrefix(pkg.PkgObj, s.options.GOROOT) {
665-
// fall back to first GOPATH workspace
666-
firstGopathWorkspace := filepath.SplitList(s.options.GOPATH)[0]
667-
if err := s.writeLibraryPackage(archive, filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(s.options.GOROOT):])); err != nil {
668-
return nil, err
669-
}
670-
return archive, nil
671-
}
672-
return nil, err
673-
}
674-
650+
s.buildCache.StoreArchive(archive)
651+
s.UpToDateArchives[pkg.ImportPath] = archive
675652
return archive, nil
676653
}
677654

678-
// writeLibraryPackage writes a compiled package archive to disk at pkgObj path.
679-
func (s *Session) writeLibraryPackage(archive *compiler.Archive, pkgObj string) error {
680-
if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil {
681-
return err
682-
}
683-
684-
objFile, err := os.Create(pkgObj)
685-
if err != nil {
686-
return err
687-
}
688-
defer objFile.Close()
689-
690-
return compiler.WriteArchive(archive, objFile)
691-
}
692-
693655
// WriteCommandPackage writes the final JavaScript output file at pkgObj path.
694656
func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string) error {
695657
if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil {
@@ -719,7 +681,7 @@ func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string)
719681
}
720682

721683
deps, err := compiler.ImportDependencies(archive, func(path string) (*compiler.Archive, error) {
722-
if archive, ok := s.Archives[path]; ok {
684+
if archive, ok := s.UpToDateArchives[path]; ok {
723685
return archive, nil
724686
}
725687
_, archive, err := s.buildImportPathWithSrcDir(path, "")
@@ -788,6 +750,10 @@ func hasGopathPrefix(file, gopath string) (hasGopathPrefix bool, prefixLen int)
788750
// WaitForChange watches file system events and returns if either when one of
789751
// the source files is modified.
790752
func (s *Session) WaitForChange() {
753+
// Will need to re-validate up-to-dateness of all archives, so flush them from
754+
// memory.
755+
s.UpToDateArchives = map[string]*compiler.Archive{}
756+
791757
s.options.PrintSuccess("watching for changes...\n")
792758
for {
793759
select {

‎build/cache.go‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

0 commit comments

Comments
 (0)