Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit aa479c0

Browse files
nevkontaktenevkontakte
committed
Build sequence and caching overhaul.
The biggest change in this commit is a rewrite of build cache machinery. Previously GopherJS used to store its build cache under $GOPATH/pkg/$GOOS_$GOARCH, and after Go Modules introduction some parts of it were stored under os.UserCacheDir(). Starting from this commit, *all* build cache is located under os.UserCacheDir(), in a manner similar to the modern Go tool. The cache is keyed by a set of build options (such as minification, compiler version, etc.) to ensure that incompatible archives aren't picked up (see gopherjs#440 for example). This change doesn't solve *all* possible cache-related issues (for example, it still relies on timestamps to invalidate the cache, see gopherjs#805), but should eliminate a large class of confusing failure modes. The second important change is to the build order. Previously GopherJS could initiate build of an imported dependency while building the package that imported it (typically for an augmented stdlib package). Now the build is stricter, all imports are built upfront, preventing weird cyclic builds that could happened under the previous system. Lastly, package source modification time has been refactored in a way that doesn't require modification of Package.SrcModTime after the package was loaded. The old side-effect-based behavior was difficult to get right and debug.
1 parent 15ed2e8 commit aa479c0

File tree

13 files changed

+429
-221
lines changed

13 files changed

+429
-221
lines changed

‎build/build.go‎

Lines changed: 120 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ import (
2828
"github.com/gopherjs/gopherjs/compiler/gopherjspkg"
2929
"github.com/gopherjs/gopherjs/compiler/natives"
3030
"github.com/neelance/sourcemap"
31-
"github.com/shurcooL/httpfs/vfsutil"
3231
"golang.org/x/tools/go/buildutil"
3332

33+
"github.com/gopherjs/gopherjs/build/cache"
3434
_ "github.com/gopherjs/gopherjs/build/versionhack" // go/build release tags hack.
3535
)
3636

@@ -71,20 +71,6 @@ func NewBuildContext(installSuffix string, buildTags []string) XContext {
7171
}
7272
}
7373

74-
// statFile returns an os.FileInfo describing the named file.
75-
// For files in "$GOROOT/src/github.com/gopherjs/gopherjs" directory,
76-
// gopherjspkg.FS is consulted first.
77-
func statFile(path string) (os.FileInfo, error) {
78-
gopherjsRoot := filepath.Join(DefaultGOROOT, "src", "github.com", "gopherjs", "gopherjs")
79-
if strings.HasPrefix(path, gopherjsRoot+string(filepath.Separator)) {
80-
path = filepath.ToSlash(path[len(gopherjsRoot):])
81-
if fi, err := vfsutil.Stat(gopherjspkg.FS, path); err == nil {
82-
return fi, nil
83-
}
84-
}
85-
return os.Stat(path)
86-
}
87-
8874
// Import returns details about the Go package named by the import path. If the
8975
// path is a local import path naming a package that can be imported using
9076
// a standard import path, the returned package will set p.ImportPath to
@@ -117,19 +103,6 @@ func importWithSrcDir(xctx XContext, path string, srcDir string, mode build.Impo
117103
return nil, err
118104
}
119105

120-
if pkg.IsCommand() {
121-
pkg.PkgObj = filepath.Join(pkg.BinDir, filepath.Base(pkg.ImportPath)+".js")
122-
}
123-
124-
if _, err := os.Stat(pkg.PkgObj); os.IsNotExist(err) && strings.HasPrefix(pkg.PkgObj, DefaultGOROOT) {
125-
// fall back to GOPATH
126-
firstGopathWorkspace := filepath.SplitList(build.Default.GOPATH)[0] // TODO: Need to check inside all GOPATH workspaces.
127-
gopathPkgObj := filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(DefaultGOROOT):])
128-
if _, err := os.Stat(gopathPkgObj); err == nil {
129-
pkg.PkgObj = gopathPkgObj
130-
}
131-
}
132-
133106
return pkg, nil
134107
}
135108

@@ -375,8 +348,8 @@ func (o *Options) PrintSuccess(format string, a ...interface{}) {
375348
type PackageData struct {
376349
*build.Package
377350
JSFiles []string
378-
IsTest bool // IsTest is true if the package is being built for running tests.
379-
SrcModTime time.Time
351+
IsTest bool // IsTest is true if the package is being built for running tests.
352+
SrcModTime time.Time // Most recent source modification time, excluding deps.
380353
UpToDate bool
381354
IsVirtual bool // If true, the package does not have a corresponding physical directory on disk.
382355

@@ -420,16 +393,42 @@ func (p *PackageData) XTestPackage() *PackageData {
420393
}
421394
}
422395

396+
// Sources returns a slice of buildable Go and JS source files in the package.
397+
func (p *PackageData) Sources() []string {
398+
return append(append([]string{}, p.GoFiles...), p.JSFiles...)
399+
}
400+
401+
// InstallPath returns the path where "gopherjs install" command should place the
402+
// generated output.
403+
func (p *PackageData) InstallPath() string {
404+
if p.IsCommand() {
405+
name := filepath.Base(p.ImportPath) + ".js"
406+
// For executable packages, mimic go tool behavior if possible.
407+
if gobin := os.Getenv("GOBIN"); gobin != "" {
408+
return filepath.Join(gobin, name)
409+
} else if gopath := os.Getenv("GOPATH"); gopath != "" {
410+
return filepath.Join(gopath, "bin", name)
411+
} else if home, err := os.UserHomeDir(); err == nil {
412+
return filepath.Join(home, "go", "bin", name)
413+
}
414+
}
415+
return p.PkgObj
416+
}
417+
423418
// Session manages internal state GopherJS requires to perform a build.
424419
//
425420
// This is the main interface to GopherJS build system. Session lifetime is
426421
// roughly equivalent to a single GopherJS tool invocation.
427422
type Session struct {
428-
options *Options
429-
xctx XContext
430-
Archives map[string]*compiler.Archive
431-
Types map[string]*types.Package
432-
Watcher *fsnotify.Watcher
423+
options *Options
424+
xctx XContext
425+
buildCache cache.BuildCache
426+
// Binary archives produced during the current session and assumed to be
427+
// up to date with input sources and dependencies. In the -w ("watch") mode
428+
// must be cleared upon entering watching.
429+
UpToDateArchives map[string]*compiler.Archive
430+
Types map[string]*types.Package
431+
Watcher *fsnotify.Watcher
433432
}
434433

435434
// NewSession creates a new GopherJS build session.
@@ -448,10 +447,18 @@ func NewSession(options *Options) (*Session, error) {
448447
}
449448

450449
s := &Session{
451-
options: options,
452-
Archives: make(map[string]*compiler.Archive),
450+
options: options,
451+
UpToDateArchives: make(map[string]*compiler.Archive),
453452
}
454453
s.xctx = NewBuildContext(s.InstallSuffix(), s.options.BuildTags)
454+
s.buildCache = cache.BuildCache{
455+
GOOS: s.xctx.GOOS(),
456+
GOARCH: "js",
457+
GOROOT: options.GOROOT,
458+
GOPATH: options.GOPATH,
459+
BuildTags: options.BuildTags,
460+
Minify: options.Minify,
461+
}
455462
s.Types = make(map[string]*types.Package)
456463
if options.Watch {
457464
if out, err := exec.Command("ulimit", "-n").Output(); err == nil {
@@ -466,6 +473,12 @@ func NewSession(options *Options) (*Session, error) {
466473
return nil, err
467474
}
468475
}
476+
// FIXME(nevkontakte): This is a temporary hack to make sure we account for
477+
// dependencies introduced by standard library overlays. Find a better solution
478+
// before upstreaming.
479+
s.BuildImportPath("github.com/gopherjs/gopherjs/nosync")
480+
s.BuildImportPath("github.com/gopherjs/gopherjs/js")
481+
469482
return s, nil
470483
}
471484

@@ -548,70 +561,61 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag
548561

549562
// BuildPackage compiles an already loaded package.
550563
func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
551-
if archive, ok := s.Archives[pkg.ImportPath]; ok {
552-
return archive, nil
564+
if a, ok := s.UpToDateArchives[pkg.ImportPath]; ok {
565+
return a, nil
553566
}
567+
modTime := pkg.SrcModTime // Package and its dependencies modification time.
554568

555-
if pkg.PkgObj != "" {
556-
var fileInfo os.FileInfo
557-
gopherjsBinary, err := os.Executable()
558-
if err == nil {
559-
fileInfo, err = os.Stat(gopherjsBinary)
560-
if err == nil {
561-
pkg.SrcModTime = fileInfo.ModTime()
562-
}
569+
// Recursively build all dependencies first.
570+
imports := []string{}
571+
for _, impPath := range pkg.Imports {
572+
imports = append(imports, impPath)
573+
}
574+
575+
// FIXME(nevkontakte): This is a temporary hack to make sure we account for
576+
// dependencies introduced by standard library overlays. Find a better solution
577+
// before upstreaming.
578+
nativesContext := embeddedCtx(&withPrefix{fs: natives.FS, prefix: DefaultGOROOT}, "", nil)
579+
if pkg.ImportPath == "syscall" {
580+
// Special handling for the syscall package, which uses OS native
581+
// GOOS/GOARCH pair. This will no longer be necessary after
582+
// https://github.com/gopherjs/gopherjs/issues/693.
583+
nativesContext.bctx.GOARCH = build.Default.GOARCH
584+
nativesContext.bctx.BuildTags = append(nativesContext.bctx.BuildTags, "js")
585+
}
586+
if nativesPkg, err := nativesContext.Import(pkg.ImportPath, "", 0); err == nil {
587+
for _, impPath := range nativesPkg.Imports {
588+
imports = append(imports, impPath)
589+
}
590+
}
591+
// fmt.Println(pkg.ImportPath, "imports", imports)
592+
593+
for _, importedPkgPath := range imports {
594+
if importedPkgPath == "unsafe" {
595+
continue
563596
}
597+
_, importedArchive, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
564598
if err != nil {
565-
os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
566-
pkg.SrcModTime = time.Now()
599+
return nil, err
567600
}
568-
569-
for _, importedPkgPath := range pkg.Imports {
570-
if importedPkgPath == "unsafe" {
571-
continue
572-
}
573-
importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir)
574-
if err != nil {
575-
return nil, err
576-
}
577-
impModTime := importedPkg.SrcModTime
578-
if impModTime.After(pkg.SrcModTime) {
579-
pkg.SrcModTime = impModTime
580-
}
601+
if importedArchive.BuildTime.After(pkg.SrcModTime) {
602+
modTime = importedArchive.BuildTime
581603
}
604+
}
582605

583-
for _, name := range append(pkg.GoFiles, pkg.JSFiles...) {
584-
fileInfo, err := statFile(filepath.Join(pkg.Dir, name))
585-
if err != nil {
586-
return nil, err
587-
}
588-
if fileInfo.ModTime().After(pkg.SrcModTime) {
589-
pkg.SrcModTime = fileInfo.ModTime()
590-
}
606+
archive := s.buildCache.LoadArchive(pkg.ImportPath)
607+
if archive != nil && !modTime.After(archive.BuildTime) {
608+
if err := archive.RegisterTypes(s.Types); err != nil {
609+
panic(fmt.Errorf("Failed to load type information from %v: %w", archive, err))
591610
}
611+
s.UpToDateArchives[pkg.ImportPath] = archive
612+
// Existing archive is up to date, no need to build it from scratch.
613+
return archive, nil
614+
}
592615

593-
pkgObjFileInfo, err := os.Stat(pkg.PkgObj)
594-
if err == nil && !pkg.SrcModTime.After(pkgObjFileInfo.ModTime()) {
595-
// package object is up to date, load from disk if library
596-
pkg.UpToDate = true
597-
if pkg.IsCommand() {
598-
return nil, nil
599-
}
600-
601-
objFile, err := os.Open(pkg.PkgObj)
602-
if err != nil {
603-
return nil, err
604-
}
605-
defer objFile.Close()
606-
607-
archive, err := compiler.ReadArchive(pkg.PkgObj, pkg.ImportPath, objFile, s.Types)
608-
if err != nil {
609-
return nil, err
610-
}
611-
612-
s.Archives[pkg.ImportPath] = archive
613-
return archive, err
614-
}
616+
// Existing archive is out of date or doesn't exist, let's build the package.
617+
if s.options.Verbose {
618+
fmt.Println(pkg.ImportPath)
615619
}
616620

617621
fileSet := token.NewFileSet()
@@ -620,22 +624,11 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
620624
return nil, err
621625
}
622626

623-
localImportPathCache := make(map[string]*compiler.Archive)
624627
importContext := &compiler.ImportContext{
625628
Packages: s.Types,
626-
Import: func(path string) (*compiler.Archive, error) {
627-
if archive, ok := localImportPathCache[path]; ok {
628-
return archive, nil
629-
}
630-
_, archive, err := s.buildImportPathWithSrcDir(path, pkg.Dir)
631-
if err != nil {
632-
return nil, err
633-
}
634-
localImportPathCache[path] = archive
635-
return archive, nil
636-
},
629+
Import: s.importResolverFor(pkg),
637630
}
638-
archive, err := compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
631+
archive, err = compiler.Compile(pkg.ImportPath, files, fileSet, importContext, s.options.Minify)
639632
if err != nil {
640633
return nil, err
641634
}
@@ -650,44 +643,27 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
650643
archive.IncJSCode = append(archive.IncJSCode, []byte("\n\t}).call($global);\n")...)
651644
}
652645

653-
if s.options.Verbose {
654-
fmt.Println(pkg.ImportPath)
655-
}
656-
657-
s.Archives[pkg.ImportPath] = archive
658-
659-
if pkg.PkgObj == "" || pkg.IsCommand() {
660-
return archive, nil
661-
}
662-
663-
if err := s.writeLibraryPackage(archive, pkg.PkgObj); err != nil {
664-
if strings.HasPrefix(pkg.PkgObj, s.options.GOROOT) {
665-
// fall back to first GOPATH workspace
666-
firstGopathWorkspace := filepath.SplitList(s.options.GOPATH)[0]
667-
if err := s.writeLibraryPackage(archive, filepath.Join(firstGopathWorkspace, pkg.PkgObj[len(s.options.GOROOT):])); err != nil {
668-
return nil, err
669-
}
670-
return archive, nil
671-
}
672-
return nil, err
673-
}
674-
646+
s.buildCache.StoreArchive(archive)
647+
s.UpToDateArchives[pkg.ImportPath] = archive
675648
return archive, nil
676649
}
677650

678-
// writeLibraryPackage writes a compiled package archive to disk at pkgObj path.
679-
func (s *Session) writeLibraryPackage(archive *compiler.Archive, pkgObj string) error {
680-
if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil {
681-
return err
682-
}
683-
684-
objFile, err := os.Create(pkgObj)
685-
if err != nil {
686-
return err
651+
// importResolverFor returns a function, which returns a compiled package archive
652+
// given an import path. Import paths are resolved relative to the |pkg|.
653+
func (s *Session) importResolverFor(pkg *PackageData) func(string) (*compiler.Archive, error) {
654+
return func(path string) (*compiler.Archive, error) {
655+
// Import path can be relative or refer to a vendored package, resolve it
656+
// first into an actual package.
657+
imported, err := s.xctx.Import(path, pkg.Dir, build.FindOnly)
658+
if err != nil {
659+
return nil, fmt.Errorf("failed to resolve import path %q relative to %q: %w", path, pkg.ImportPath, err)
660+
}
661+
// Return the corresponding archive.
662+
if archive, ok := s.UpToDateArchives[imported.ImportPath]; ok {
663+
return archive, nil
664+
}
665+
panic(fmt.Errorf("build order error: package %q hasn't been built before being imported by %q", imported.ImportPath, pkg.ImportPath))
687666
}
688-
defer objFile.Close()
689-
690-
return compiler.WriteArchive(archive, objFile)
691667
}
692668

693669
// WriteCommandPackage writes the final JavaScript output file at pkgObj path.
@@ -719,7 +695,7 @@ func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string)
719695
}
720696

721697
deps, err := compiler.ImportDependencies(archive, func(path string) (*compiler.Archive, error) {
722-
if archive, ok := s.Archives[path]; ok {
698+
if archive, ok := s.UpToDateArchives[path]; ok {
723699
return archive, nil
724700
}
725701
_, archive, err := s.buildImportPathWithSrcDir(path, "")
@@ -788,6 +764,10 @@ func hasGopathPrefix(file, gopath string) (hasGopathPrefix bool, prefixLen int)
788764
// WaitForChange watches file system events and returns if either when one of
789765
// the source files is modified.
790766
func (s *Session) WaitForChange() {
767+
// Will need to re-validate up-to-dateness of all archives, so flush them from
768+
// memory.
769+
s.UpToDateArchives = map[string]*compiler.Archive{}
770+
791771
s.options.PrintSuccess("watching for changes...\n")
792772
for {
793773
select {

0 commit comments

Comments
 (0)