Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 113c92b

Browse files
grantnelson-wfgrantnelson-wf
authored
Merge pull request #1390 from Workiva/serializeSources
Update caching to use serialization of sources
2 parents e890dcd + 8d4b7e5 commit 113c92b

File tree

10 files changed

+682
-370
lines changed

10 files changed

+682
-370
lines changed

‎build/build.go‎

Lines changed: 69 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
log "github.com/sirupsen/logrus"
3030
"golang.org/x/tools/go/buildutil"
3131

32+
"github.com/gopherjs/gopherjs/build/cache"
3233
"github.com/gopherjs/gopherjs/compiler"
3334
"github.com/gopherjs/gopherjs/compiler/astutil"
3435
"github.com/gopherjs/gopherjs/compiler/incjs"
@@ -744,8 +745,9 @@ func (p *PackageData) InstallPath() string {
744745
// This is the main interface to GopherJS build system. Session lifetime is
745746
// roughly equivalent to a single GopherJS tool invocation.
746747
type Session struct {
747-
options *Options
748-
xctx XContext
748+
options *Options
749+
xctx XContext
750+
buildCache cache.Cache
749751

750752
// importPaths is a map of the resolved import paths given the
751753
// source directory (first key) and the unresolved import path (second key).
@@ -787,6 +789,25 @@ func NewSession(options *Options) (*Session, error) {
787789
return nil, err
788790
}
789791

792+
// If the cache is enabled, initialize the build cache.
793+
// Disable caching by leaving buildCache set to nil.
794+
//
795+
// TODO(grantnelson-wf): Currently the build cache is slower than
796+
// parsing and augmenting the files, so we disable it for now.
797+
// Re-enable it once the cache performance is improved.
798+
const disableDefaultCache = true
799+
if !s.options.NoCache && !disableDefaultCache {
800+
s.buildCache = &cache.BuildCache{
801+
GOOS: env.GOOS,
802+
GOARCH: env.GOARCH,
803+
GOROOT: env.GOROOT,
804+
GOPATH: env.GOPATH,
805+
BuildTags: append([]string{}, env.BuildTags...),
806+
TestedPackage: options.TestedPackage,
807+
Version: compiler.Version,
808+
}
809+
}
810+
790811
if options.Watch {
791812
if out, err := exec.Command("ulimit", "-n").Output(); err == nil {
792813
if n, err := strconv.Atoi(strings.TrimSpace(string(out))); err == nil && n < 1024 {
@@ -913,26 +934,19 @@ func (s *Session) BuildProject(pkg *PackageData) (*compiler.Archive, error) {
913934
if pkg.IsTest {
914935
rootSrcs, err = s.loadTestPackage(pkg)
915936
} else {
916-
rootSrcs, err = s.loadPackages(pkg)
937+
rootSrcs, err = s.LoadPackages(pkg)
917938
}
918939
if err != nil {
919940
return nil, err
920941
}
921942

922-
// TODO(grantnelson-wf): We could investigate caching the results of
923-
// the sources prior to preparing them to avoid re-parsing the same
924-
// sources and augmenting them when the files on disk haven't changed.
925-
// This would require a way to determine if the sources are up-to-date
926-
// which could be done with the left over srcModTime from when the archives
927-
// were being cached.
928-
929943
// Compile the project into Archives containing the generated JS.
930944
return s.prepareAndCompilePackages(rootSrcs)
931945
}
932946

933-
// getSortedSources returns the sources sorted by import path.
947+
// GetSortedSources returns the sources sorted by import path.
934948
// The files in the sources may still not be sorted yet.
935-
func (s *Session) getSortedSources() []*sources.Sources {
949+
func (s *Session) GetSortedSources() []*sources.Sources {
936950
allSources := make([]*sources.Sources, 0, len(s.sources))
937951
for _, srcs := range s.sources {
938952
allSources = append(allSources, srcs)
@@ -942,11 +956,11 @@ func (s *Session) getSortedSources() []*sources.Sources {
942956
}
943957

944958
func (s *Session) loadTestPackage(pkg *PackageData) (*sources.Sources, error) {
945-
_, err := s.loadPackages(pkg.TestPackage())
959+
_, err := s.LoadPackages(pkg.TestPackage())
946960
if err != nil {
947961
return nil, err
948962
}
949-
_, err = s.loadPackages(pkg.XTestPackage())
963+
_, err = s.LoadPackages(pkg.XTestPackage())
950964
if err != nil {
951965
return nil, err
952966
}
@@ -993,7 +1007,7 @@ func (s *Session) loadImportPathWithSrcDir(path, srcDir string) (*PackageData, *
9931007
return nil, nil, err
9941008
}
9951009

996-
srcs, err := s.loadPackages(pkg)
1010+
srcs, err := s.LoadPackages(pkg)
9971011
if err != nil {
9981012
return nil, nil, err
9991013
}
@@ -1040,11 +1054,11 @@ var getExeModTime = func() func() time.Time {
10401054
}
10411055
}()
10421056

1043-
// loadPackages will recursively load and parse the given package and
1057+
// LoadPackages will recursively load and parse the given package and
10441058
// its dependencies. This will return the sources for the given package.
10451059
// The returned source and sources for the dependencies will be added
10461060
// to the session's sources map.
1047-
func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) {
1061+
func (s *Session) LoadPackages(pkg *PackageData) (*sources.Sources, error) {
10481062
if srcs, ok := s.sources[pkg.ImportPath]; ok {
10491063
return srcs, nil
10501064
}
@@ -1071,27 +1085,46 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) {
10711085
pkg.SrcModTime = fileModTime
10721086
}
10731087

1074-
// Build the package by parsing and augmenting the original files with overlay files.
1075-
fileSet := token.NewFileSet()
1076-
files, overlayJsFiles, err := parseAndAugment(s.xctx, pkg, pkg.IsTest, fileSet)
1077-
if err != nil {
1078-
return nil, err
1079-
}
1080-
embed, err := embedFiles(pkg, fileSet, files)
1081-
if err != nil {
1082-
return nil, err
1083-
}
1084-
if embed != nil {
1085-
files = append(files, embed)
1088+
// Try to load the package from the build cache.
1089+
var srcs *sources.Sources
1090+
if s.buildCache != nil {
1091+
cachedSrcs := &sources.Sources{}
1092+
if s.buildCache.Load(cachedSrcs, pkg.ImportPath, pkg.SrcModTime) {
1093+
srcs = cachedSrcs
1094+
}
10861095
}
10871096

1088-
srcs := &sources.Sources{
1089-
ImportPath: pkg.ImportPath,
1090-
Dir: pkg.Dir,
1091-
Files: files,
1092-
FileSet: fileSet,
1093-
JSFiles: append(pkg.JSFiles, overlayJsFiles...),
1097+
// If the package was not found in the cache, build the package
1098+
// by parsing and augmenting the original files with overlay files.
1099+
if srcs == nil {
1100+
fileSet := token.NewFileSet()
1101+
files, overlayJsFiles, err := parseAndAugment(s.xctx, pkg, pkg.IsTest, fileSet)
1102+
if err != nil {
1103+
return nil, err
1104+
}
1105+
embed, err := embedFiles(pkg, fileSet, files)
1106+
if err != nil {
1107+
return nil, err
1108+
}
1109+
if embed != nil {
1110+
files = append(files, embed)
1111+
}
1112+
1113+
srcs = &sources.Sources{
1114+
ImportPath: pkg.ImportPath,
1115+
Dir: pkg.Dir,
1116+
Files: files,
1117+
FileSet: fileSet,
1118+
JSFiles: append(pkg.JSFiles, overlayJsFiles...),
1119+
}
1120+
1121+
// Store the built package in the cache for future use.
1122+
if s.buildCache != nil {
1123+
s.buildCache.Store(srcs, srcs.ImportPath, time.Now())
1124+
}
10941125
}
1126+
1127+
// Add the sources to the session's sources map.
10951128
s.sources[pkg.ImportPath] = srcs
10961129

10971130
// Import dependencies from the augmented files,
@@ -1108,7 +1141,7 @@ func (s *Session) loadPackages(pkg *PackageData) (*sources.Sources, error) {
11081141

11091142
func (s *Session) prepareAndCompilePackages(rootSrcs *sources.Sources) (*compiler.Archive, error) {
11101143
tContext := types.NewContext()
1111-
allSources := s.getSortedSources()
1144+
allSources := s.GetSortedSources()
11121145

11131146
// Prepare and analyze the source code.
11141147
// This will be performed recursively for all dependencies.

0 commit comments

Comments
 (0)