From 27afc785061907376f93acf8e40e33ed0935a1f0 Mon Sep 17 00:00:00 2001 From: Nevkontakte Date: Sat, 25 Dec 2021 17:24:56 +0000 Subject: [PATCH 1/3] `build` package cleanups. - Add exported symbol comments golint complains about. - Remove Session.BuildDir, which appears to be unused. --- build/build.go | 58 ++++++++++++++++++++++++++---------------------- build/context.go | 15 ++++++++----- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/build/build.go b/build/build.go index d9c0a2f42..57cc01c40 100644 --- a/build/build.go +++ b/build/build.go @@ -37,7 +37,7 @@ import ( // DefaultGOROOT is the default GOROOT value for builds. // // It uses the GOPHERJS_GOROOT environment variable if it is set, -// or else the default GOROOT value of the system Go distrubtion. +// or else the default GOROOT value of the system Go distribution. var DefaultGOROOT = func() string { if goroot, ok := os.LookupEnv("GOPHERJS_GOROOT"); ok { // GopherJS-specific GOROOT value takes precedence. @@ -47,6 +47,8 @@ var DefaultGOROOT = func() string { return build.Default.GOROOT }() +// ImportCError is returned when GopherJS attempts to build a package that uses +// CGo. type ImportCError struct { pkgPath string } @@ -378,6 +380,7 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke return files, nil } +// Options controls build process behavior. type Options struct { GOROOT string GOPATH string @@ -391,6 +394,7 @@ type Options struct { BuildTags []string } +// PrintError message to the terminal. func (o *Options) PrintError(format string, a ...interface{}) { if o.Color { format = "\x1B[31m" + format + "\x1B[39m" @@ -398,6 +402,7 @@ func (o *Options) PrintError(format string, a ...interface{}) { fmt.Fprintf(os.Stderr, format, a...) } +// PrintSuccess message to the terminal. func (o *Options) PrintSuccess(format string, a ...interface{}) { if o.Color { format = "\x1B[32m" + format + "\x1B[39m" @@ -455,6 +460,10 @@ func (p *PackageData) XTestPackage() *PackageData { } } +// Session manages internal state GopherJS requires to perform a build. +// +// This is the main interface to GopherJS build system. Session lifetime is +// roughly equivalent to a single GopherJS tool invocation. type Session struct { options *Options xctx XContext @@ -463,6 +472,7 @@ type Session struct { Watcher *fsnotify.Watcher } +// NewSession creates a new GopherJS build session. func NewSession(options *Options) (*Session, error) { if options.GOROOT == "" { options.GOROOT = DefaultGOROOT @@ -499,9 +509,10 @@ func NewSession(options *Options) (*Session, error) { return s, nil } -// BuildContext returns the session's build context. +// XContext returns the session's build context. func (s *Session) XContext() XContext { return s.xctx } +// InstallSuffix returns the suffix added to the generated output file. func (s *Session) InstallSuffix() string { if s.options.Minify { return "min" @@ -514,30 +525,10 @@ func (s *Session) GoRelease() string { return compiler.GoRelease(s.options.GOROOT) } -func (s *Session) BuildDir(packagePath string, importPath string, pkgObj string) error { - if s.Watcher != nil { - s.Watcher.Add(packagePath) - } - pkg, err := s.xctx.Import(".", packagePath, 0) - if err != nil { - return err - } - - archive, err := s.BuildPackage(pkg) - if err != nil { - return err - } - if pkgObj == "" { - pkgObj = filepath.Base(packagePath) + ".js" - } - if pkg.IsCommand() && !pkg.UpToDate { - if err := s.WriteCommandPackage(archive, pkgObj); err != nil { - return err - } - } - return nil -} - +// BuildFiles passed to the GopherJS tool as if they were a package. +// +// A ephemeral package will be created with only the provided files. This +// function is intended for use with, for example, `gopherjs run main.go`. func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath string) error { pkg := &PackageData{ Package: &build.Package{ @@ -566,11 +557,18 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath stri return s.WriteCommandPackage(archive, pkgObj) } +// BuildImportPath loads and compiles package with the given import path. +// +// Relative paths are interpreted relative to the current working dir. func (s *Session) BuildImportPath(path string) (*compiler.Archive, error) { _, archive, err := s.buildImportPathWithSrcDir(path, "") return archive, err } +// buildImportPathWithSrcDir builds the package specified by the import path. +// +// Relative import paths are interpreted relative to the passed srcDir. If +// srcDir is empty, current working directory is assumed. func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*PackageData, *compiler.Archive, error) { pkg, err := importWithSrcDir(s.xctx, path, srcDir, 0, s.InstallSuffix()) if s.Watcher != nil && pkg != nil { // add watch even on error @@ -588,6 +586,7 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag return pkg, archive, nil } +// BuildPackage compiles an already loaded package. func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if archive, ok := s.Archives[pkg.ImportPath]; ok { return archive, nil @@ -732,6 +731,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return archive, nil } +// writeLibraryPackage writes a compiled package archive to disk at pkgObj path. func (s *Session) writeLibraryPackage(archive *compiler.Archive, pkgObj string) error { if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil { return err @@ -746,6 +746,7 @@ func (s *Session) writeLibraryPackage(archive *compiler.Archive, pkgObj string) return compiler.WriteArchive(archive, objFile) } +// WriteCommandPackage writes the final JavaScript output file at pkgObj path. func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string) error { if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil { return err @@ -786,6 +787,7 @@ func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string) return compiler.WriteProgramCode(deps, sourceMapFilter, s.GoRelease()) } +// NewMappingCallback creates a new callback for source map generation. func NewMappingCallback(m *sourcemap.Map, goroot, gopath string, localMap bool) func(generatedLine, generatedColumn int, originalPos token.Position) { return func(generatedLine, generatedColumn int, originalPos token.Position) { if !originalPos.IsValid() { @@ -810,6 +812,8 @@ func NewMappingCallback(m *sourcemap.Map, goroot, gopath string, localMap bool) } } +// jsFilesFromDir finds and loads any *.inc.js packages in the build context +// directory. func jsFilesFromDir(bctx *build.Context, dir string) ([]string, error) { files, err := buildutil.ReadDir(bctx, dir) if err != nil { @@ -837,6 +841,8 @@ func hasGopathPrefix(file, gopath string) (hasGopathPrefix bool, prefixLen int) return false, 0 } +// WaitForChange watches file system events and returns if either when one of +// the source files is modified. func (s *Session) WaitForChange() { s.options.PrintSuccess("watching for changes...\n") for { diff --git a/build/context.go b/build/context.go index 3a2279297..5fea2fcdb 100644 --- a/build/context.go +++ b/build/context.go @@ -161,20 +161,25 @@ func (sc simpleCtx) applyPackageTweaks(importPath string, mode build.ImportMode) bctx := sc.bctx switch importPath { case "syscall": - // syscall needs to use a typical GOARCH like amd64 to pick up definitions for _Socklen, BpfInsn, IFNAMSIZ, Timeval, BpfStat, SYS_FCNTL, Flock_t, etc. + // syscall needs to use a typical GOARCH like amd64 to pick up definitions + // for _Socklen, BpfInsn, IFNAMSIZ, Timeval, BpfStat, SYS_FCNTL, Flock_t, + // etc. bctx.GOARCH = build.Default.GOARCH bctx.InstallSuffix += build.Default.GOARCH case "syscall/js": if !sc.isVirtual { - // There are no buildable files in this package upstream, but we need to use files in the virtual directory. + // There are no buildable files in this package upstream, but we need to + // use files in the virtual directory. mode |= build.FindOnly } case "crypto/x509", "os/user": - // These stdlib packages have cgo and non-cgo versions (via build tags); we want the latter. + // These stdlib packages have cgo and non-cgo versions (via build tags); we + // want the latter. bctx.CgoEnabled = false case "github.com/gopherjs/gopherjs/js", "github.com/gopherjs/gopherjs/nosync": - // These packages are already embedded via gopherjspkg.FS virtual filesystem (which can be - // safely vendored). Don't try to use vendor directory to resolve them. + // These packages are already embedded via gopherjspkg.FS virtual filesystem + // (which can be safely vendored). Don't try to use vendor directory to + // resolve them. mode |= build.IgnoreVendor } From 6bdf1b4da452373d42122ceef2035e50fc510ceb Mon Sep 17 00:00:00 2001 From: Nevkontakte Date: Sat, 25 Dec 2021 21:23:04 +0000 Subject: [PATCH 2/3] Apply package content tweaks before returning it from XContext. This ensures that all code paths only see adjusted package sources for packages like `os` or `runtime`. This fixes a bug when `gopherjs build runtime` was trying to build the whole runtime package and causing a failure. --- build/build.go | 43 ------------------------------- build/context.go | 66 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 63 insertions(+), 46 deletions(-) diff --git a/build/build.go b/build/build.go index 57cc01c40..fbbe122a8 100644 --- a/build/build.go +++ b/build/build.go @@ -117,49 +117,6 @@ func importWithSrcDir(xctx XContext, path string, srcDir string, mode build.Impo return nil, err } - switch path { - case "os": - pkg.GoFiles = excludeExecutable(pkg.GoFiles) // Need to exclude executable implementation files, because some of them contain package scope variables that perform (indirectly) syscalls on init. - // Prefer the dirent_${GOOS}.go version, to make the build pass on both linux - // and darwin. - // In the long term, our builds should produce the same output regardless - // of the host OS: https://github.com/gopherjs/gopherjs/issues/693. - pkg.GoFiles = exclude(pkg.GoFiles, "dirent_js.go") - case "runtime": - pkg.GoFiles = []string{} // Package sources are completely replaced in natives. - case "runtime/internal/sys": - pkg.GoFiles = []string{fmt.Sprintf("zgoos_%s.go", xctx.GOOS()), "zversion.go"} - case "runtime/pprof": - pkg.GoFiles = nil - case "internal/poll": - pkg.GoFiles = exclude(pkg.GoFiles, "fd_poll_runtime.go") - case "sync": - // GopherJS completely replaces sync.Pool implementation with a simpler one, - // since it always executes in a single-threaded environment. - pkg.GoFiles = exclude(pkg.GoFiles, "pool.go") - case "crypto/rand": - pkg.GoFiles = []string{"rand.go", "util.go"} - pkg.TestGoFiles = exclude(pkg.TestGoFiles, "rand_linux_test.go") // Don't want linux-specific tests (since linux-specific package files are excluded too). - case "crypto/x509": - // GopherJS doesn't support loading OS root certificates regardless of the - // OS. The substitution below allows to avoid build dependency on Mac OS - // implementation, which won't be used anyway. - // - // Just like above, https://github.com/gopherjs/gopherjs/issues/693 is - // probably the best long-term option. - pkg.GoFiles = include( - exclude(pkg.GoFiles, fmt.Sprintf("root_%s.go", xctx.GOOS())), - "root_unix.go", "root_js.go") - case "syscall/js": - // Reuse upstream tests to ensure conformance, but completely replace - // implementation. - pkg.XTestGoFiles = append(pkg.TestGoFiles, "js_test.go") - } - - if len(pkg.CgoFiles) > 0 { - return nil, &ImportCError{path} - } - if pkg.IsCommand() { pkg.PkgObj = filepath.Join(pkg.BinDir, filepath.Base(pkg.ImportPath)+".js") } diff --git a/build/context.go b/build/context.go index 5fea2fcdb..bd49278e0 100644 --- a/build/context.go +++ b/build/context.go @@ -41,7 +41,7 @@ type simpleCtx struct { // Import implements XContext.Import(). func (sc simpleCtx) Import(importPath string, srcDir string, mode build.ImportMode) (*PackageData, error) { - bctx, mode := sc.applyPackageTweaks(importPath, mode) + bctx, mode := sc.applyPreloadTweaks(importPath, mode) pkg, err := bctx.Import(importPath, srcDir, mode) if err != nil { return nil, err @@ -54,6 +54,12 @@ func (sc simpleCtx) Import(importPath string, srcDir string, mode build.ImportMo if !path.IsAbs(pkg.Dir) { pkg.Dir = mustAbs(pkg.Dir) } + pkg = sc.applyPostloadTweaks(pkg) + + if len(pkg.CgoFiles) > 0 { + return nil, &ImportCError{pkg.ImportPath} + } + return &PackageData{ Package: pkg, IsVirtual: sc.isVirtual, @@ -152,12 +158,12 @@ func (sc simpleCtx) gotool(subcommand string, args ...string) (string, error) { return stdout.String(), nil } -// applyPackageTweaks makes several package-specific adjustments to package importing. +// applyPreloadTweaks makes several package-specific adjustments to package importing. // // Ideally this method would not be necessary, but currently several packages // require special handing in order to be compatible with GopherJS. This method // returns a copy of the build context, keeping the original one intact. -func (sc simpleCtx) applyPackageTweaks(importPath string, mode build.ImportMode) (build.Context, build.ImportMode) { +func (sc simpleCtx) applyPreloadTweaks(importPath string, mode build.ImportMode) (build.Context, build.ImportMode) { bctx := sc.bctx switch importPath { case "syscall": @@ -186,6 +192,60 @@ func (sc simpleCtx) applyPackageTweaks(importPath string, mode build.ImportMode) return bctx, mode } +// applyPostloadTweaks makes adjustments to the contents of the loaded package. +// +// Some of the standard library packages require additional tweaks that are not +// covered by our augmentation logic, for example excluding or including +// particular source files. This method ensures that all such tweaks are applied +// before the package is returned to the caller. +func (sc simpleCtx) applyPostloadTweaks(pkg *build.Package) *build.Package { + if sc.isVirtual { + // GopherJS overlay package sources don't need tweaks to their content, + // since we already control them directly. + return pkg + } + switch pkg.ImportPath { + case "os": + pkg.GoFiles = excludeExecutable(pkg.GoFiles) // Need to exclude executable implementation files, because some of them contain package scope variables that perform (indirectly) syscalls on init. + // Prefer the dirent_${GOOS}.go version, to make the build pass on both linux + // and darwin. + // In the long term, our builds should produce the same output regardless + // of the host OS: https://github.com/gopherjs/gopherjs/issues/693. + pkg.GoFiles = exclude(pkg.GoFiles, "dirent_js.go") + case "runtime": + pkg.GoFiles = []string{} // Package sources are completely replaced in natives. + case "runtime/internal/sys": + pkg.GoFiles = []string{fmt.Sprintf("zgoos_%s.go", sc.GOOS()), "zversion.go"} + case "runtime/pprof": + pkg.GoFiles = nil + case "internal/poll": + pkg.GoFiles = exclude(pkg.GoFiles, "fd_poll_runtime.go") + case "sync": + // GopherJS completely replaces sync.Pool implementation with a simpler one, + // since it always executes in a single-threaded environment. + pkg.GoFiles = exclude(pkg.GoFiles, "pool.go") + case "crypto/rand": + pkg.GoFiles = []string{"rand.go", "util.go"} + pkg.TestGoFiles = exclude(pkg.TestGoFiles, "rand_linux_test.go") // Don't want linux-specific tests (since linux-specific package files are excluded too). + case "crypto/x509": + // GopherJS doesn't support loading OS root certificates regardless of the + // OS. The substitution below allows to avoid build dependency on Mac OS + // implementation, which won't be used anyway. + // + // Just like above, https://github.com/gopherjs/gopherjs/issues/693 is + // probably the best long-term option. + pkg.GoFiles = include( + exclude(pkg.GoFiles, fmt.Sprintf("root_%s.go", sc.GOOS())), + "root_unix.go", "root_js.go") + case "syscall/js": + // Reuse upstream tests to ensure conformance, but completely replace + // implementation. + pkg.XTestGoFiles = append(pkg.TestGoFiles, "js_test.go") + } + + return pkg +} + func (sc simpleCtx) rewritePkgObj(orig string) string { if orig == "" { return orig From 181d09fdea3daa3811e8140a985283976c449f65 Mon Sep 17 00:00:00 2001 From: Nevkontakte Date: Sat, 25 Dec 2021 21:59:07 +0000 Subject: [PATCH 3/3] Update package's list of imports after we filter out sources. This ensures that callers only see imports that actually remain in the sources after we excluded irrelevant parts of the upstream package sources. --- build/build.go | 18 +----------------- build/context.go | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/build/build.go b/build/build.go index fbbe122a8..0f9666f7c 100644 --- a/build/build.go +++ b/build/build.go @@ -564,23 +564,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } for _, importedPkgPath := range pkg.Imports { - // Ignore all imports that aren't mentioned in import specs of pkg. - // For example, this ignores imports such as runtime/internal/sys and runtime/internal/atomic. - ignored := true - for _, pos := range pkg.ImportPos[importedPkgPath] { - importFile := filepath.Base(pos.Filename) - for _, file := range pkg.GoFiles { - if importFile == file { - ignored = false - break - } - } - if !ignored { - break - } - } - - if importedPkgPath == "unsafe" || ignored { + if importedPkgPath == "unsafe" { continue } importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir) diff --git a/build/context.go b/build/context.go index bd49278e0..b980aa7de 100644 --- a/build/context.go +++ b/build/context.go @@ -3,6 +3,7 @@ package build import ( "fmt" "go/build" + "go/token" "net/http" "os" "os/exec" @@ -240,9 +241,14 @@ func (sc simpleCtx) applyPostloadTweaks(pkg *build.Package) *build.Package { case "syscall/js": // Reuse upstream tests to ensure conformance, but completely replace // implementation. - pkg.XTestGoFiles = append(pkg.TestGoFiles, "js_test.go") + pkg.GoFiles = []string{} + pkg.XTestGoFiles = append(pkg.XTestGoFiles, "js_test.go") } + pkg.Imports, pkg.ImportPos = updateImports(pkg.GoFiles, pkg.ImportPos) + pkg.TestImports, pkg.TestImportPos = updateImports(pkg.TestGoFiles, pkg.TestImportPos) + pkg.XTestImports, pkg.XTestImportPos = updateImports(pkg.XTestGoFiles, pkg.XTestImportPos) + return pkg } @@ -389,3 +395,31 @@ func IsPkgNotFound(err error) bool { (strings.Contains(err.Error(), "cannot find package") || // Modules off. strings.Contains(err.Error(), "is not in GOROOT")) // Modules on. } + +// updateImports package's list of import paths to only those present in sources +// after post-load tweaks. +func updateImports(sources []string, importPos map[string][]token.Position) (newImports []string, newImportPos map[string][]token.Position) { + if importPos == nil { + // Short-circuit for tests when no imports are loaded. + return nil, nil + } + sourceSet := map[string]bool{} + for _, source := range sources { + sourceSet[source] = true + } + + newImportPos = map[string][]token.Position{} + for importPath, positions := range importPos { + for _, pos := range positions { + if sourceSet[filepath.Base(pos.Filename)] { + newImportPos[importPath] = append(newImportPos[importPath], pos) + } + } + } + + for importPath := range newImportPos { + newImports = append(newImports, importPath) + } + sort.Strings(newImports) + return newImports, newImportPos +}