diff --git a/build/build.go b/build/build.go index d9c0a2f42..0f9666f7c 100644 --- a/build/build.go +++ b/build/build.go @@ -37,7 +37,7 @@ import ( // DefaultGOROOT is the default GOROOT value for builds. // // It uses the GOPHERJS_GOROOT environment variable if it is set, -// or else the default GOROOT value of the system Go distrubtion. +// or else the default GOROOT value of the system Go distribution. var DefaultGOROOT = func() string { if goroot, ok := os.LookupEnv("GOPHERJS_GOROOT"); ok { // GopherJS-specific GOROOT value takes precedence. @@ -47,6 +47,8 @@ var DefaultGOROOT = func() string { return build.Default.GOROOT }() +// ImportCError is returned when GopherJS attempts to build a package that uses +// CGo. type ImportCError struct { pkgPath string } @@ -115,49 +117,6 @@ func importWithSrcDir(xctx XContext, path string, srcDir string, mode build.Impo return nil, err } - switch path { - case "os": - pkg.GoFiles = excludeExecutable(pkg.GoFiles) // Need to exclude executable implementation files, because some of them contain package scope variables that perform (indirectly) syscalls on init. - // Prefer the dirent_${GOOS}.go version, to make the build pass on both linux - // and darwin. - // In the long term, our builds should produce the same output regardless - // of the host OS: https://github.com/gopherjs/gopherjs/issues/693. - pkg.GoFiles = exclude(pkg.GoFiles, "dirent_js.go") - case "runtime": - pkg.GoFiles = []string{} // Package sources are completely replaced in natives. - case "runtime/internal/sys": - pkg.GoFiles = []string{fmt.Sprintf("zgoos_%s.go", xctx.GOOS()), "zversion.go"} - case "runtime/pprof": - pkg.GoFiles = nil - case "internal/poll": - pkg.GoFiles = exclude(pkg.GoFiles, "fd_poll_runtime.go") - case "sync": - // GopherJS completely replaces sync.Pool implementation with a simpler one, - // since it always executes in a single-threaded environment. - pkg.GoFiles = exclude(pkg.GoFiles, "pool.go") - case "crypto/rand": - pkg.GoFiles = []string{"rand.go", "util.go"} - pkg.TestGoFiles = exclude(pkg.TestGoFiles, "rand_linux_test.go") // Don't want linux-specific tests (since linux-specific package files are excluded too). - case "crypto/x509": - // GopherJS doesn't support loading OS root certificates regardless of the - // OS. The substitution below allows to avoid build dependency on Mac OS - // implementation, which won't be used anyway. - // - // Just like above, https://github.com/gopherjs/gopherjs/issues/693 is - // probably the best long-term option. - pkg.GoFiles = include( - exclude(pkg.GoFiles, fmt.Sprintf("root_%s.go", xctx.GOOS())), - "root_unix.go", "root_js.go") - case "syscall/js": - // Reuse upstream tests to ensure conformance, but completely replace - // implementation. - pkg.XTestGoFiles = append(pkg.TestGoFiles, "js_test.go") - } - - if len(pkg.CgoFiles) > 0 { - return nil, &ImportCError{path} - } - if pkg.IsCommand() { pkg.PkgObj = filepath.Join(pkg.BinDir, filepath.Base(pkg.ImportPath)+".js") } @@ -378,6 +337,7 @@ func parseAndAugment(xctx XContext, pkg *PackageData, isTest bool, fileSet *toke return files, nil } +// Options controls build process behavior. type Options struct { GOROOT string GOPATH string @@ -391,6 +351,7 @@ type Options struct { BuildTags []string } +// PrintError message to the terminal. func (o *Options) PrintError(format string, a ...interface{}) { if o.Color { format = "\x1B[31m" + format + "\x1B[39m" @@ -398,6 +359,7 @@ func (o *Options) PrintError(format string, a ...interface{}) { fmt.Fprintf(os.Stderr, format, a...) } +// PrintSuccess message to the terminal. func (o *Options) PrintSuccess(format string, a ...interface{}) { if o.Color { format = "\x1B[32m" + format + "\x1B[39m" @@ -455,6 +417,10 @@ func (p *PackageData) XTestPackage() *PackageData { } } +// Session manages internal state GopherJS requires to perform a build. +// +// This is the main interface to GopherJS build system. Session lifetime is +// roughly equivalent to a single GopherJS tool invocation. type Session struct { options *Options xctx XContext @@ -463,6 +429,7 @@ type Session struct { Watcher *fsnotify.Watcher } +// NewSession creates a new GopherJS build session. func NewSession(options *Options) (*Session, error) { if options.GOROOT == "" { options.GOROOT = DefaultGOROOT @@ -499,9 +466,10 @@ func NewSession(options *Options) (*Session, error) { return s, nil } -// BuildContext returns the session's build context. +// XContext returns the session's build context. func (s *Session) XContext() XContext { return s.xctx } +// InstallSuffix returns the suffix added to the generated output file. func (s *Session) InstallSuffix() string { if s.options.Minify { return "min" @@ -514,30 +482,10 @@ func (s *Session) GoRelease() string { return compiler.GoRelease(s.options.GOROOT) } -func (s *Session) BuildDir(packagePath string, importPath string, pkgObj string) error { - if s.Watcher != nil { - s.Watcher.Add(packagePath) - } - pkg, err := s.xctx.Import(".", packagePath, 0) - if err != nil { - return err - } - - archive, err := s.BuildPackage(pkg) - if err != nil { - return err - } - if pkgObj == "" { - pkgObj = filepath.Base(packagePath) + ".js" - } - if pkg.IsCommand() && !pkg.UpToDate { - if err := s.WriteCommandPackage(archive, pkgObj); err != nil { - return err - } - } - return nil -} - +// BuildFiles passed to the GopherJS tool as if they were a package. +// +// A ephemeral package will be created with only the provided files. This +// function is intended for use with, for example, `gopherjs run main.go`. func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath string) error { pkg := &PackageData{ Package: &build.Package{ @@ -566,11 +514,18 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, packagePath stri return s.WriteCommandPackage(archive, pkgObj) } +// BuildImportPath loads and compiles package with the given import path. +// +// Relative paths are interpreted relative to the current working dir. func (s *Session) BuildImportPath(path string) (*compiler.Archive, error) { _, archive, err := s.buildImportPathWithSrcDir(path, "") return archive, err } +// buildImportPathWithSrcDir builds the package specified by the import path. +// +// Relative import paths are interpreted relative to the passed srcDir. If +// srcDir is empty, current working directory is assumed. func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*PackageData, *compiler.Archive, error) { pkg, err := importWithSrcDir(s.xctx, path, srcDir, 0, s.InstallSuffix()) if s.Watcher != nil && pkg != nil { // add watch even on error @@ -588,6 +543,7 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag return pkg, archive, nil } +// BuildPackage compiles an already loaded package. func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if archive, ok := s.Archives[pkg.ImportPath]; ok { return archive, nil @@ -608,23 +564,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } for _, importedPkgPath := range pkg.Imports { - // Ignore all imports that aren't mentioned in import specs of pkg. - // For example, this ignores imports such as runtime/internal/sys and runtime/internal/atomic. - ignored := true - for _, pos := range pkg.ImportPos[importedPkgPath] { - importFile := filepath.Base(pos.Filename) - for _, file := range pkg.GoFiles { - if importFile == file { - ignored = false - break - } - } - if !ignored { - break - } - } - - if importedPkgPath == "unsafe" || ignored { + if importedPkgPath == "unsafe" { continue } importedPkg, _, err := s.buildImportPathWithSrcDir(importedPkgPath, pkg.Dir) @@ -732,6 +672,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { return archive, nil } +// writeLibraryPackage writes a compiled package archive to disk at pkgObj path. func (s *Session) writeLibraryPackage(archive *compiler.Archive, pkgObj string) error { if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil { return err @@ -746,6 +687,7 @@ func (s *Session) writeLibraryPackage(archive *compiler.Archive, pkgObj string) return compiler.WriteArchive(archive, objFile) } +// WriteCommandPackage writes the final JavaScript output file at pkgObj path. func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string) error { if err := os.MkdirAll(filepath.Dir(pkgObj), 0777); err != nil { return err @@ -786,6 +728,7 @@ func (s *Session) WriteCommandPackage(archive *compiler.Archive, pkgObj string) return compiler.WriteProgramCode(deps, sourceMapFilter, s.GoRelease()) } +// NewMappingCallback creates a new callback for source map generation. func NewMappingCallback(m *sourcemap.Map, goroot, gopath string, localMap bool) func(generatedLine, generatedColumn int, originalPos token.Position) { return func(generatedLine, generatedColumn int, originalPos token.Position) { if !originalPos.IsValid() { @@ -810,6 +753,8 @@ func NewMappingCallback(m *sourcemap.Map, goroot, gopath string, localMap bool) } } +// jsFilesFromDir finds and loads any *.inc.js packages in the build context +// directory. func jsFilesFromDir(bctx *build.Context, dir string) ([]string, error) { files, err := buildutil.ReadDir(bctx, dir) if err != nil { @@ -837,6 +782,8 @@ func hasGopathPrefix(file, gopath string) (hasGopathPrefix bool, prefixLen int) return false, 0 } +// WaitForChange watches file system events and returns if either when one of +// the source files is modified. func (s *Session) WaitForChange() { s.options.PrintSuccess("watching for changes...\n") for { diff --git a/build/context.go b/build/context.go index 3a2279297..b980aa7de 100644 --- a/build/context.go +++ b/build/context.go @@ -3,6 +3,7 @@ package build import ( "fmt" "go/build" + "go/token" "net/http" "os" "os/exec" @@ -41,7 +42,7 @@ type simpleCtx struct { // Import implements XContext.Import(). func (sc simpleCtx) Import(importPath string, srcDir string, mode build.ImportMode) (*PackageData, error) { - bctx, mode := sc.applyPackageTweaks(importPath, mode) + bctx, mode := sc.applyPreloadTweaks(importPath, mode) pkg, err := bctx.Import(importPath, srcDir, mode) if err != nil { return nil, err @@ -54,6 +55,12 @@ func (sc simpleCtx) Import(importPath string, srcDir string, mode build.ImportMo if !path.IsAbs(pkg.Dir) { pkg.Dir = mustAbs(pkg.Dir) } + pkg = sc.applyPostloadTweaks(pkg) + + if len(pkg.CgoFiles) > 0 { + return nil, &ImportCError{pkg.ImportPath} + } + return &PackageData{ Package: pkg, IsVirtual: sc.isVirtual, @@ -152,35 +159,99 @@ func (sc simpleCtx) gotool(subcommand string, args ...string) (string, error) { return stdout.String(), nil } -// applyPackageTweaks makes several package-specific adjustments to package importing. +// applyPreloadTweaks makes several package-specific adjustments to package importing. // // Ideally this method would not be necessary, but currently several packages // require special handing in order to be compatible with GopherJS. This method // returns a copy of the build context, keeping the original one intact. -func (sc simpleCtx) applyPackageTweaks(importPath string, mode build.ImportMode) (build.Context, build.ImportMode) { +func (sc simpleCtx) applyPreloadTweaks(importPath string, mode build.ImportMode) (build.Context, build.ImportMode) { bctx := sc.bctx switch importPath { case "syscall": - // syscall needs to use a typical GOARCH like amd64 to pick up definitions for _Socklen, BpfInsn, IFNAMSIZ, Timeval, BpfStat, SYS_FCNTL, Flock_t, etc. + // syscall needs to use a typical GOARCH like amd64 to pick up definitions + // for _Socklen, BpfInsn, IFNAMSIZ, Timeval, BpfStat, SYS_FCNTL, Flock_t, + // etc. bctx.GOARCH = build.Default.GOARCH bctx.InstallSuffix += build.Default.GOARCH case "syscall/js": if !sc.isVirtual { - // There are no buildable files in this package upstream, but we need to use files in the virtual directory. + // There are no buildable files in this package upstream, but we need to + // use files in the virtual directory. mode |= build.FindOnly } case "crypto/x509", "os/user": - // These stdlib packages have cgo and non-cgo versions (via build tags); we want the latter. + // These stdlib packages have cgo and non-cgo versions (via build tags); we + // want the latter. bctx.CgoEnabled = false case "github.com/gopherjs/gopherjs/js", "github.com/gopherjs/gopherjs/nosync": - // These packages are already embedded via gopherjspkg.FS virtual filesystem (which can be - // safely vendored). Don't try to use vendor directory to resolve them. + // These packages are already embedded via gopherjspkg.FS virtual filesystem + // (which can be safely vendored). Don't try to use vendor directory to + // resolve them. mode |= build.IgnoreVendor } return bctx, mode } +// applyPostloadTweaks makes adjustments to the contents of the loaded package. +// +// Some of the standard library packages require additional tweaks that are not +// covered by our augmentation logic, for example excluding or including +// particular source files. This method ensures that all such tweaks are applied +// before the package is returned to the caller. +func (sc simpleCtx) applyPostloadTweaks(pkg *build.Package) *build.Package { + if sc.isVirtual { + // GopherJS overlay package sources don't need tweaks to their content, + // since we already control them directly. + return pkg + } + switch pkg.ImportPath { + case "os": + pkg.GoFiles = excludeExecutable(pkg.GoFiles) // Need to exclude executable implementation files, because some of them contain package scope variables that perform (indirectly) syscalls on init. + // Prefer the dirent_${GOOS}.go version, to make the build pass on both linux + // and darwin. + // In the long term, our builds should produce the same output regardless + // of the host OS: https://github.com/gopherjs/gopherjs/issues/693. + pkg.GoFiles = exclude(pkg.GoFiles, "dirent_js.go") + case "runtime": + pkg.GoFiles = []string{} // Package sources are completely replaced in natives. + case "runtime/internal/sys": + pkg.GoFiles = []string{fmt.Sprintf("zgoos_%s.go", sc.GOOS()), "zversion.go"} + case "runtime/pprof": + pkg.GoFiles = nil + case "internal/poll": + pkg.GoFiles = exclude(pkg.GoFiles, "fd_poll_runtime.go") + case "sync": + // GopherJS completely replaces sync.Pool implementation with a simpler one, + // since it always executes in a single-threaded environment. + pkg.GoFiles = exclude(pkg.GoFiles, "pool.go") + case "crypto/rand": + pkg.GoFiles = []string{"rand.go", "util.go"} + pkg.TestGoFiles = exclude(pkg.TestGoFiles, "rand_linux_test.go") // Don't want linux-specific tests (since linux-specific package files are excluded too). + case "crypto/x509": + // GopherJS doesn't support loading OS root certificates regardless of the + // OS. The substitution below allows to avoid build dependency on Mac OS + // implementation, which won't be used anyway. + // + // Just like above, https://github.com/gopherjs/gopherjs/issues/693 is + // probably the best long-term option. + pkg.GoFiles = include( + exclude(pkg.GoFiles, fmt.Sprintf("root_%s.go", sc.GOOS())), + "root_unix.go", "root_js.go") + case "syscall/js": + // Reuse upstream tests to ensure conformance, but completely replace + // implementation. + pkg.GoFiles = []string{} + pkg.XTestGoFiles = append(pkg.XTestGoFiles, "js_test.go") + } + + pkg.Imports, pkg.ImportPos = updateImports(pkg.GoFiles, pkg.ImportPos) + pkg.TestImports, pkg.TestImportPos = updateImports(pkg.TestGoFiles, pkg.TestImportPos) + pkg.XTestImports, pkg.XTestImportPos = updateImports(pkg.XTestGoFiles, pkg.XTestImportPos) + + return pkg +} + func (sc simpleCtx) rewritePkgObj(orig string) string { if orig == "" { return orig @@ -324,3 +395,31 @@ func IsPkgNotFound(err error) bool { (strings.Contains(err.Error(), "cannot find package") || // Modules off. strings.Contains(err.Error(), "is not in GOROOT")) // Modules on. } + +// updateImports package's list of import paths to only those present in sources +// after post-load tweaks. +func updateImports(sources []string, importPos map[string][]token.Position) (newImports []string, newImportPos map[string][]token.Position) { + if importPos == nil { + // Short-circuit for tests when no imports are loaded. + return nil, nil + } + sourceSet := map[string]bool{} + for _, source := range sources { + sourceSet[source] = true + } + + newImportPos = map[string][]token.Position{} + for importPath, positions := range importPos { + for _, pos := range positions { + if sourceSet[filepath.Base(pos.Filename)] { + newImportPos[importPath] = append(newImportPos[importPath], pos) + } + } + } + + for importPath := range newImportPos { + newImports = append(newImports, importPath) + } + sort.Strings(newImports) + return newImports, newImportPos +}