Replace gotool package with go list and buildutil.ExpandPatterns()

gotool package matching logic was based on the old version of go (most likely 1.8 or so), and its patterns were not matching packages from third-party modules. We use `go list` instead to match packages on the physical file system. We also provide a shim for VFS-based build contexts using buildutil package, which should be good enough for the few cases we have.
gopherjs · nevkontakte · Aug 2, 2021 · Jun 21, 2021 · Jul 18, 2021 · Jul 18, 2021
commit c8b3cda0e78f9158aec5daccf8d613e1c53a6b81
diff --git a/build/build_test.go b/build/build_test.go
@@ -8,7 +8,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/kisielk/gotool"
 	"github.com/shurcooL/go/importgraphutil"
 )
 
@@ -64,7 +63,12 @@ func TestNativesDontImportExtraPackages(t *testing.T) {
 	// Then, github.com/gopherjs/gopherjs/build.parseAndAugment(*build.Package) returns []*ast.File.
 	// Those augmented parsed Go files of the package are checked, one file at at time, one import
 	// at a time. Each import is verified to belong in the set of allowed real imports.
-	for _, pkg := range gotool.ImportPaths([]string{"std"}) {
+	matches, matchErr := simpleCtx{bctx: stdOnly}.Match([]string{"std"})
+	if matchErr != nil {
+		t.Fatalf("Failed to list standard library packages: %s", err)
+	}
+	for _, pkg := range matches {
+		t.Logf("Checking package %s...", pkg)
 		// Normal package.
 		{
 			// Import the real normal package, and populate its real import set.

diff --git a/build/context.go b/build/context.go
@@ -4,13 +4,15 @@ import (
 	"fmt"
 	"go/build"
 	"net/http"
+	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"sort"
 	"strings"
 
 	"github.com/gopherjs/gopherjs/compiler"
-	"github.com/kisielk/gotool"
+	"golang.org/x/tools/go/buildutil"
 )
 
 // XContext is an extension of go/build.Context with GopherJS-specifc features.
@@ -27,7 +29,7 @@ type XContext interface {
 	GOOS() string
 
 	// Match explans build patterns into a set of matching import paths (see go help packages).
-	Match(patterns []string) []string
+	Match(patterns []string) ([]string, error)
 }
 
 // simpleCtx is a wrapper around go/build.Context with support for GopherJS-specific
@@ -61,18 +63,95 @@ func (sc simpleCtx) Import(importPath string, srcDir string, mode build.ImportMo
 }
 
 // Match implements XContext.Match.
-func (sc simpleCtx) Match(patterns []string) []string {
-	// TODO(nevkontakte): The gotool library prints warnings directly to stderr
-	// when it matched no packages. This may be misleading with chained contexts
-	// when a package is only found in the secondary context. Perhaps we could
-	// replace gotool package with golang.org/x/tools/go/buildutil.ExpandPatterns()
-	// at the cost of a slightly more limited pattern support compared to go tool?
-	tool := gotool.Context{BuildContext: sc.bctx}
-	return tool.ImportPaths(patterns)
+func (sc simpleCtx) Match(patterns []string) ([]string, error) {
+	if sc.isVirtual {
+		// We can't use go tool to enumerate packages in a virtual file system,
+		// so we fall back onto a simpler implementation provided by the buildutil
+		// package. It doesn't support all valid patterns, but should be good enough.
+		//
+		// Note: this code path will become unnecessary after
+		// https://github.com/gopherjs/gopherjs/issues/1021 is implemented.
+		args := []string{}
+		for _, p := range patterns {
+			switch p {
+			case "all":
+				args = append(args, "...")
+			case "std", "main", "cmd":
+				// These patterns are not supported by buildutil.ExpandPatterns(),
+				// but they would be matched by the real context correctly, so skip them.
+			default:
+				args = append(args, p)
+			}
+		}
+		matches := []string{}
+		for importPath := range buildutil.ExpandPatterns(&sc.bctx, args) {
+			if importPath[0] == '.' {
+				p, err := sc.Import(importPath, ".", build.FindOnly)
+				// Resolve relative patterns into canonical import paths.
+				if err != nil {
+					continue
+				}
+				importPath = p.ImportPath
+			}
+			matches = append(matches, importPath)
+		}
+		sort.Strings(matches)
+		return matches, nil
+	}
+
+	args := append([]string{
+		"-e", "-compiler=gc",
+		"-tags=" + strings.Join(sc.bctx.BuildTags, ","),
+		"-installsuffix=" + sc.bctx.InstallSuffix,
+		"-f={{.ImportPath}}",
+		"--",
+	}, patterns...)
+
+	out, err := sc.gotool("list", args...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list packages on FS: %w", err)
+	}
+	matches := strings.Split(strings.TrimSpace(out), "\n")
+	sort.Strings(matches)
+	return matches, nil
 }
 
 func (sc simpleCtx) GOOS() string { return sc.bctx.GOOS }
 
+// gotool executes the go tool set up for the build context and returns standard output.
+func (sc simpleCtx) gotool(subcommand string, args ...string) (string, error) {
+	if sc.isVirtual {
+		panic(fmt.Errorf("can't use go tool with a virtual build context"))
+	}
+	args = append([]string{subcommand}, args...)
+	cmd := exec.Command("go", args...)
+
+	if sc.bctx.Dir != "" {
+		cmd.Dir = sc.bctx.Dir
+	}
+
+	var stdout, stderr strings.Builder
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	cgo := "0"
+	if sc.bctx.CgoEnabled {
+		cgo = "1"
+	}
+	cmd.Env = append(os.Environ(),
+		"GOOS="+sc.bctx.GOOS,
+		"GOARCH="+sc.bctx.GOARCH,
+		"GOROOT="+sc.bctx.GOROOT,
+		"GOPATH="+sc.bctx.GOPATH,
+		"CGO_ENABLED="+cgo,
+	)
+
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("go tool error: %v: %w\n%s", cmd, err, stderr.String())
+	}
+	return stdout.String(), nil
+}
+
 // applyPackageTweaks makes several package-specific adjustments to package importing.
 //
 // Ideally this method would not be necessary, but currently several packages
@@ -211,18 +290,27 @@ func (cc chainedCtx) GOOS() string { return cc.primary.GOOS() }
 //
 // Packages from both contexts are included and returned as a deduplicated
 // sorted list.
-func (cc chainedCtx) Match(patterns []string) []string {
+func (cc chainedCtx) Match(patterns []string) ([]string, error) {
+	m1, err := cc.primary.Match(patterns)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list packages in the primary context: %s", err)
+	}
+	m2, err := cc.secondary.Match(patterns)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list packages in the secondary context: %s", err)
+	}
+
 	seen := map[string]bool{}
 	matches := []string{}
-	for _, m := range append(cc.primary.Match(patterns), cc.secondary.Match(patterns)...) {
+	for _, m := range append(m1, m2...) {
 		if seen[m] {
 			continue
 		}
 		seen[m] = true
 		matches = append(matches, m)
 	}
 	sort.Strings(matches)
-	return matches
+	return matches, nil
 }
 
 // IsPkgNotFound returns true if the error was caused by package not found.

diff --git a/go.mod b/go.mod
@@ -5,7 +5,6 @@ go 1.16
 require (
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/google/go-cmp v0.5.5
-	github.com/kisielk/gotool v1.0.0
 	github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86
 	github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c
 	github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636

diff --git a/go.sum b/go.sum
@@ -100,7 +100,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=

diff --git a/tool.go b/tool.go
@@ -125,7 +125,10 @@ func main() {
 
 				xctx := gbuild.NewBuildContext(s.InstallSuffix(), options.BuildTags)
 				// Expand import path patterns.
-				pkgs := xctx.Match(args)
+				pkgs, err := xctx.Match(args)
+				if err != nil {
+					return fmt.Errorf("failed to expand patterns %v: %w", args, err)
+				}
 
 				for _, pkgPath := range pkgs {
 					if s.Watcher != nil {
@@ -185,7 +188,10 @@ func main() {
 			err = func() error {
 				// Expand import path patterns.
 				xctx := gbuild.NewBuildContext(s.InstallSuffix(), options.BuildTags)
-				pkgs := xctx.Match(args)
+				pkgs, err := xctx.Match(args)
+				if err != nil {
+					return fmt.Errorf("failed to expand patterns %v: %w", args, err)
+				}
 
 				if cmd.Name() == "get" {
 					goGet := exec.Command("go", append([]string{"get", "-d", "-tags=js"}, pkgs...)...)
@@ -316,17 +322,20 @@ func main() {
 		err := func() error {
 			// Expand import path patterns.
 			patternContext := gbuild.NewBuildContext("", options.BuildTags)
-			args = patternContext.Match(args)
+			matches, err := patternContext.Match(args)
+			if err != nil {
+				return fmt.Errorf("failed to expand patterns %v: %w", args, err)
+			}
 
-			if *compileOnly && len(args) > 1 {
+			if *compileOnly && len(matches) > 1 {
 				return errors.New("cannot use -c flag with multiple packages")
 			}
-			if *outputFilename != "" && len(args) > 1 {
+			if *outputFilename != "" && len(matches) > 1 {
 				return errors.New("cannot use -o flag with multiple packages")
 			}
 
-			pkgs := make([]*gbuild.PackageData, len(args))
-			for i, pkgPath := range args {
+			pkgs := make([]*gbuild.PackageData, len(matches))
+			for i, pkgPath := range matches {
 				var err error
 				pkgs[i], err = gbuild.Import(pkgPath, 0, "", options.BuildTags)
 				if err != nil {