diff --git a/cmd/callgraph/main.go b/cmd/callgraph/main.go index 7853826b8fc..4443f172f7e 100644 --- a/cmd/callgraph/main.go +++ b/cmd/callgraph/main.go @@ -226,7 +226,7 @@ func doCallgraph(dir, gopath, algo, format string, tests bool, args []string) er // NB: RTA gives us Reachable and RuntimeTypes too. case "vta": - cg = vta.CallGraph(ssautil.AllFunctions(prog), cha.CallGraph(prog)) + cg = vta.CallGraph(ssautil.AllFunctions(prog), nil) default: return fmt.Errorf("unknown algorithm: %s", algo) diff --git a/go.mod b/go.mod index 1f4bd3af069..003d83773df 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,14 @@ module golang.org/x/tools -go 1.19 // => default GODEBUG has gotypesalias=0 +go 1.22.0 // => default GODEBUG has gotypesalias=0 require ( github.com/google/go-cmp v0.6.0 github.com/yuin/goldmark v1.4.13 - golang.org/x/mod v0.20.0 - golang.org/x/net v0.28.0 + golang.org/x/mod v0.21.0 + golang.org/x/net v0.29.0 golang.org/x/sync v0.8.0 golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457 ) -require golang.org/x/sys v0.23.0 // indirect +require golang.org/x/sys v0.25.0 // indirect diff --git a/go.sum b/go.sum index 57646cc0f47..eae2ee53cc7 100644 --- a/go.sum +++ b/go.sum @@ -2,13 +2,13 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457 h1:zf5N6UOrA487eEFacMePxjXAJctxKmyjKUsjA11Uzuk= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= diff --git a/go/analysis/passes/copylock/copylock.go b/go/analysis/passes/copylock/copylock.go index 8f6e7db6a27..0d63cd16124 100644 --- a/go/analysis/passes/copylock/copylock.go +++ b/go/analysis/passes/copylock/copylock.go @@ -20,6 +20,7 @@ import ( "golang.org/x/tools/go/ast/inspector" "golang.org/x/tools/internal/aliases" "golang.org/x/tools/internal/typeparams" + "golang.org/x/tools/internal/versions" ) const Doc = `check for locks erroneously passed by value @@ -40,18 +41,25 @@ var Analyzer = &analysis.Analyzer{ func run(pass *analysis.Pass) (interface{}, error) { inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + var goversion string // effective file version ("" => unknown) nodeFilter := []ast.Node{ (*ast.AssignStmt)(nil), (*ast.CallExpr)(nil), (*ast.CompositeLit)(nil), + (*ast.File)(nil), (*ast.FuncDecl)(nil), (*ast.FuncLit)(nil), (*ast.GenDecl)(nil), (*ast.RangeStmt)(nil), (*ast.ReturnStmt)(nil), } - inspect.Preorder(nodeFilter, func(node ast.Node) { + inspect.WithStack(nodeFilter, func(node ast.Node, push bool, stack []ast.Node) bool { + if !push { + return false + } switch node := node.(type) { + case *ast.File: + goversion = versions.FileVersion(pass.TypesInfo, node) case *ast.RangeStmt: checkCopyLocksRange(pass, node) case *ast.FuncDecl: @@ -61,7 +69,7 @@ func run(pass *analysis.Pass) (interface{}, error) { case *ast.CallExpr: checkCopyLocksCallExpr(pass, node) case *ast.AssignStmt: - checkCopyLocksAssign(pass, node) + checkCopyLocksAssign(pass, node, goversion, parent(stack)) case *ast.GenDecl: checkCopyLocksGenDecl(pass, node) case *ast.CompositeLit: @@ -69,16 +77,36 @@ func run(pass *analysis.Pass) (interface{}, error) { case *ast.ReturnStmt: checkCopyLocksReturnStmt(pass, node) } + return true }) return nil, nil } // checkCopyLocksAssign checks whether an assignment // copies a lock. -func checkCopyLocksAssign(pass *analysis.Pass, as *ast.AssignStmt) { - for i, x := range as.Rhs { +func checkCopyLocksAssign(pass *analysis.Pass, assign *ast.AssignStmt, goversion string, parent ast.Node) { + lhs := assign.Lhs + for i, x := range assign.Rhs { if path := lockPathRhs(pass, x); path != nil { - pass.ReportRangef(x, "assignment copies lock value to %v: %v", analysisutil.Format(pass.Fset, as.Lhs[i]), path) + pass.ReportRangef(x, "assignment copies lock value to %v: %v", analysisutil.Format(pass.Fset, assign.Lhs[i]), path) + lhs = nil // An lhs has been reported. We prefer the assignment warning and do not report twice. + } + } + + // After GoVersion 1.22, loop variables are implicitly copied on each iteration. + // So a for statement may inadvertently copy a lock when any of the + // iteration variables contain locks. + if assign.Tok == token.DEFINE && versions.AtLeast(goversion, versions.Go1_22) { + if parent, _ := parent.(*ast.ForStmt); parent != nil && parent.Init == assign { + for _, l := range lhs { + if id, ok := l.(*ast.Ident); ok && id.Name != "_" { + if obj := pass.TypesInfo.Defs[id]; obj != nil && obj.Type() != nil { + if path := lockPath(pass.Pkg, obj.Type(), nil); path != nil { + pass.ReportRangef(l, "for loop iteration copies lock value to %v: %v", analysisutil.Format(pass.Fset, l), path) + } + } + } + } } } } @@ -340,6 +368,14 @@ func lockPath(tpkg *types.Package, typ types.Type, seen map[types.Type]bool) typ return nil } +// parent returns the second from the last node on stack if it exists. +func parent(stack []ast.Node) ast.Node { + if len(stack) >= 2 { + return stack[len(stack)-2] + } + return nil +} + var lockerType *types.Interface // Construct a sync.Locker interface type. diff --git a/go/analysis/passes/copylock/copylock_test.go b/go/analysis/passes/copylock/copylock_test.go index 91bef71979b..c22001ca3ea 100644 --- a/go/analysis/passes/copylock/copylock_test.go +++ b/go/analysis/passes/copylock/copylock_test.go @@ -5,13 +5,28 @@ package copylock_test import ( + "path/filepath" "testing" "golang.org/x/tools/go/analysis/analysistest" "golang.org/x/tools/go/analysis/passes/copylock" + "golang.org/x/tools/internal/testenv" + "golang.org/x/tools/internal/testfiles" ) func Test(t *testing.T) { testdata := analysistest.TestData() analysistest.Run(t, testdata, copylock.Analyzer, "a", "typeparams", "issue67787") } + +func TestVersions22(t *testing.T) { + testenv.NeedsGo1Point(t, 22) + + dir := testfiles.ExtractTxtarFileToTmp(t, filepath.Join(analysistest.TestData(), "src", "forstmt", "go22.txtar")) + analysistest.Run(t, dir, copylock.Analyzer, "golang.org/fake/forstmt") +} + +func TestVersions21(t *testing.T) { + dir := testfiles.ExtractTxtarFileToTmp(t, filepath.Join(analysistest.TestData(), "src", "forstmt", "go21.txtar")) + analysistest.Run(t, dir, copylock.Analyzer, "golang.org/fake/forstmt") +} diff --git a/go/analysis/passes/copylock/testdata/src/forstmt/go21.txtar b/go/analysis/passes/copylock/testdata/src/forstmt/go21.txtar new file mode 100644 index 00000000000..9874f35b8d6 --- /dev/null +++ b/go/analysis/passes/copylock/testdata/src/forstmt/go21.txtar @@ -0,0 +1,73 @@ +Test copylock at go version go1.21. + +-- go.mod -- +module golang.org/fake/forstmt + +go 1.21 +-- pre.go -- +//go:build go1.21 + +package forstmt + +import "sync" + +func InGo21(l []int) { + var mu sync.Mutex + var x int + + for x, mu = 0, (sync.Mutex{}); x < 10; x++ { // Not reported on '='. + } + for x, mu := 0, (sync.Mutex{}); x < 10; x++ { // Not reported before 1.22. + _ = mu.TryLock() + } + for x, _ := 0, (sync.Mutex{}); x < 10; x++ { // Not reported due to '_'. + _ = mu.TryLock() + } + for _, mu := 0, (sync.Mutex{}); x < 10; x++ { // Not reported before 1.22. + _ = mu.TryLock() + } +} +-- go22.go -- +//go:build go1.22 + +package forstmt + +import "sync" + +func InGo22(l []int) { + var mu sync.Mutex + var x int + + for x, mu = 0, (sync.Mutex{}); x < 10; x++ { // Not reported on '='. + } + for x, mu := 0, (sync.Mutex{}); x < 10; x++ { // want "for loop iteration copies lock value to mu: sync.Mutex" + _ = mu.TryLock() + } + for x, _ := 0, (sync.Mutex{}); x < 10; x++ { // Not reported due to '_'. + _ = mu.TryLock() + } + for _, mu := 0, (sync.Mutex{}); x < 10; x++ { // want "for loop iteration copies lock value to mu: sync.Mutex" + _ = mu.TryLock() + } +} +-- modver.go -- +package forstmt + +import "sync" + +func AtGo121ByModuleVersion(l []int) { + var mu sync.Mutex + var x int + + for x, mu = 0, (sync.Mutex{}); x < 10; x++ { // Not reported on '='. + } + for x, mu := 0, (sync.Mutex{}); x < 10; x++ { // Not reported before 1.22. + _ = mu.TryLock() + } + for x, _ := 0, (sync.Mutex{}); x < 10; x++ { // Not reported due to '_'. + _ = mu.TryLock() + } + for _, mu := 0, (sync.Mutex{}); x < 10; x++ { // Not reported before 1.22. + _ = mu.TryLock() + } +} diff --git a/go/analysis/passes/copylock/testdata/src/forstmt/go22.txtar b/go/analysis/passes/copylock/testdata/src/forstmt/go22.txtar new file mode 100644 index 00000000000..d9b287a5aa1 --- /dev/null +++ b/go/analysis/passes/copylock/testdata/src/forstmt/go22.txtar @@ -0,0 +1,87 @@ +Test copylock at go version go1.22. + +-- go.mod -- +module golang.org/fake/forstmt + +go 1.22 +-- pre.go -- +//go:build go1.21 + +package forstmt + +import "sync" + +func InGo21(l []int) { + var mu sync.Mutex + var x int + + for x, mu = 0, (sync.Mutex{}); x < 10; x++ { // Not reported on '='. + } + for x, mu := 0, (sync.Mutex{}); x < 10; x++ { // Not reported before 1.22. + _ = mu.TryLock() + } + for x, _ := 0, (sync.Mutex{}); x < 10; x++ { // Not reported due to '_'. + _ = mu.TryLock() + } + for _, mu := 0, (sync.Mutex{}); x < 10; x++ { // Not reported before 1.22. + _ = mu.TryLock() + } +} +-- go22.go -- +//go:build go1.22 + +package forstmt + +import "sync" + +func InGo22(l []int) { + var mu sync.Mutex + var x int + + for x, mu = 0, (sync.Mutex{}); x < 10; x++ { // Not reported on '='. + } + for x, mu := 0, (sync.Mutex{}); x < 10; x++ { // want "for loop iteration copies lock value to mu: sync.Mutex" + _ = mu.TryLock() + } + for x, _ := 0, (sync.Mutex{}); x < 10; x++ { // Not reported due to '_'. + _ = mu.TryLock() + } + for _, mu := 0, (sync.Mutex{}); x < 10; x++ { // want "for loop iteration copies lock value to mu: sync.Mutex" + _ = mu.TryLock() + } +} +-- modver.go -- +package forstmt + +import "sync" + +func InGo22ByModuleVersion(l []int) { + var mu sync.Mutex + var x int + + for x, mu = 0, (sync.Mutex{}); x < 10; x++ { // Not reported on '='. + } + for x, mu := 0, (sync.Mutex{}); x < 10; x++ { // want "for loop iteration copies lock value to mu: sync.Mutex" + _ = mu.TryLock() + } + for x, _ := 0, (sync.Mutex{}); x < 10; x++ { // Not reported due to '_'. + _ = mu.TryLock() + } + for _, mu := 0, (sync.Mutex{}); x < 10; x++ { // want "for loop iteration copies lock value to mu: sync.Mutex" + _ = mu.TryLock() + } +} +-- assign.go -- +//go:build go1.22 + +package forstmt + +import "sync" + +func ReportAssign(l []int) { + // Test we do not report a duplicate if the assignment is reported. + var mu sync.Mutex + for x, mu := 0, mu; x < 10; x++ { // want "assignment copies lock value to mu: sync.Mutex" + _ = mu.TryLock() + } +} diff --git a/go/analysis/passes/fieldalignment/fieldalignment.go b/go/analysis/passes/fieldalignment/fieldalignment.go index 8af717b4c6c..93fa39140e6 100644 --- a/go/analysis/passes/fieldalignment/fieldalignment.go +++ b/go/analysis/passes/fieldalignment/fieldalignment.go @@ -53,7 +53,7 @@ so the analyzer is not included in typical suites such as vet or gopls. Use this standalone command to run it on your code: $ go install golang.org/x/tools/go/analysis/passes/fieldalignment/cmd/fieldalignment@latest - $ go fieldalignment [packages] + $ fieldalignment [packages] ` diff --git a/go/analysis/passes/loopclosure/loopclosure_test.go b/go/analysis/passes/loopclosure/loopclosure_test.go index 9b32c1495ef..8b282027d41 100644 --- a/go/analysis/passes/loopclosure/loopclosure_test.go +++ b/go/analysis/passes/loopclosure/loopclosure_test.go @@ -24,6 +24,7 @@ func Test(t *testing.T) { } func TestVersions22(t *testing.T) { + t.Skip("Disabled for golang.org/cl/603895. Fix and re-enable.") testenv.NeedsGo1Point(t, 22) dir := testfiles.ExtractTxtarFileToTmp(t, filepath.Join(analysistest.TestData(), "src", "versions", "go22.txtar")) @@ -31,6 +32,7 @@ func TestVersions22(t *testing.T) { } func TestVersions18(t *testing.T) { + t.Skip("Disabled for golang.org/cl/603895. Fix and re-enable.") dir := testfiles.ExtractTxtarFileToTmp(t, filepath.Join(analysistest.TestData(), "src", "versions", "go18.txtar")) analysistest.Run(t, dir, loopclosure.Analyzer, "golang.org/fake/versions") } diff --git a/go/analysis/passes/printf/printf.go b/go/analysis/passes/printf/printf.go index b3453f8fc06..c548cb1c1dc 100644 --- a/go/analysis/passes/printf/printf.go +++ b/go/analysis/passes/printf/printf.go @@ -159,10 +159,11 @@ func maybePrintfWrapper(info *types.Info, decl ast.Decl) *printfWrapper { params := sig.Params() nparams := params.Len() // variadic => nonzero + // Check final parameter is "args ...interface{}". args := params.At(nparams - 1) - iface, ok := args.Type().(*types.Slice).Elem().(*types.Interface) + iface, ok := aliases.Unalias(args.Type().(*types.Slice).Elem()).(*types.Interface) if !ok || !iface.Empty() { - return nil // final (args) param is not ...interface{} + return nil } // Is second last param 'format string'? diff --git a/go/analysis/passes/printf/printf_test.go b/go/analysis/passes/printf/printf_test.go index 3506fec1fc2..b27cef51983 100644 --- a/go/analysis/passes/printf/printf_test.go +++ b/go/analysis/passes/printf/printf_test.go @@ -15,6 +15,7 @@ func Test(t *testing.T) { testdata := analysistest.TestData() printf.Analyzer.Flags.Set("funcs", "Warn,Warnf") - analysistest.Run(t, testdata, printf.Analyzer, "a", "b", "nofmt", "typeparams") + analysistest.Run(t, testdata, printf.Analyzer, + "a", "b", "nofmt", "typeparams", "issue68744") analysistest.RunWithSuggestedFixes(t, testdata, printf.Analyzer, "fix") } diff --git a/go/analysis/passes/printf/testdata/src/issue68744/issue68744.go b/go/analysis/passes/printf/testdata/src/issue68744/issue68744.go new file mode 100644 index 00000000000..79922ffbaaa --- /dev/null +++ b/go/analysis/passes/printf/testdata/src/issue68744/issue68744.go @@ -0,0 +1,13 @@ +package issue68744 + +import "fmt" + +// The use of "any" here is crucial to exercise the bug. +// (None of our earlier tests covered this vital detail!) +func wrapf(format string, args ...any) { // want wrapf:"printfWrapper" + fmt.Printf(format, args...) +} + +func _() { + wrapf("%s", 123) // want `issue68744.wrapf format %s has arg 123 of wrong type int` +} diff --git a/go/analysis/passes/stdversion/stdversion_test.go b/go/analysis/passes/stdversion/stdversion_test.go index 7b2f72de81b..e1f71fac3f5 100644 --- a/go/analysis/passes/stdversion/stdversion_test.go +++ b/go/analysis/passes/stdversion/stdversion_test.go @@ -15,6 +15,7 @@ import ( ) func Test(t *testing.T) { + t.Skip("Disabled for golang.org/cl/603895. Fix and re-enable.") // The test relies on go1.21 std symbols, but the analyzer // itself requires the go1.22 implementation of versions.FileVersions. testenv.NeedsGo1Point(t, 22) diff --git a/go/callgraph/callgraph.go b/go/callgraph/callgraph.go index a1b0ca5da36..cfbe5047efd 100644 --- a/go/callgraph/callgraph.go +++ b/go/callgraph/callgraph.go @@ -32,11 +32,6 @@ language. */ package callgraph // import "golang.org/x/tools/go/callgraph" -// TODO(adonovan): add a function to eliminate wrappers from the -// callgraph, preserving topology. -// More generally, we could eliminate "uninteresting" nodes such as -// nodes from packages we don't care about. - // TODO(zpavlinovic): decide how callgraphs handle calls to and from generic function bodies. import ( @@ -52,11 +47,11 @@ import ( // If the call graph is sound, such nodes indicate unreachable // functions. type Graph struct { - Root *Node // the distinguished root node + Root *Node // the distinguished root node (Root.Func may be nil) Nodes map[*ssa.Function]*Node // all nodes by function } -// New returns a new Graph with the specified root node. +// New returns a new Graph with the specified (optional) root node. func New(root *ssa.Function) *Graph { g := &Graph{Nodes: make(map[*ssa.Function]*Node)} g.Root = g.CreateNode(root) diff --git a/go/callgraph/cha/cha.go b/go/callgraph/cha/cha.go index 3040f3d8bbc..67a03563602 100644 --- a/go/callgraph/cha/cha.go +++ b/go/callgraph/cha/cha.go @@ -25,12 +25,10 @@ package cha // import "golang.org/x/tools/go/callgraph/cha" // TODO(zpavlinovic): update CHA for how it handles generic function bodies. import ( - "go/types" - "golang.org/x/tools/go/callgraph" + "golang.org/x/tools/go/callgraph/internal/chautil" "golang.org/x/tools/go/ssa" "golang.org/x/tools/go/ssa/ssautil" - "golang.org/x/tools/go/types/typeutil" ) // CallGraph computes the call graph of the specified program using the @@ -53,13 +51,6 @@ func CallGraph(prog *ssa.Program) *callgraph.Graph { // (io.Writer).Write is assumed to call every concrete // Write method in the program, the call graph can // contain a lot of duplication. - // - // TODO(taking): opt: consider making lazyCallees public. - // Using the same benchmarks as callgraph_test.go, removing just - // the explicit callgraph.Graph construction is 4x less memory - // and is 37% faster. - // CHA 86 ms/op 16 MB/op - // lazyCallees 63 ms/op 4 MB/op for _, g := range callees { addEdge(fnode, site, g) } @@ -83,82 +74,4 @@ func CallGraph(prog *ssa.Program) *callgraph.Graph { return cg } -// lazyCallees returns a function that maps a call site (in a function in fns) -// to its callees within fns. -// -// The resulting function is not concurrency safe. -func lazyCallees(fns map[*ssa.Function]bool) func(site ssa.CallInstruction) []*ssa.Function { - // funcsBySig contains all functions, keyed by signature. It is - // the effective set of address-taken functions used to resolve - // a dynamic call of a particular signature. - var funcsBySig typeutil.Map // value is []*ssa.Function - - // methodsByID contains all methods, grouped by ID for efficient - // lookup. - // - // We must key by ID, not name, for correct resolution of interface - // calls to a type with two (unexported) methods spelled the same but - // from different packages. The fact that the concrete type implements - // the interface does not mean the call dispatches to both methods. - methodsByID := make(map[string][]*ssa.Function) - - // An imethod represents an interface method I.m. - // (There's no go/types object for it; - // a *types.Func may be shared by many interfaces due to interface embedding.) - type imethod struct { - I *types.Interface - id string - } - // methodsMemo records, for every abstract method call I.m on - // interface type I, the set of concrete methods C.m of all - // types C that satisfy interface I. - // - // Abstract methods may be shared by several interfaces, - // hence we must pass I explicitly, not guess from m. - // - // methodsMemo is just a cache, so it needn't be a typeutil.Map. - methodsMemo := make(map[imethod][]*ssa.Function) - lookupMethods := func(I *types.Interface, m *types.Func) []*ssa.Function { - id := m.Id() - methods, ok := methodsMemo[imethod{I, id}] - if !ok { - for _, f := range methodsByID[id] { - C := f.Signature.Recv().Type() // named or *named - if types.Implements(C, I) { - methods = append(methods, f) - } - } - methodsMemo[imethod{I, id}] = methods - } - return methods - } - - for f := range fns { - if f.Signature.Recv() == nil { - // Package initializers can never be address-taken. - if f.Name() == "init" && f.Synthetic == "package initializer" { - continue - } - funcs, _ := funcsBySig.At(f.Signature).([]*ssa.Function) - funcs = append(funcs, f) - funcsBySig.Set(f.Signature, funcs) - } else if obj := f.Object(); obj != nil { - id := obj.(*types.Func).Id() - methodsByID[id] = append(methodsByID[id], f) - } - } - - return func(site ssa.CallInstruction) []*ssa.Function { - call := site.Common() - if call.IsInvoke() { - tiface := call.Value.Type().Underlying().(*types.Interface) - return lookupMethods(tiface, call.Method) - } else if g := call.StaticCallee(); g != nil { - return []*ssa.Function{g} - } else if _, ok := call.Value.(*ssa.Builtin); !ok { - fns, _ := funcsBySig.At(call.Signature()).([]*ssa.Function) - return fns - } - return nil - } -} +var lazyCallees = chautil.LazyCallees diff --git a/go/callgraph/internal/chautil/lazy.go b/go/callgraph/internal/chautil/lazy.go new file mode 100644 index 00000000000..430bfea4564 --- /dev/null +++ b/go/callgraph/internal/chautil/lazy.go @@ -0,0 +1,96 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package chautil provides helper functions related to +// class hierarchy analysis (CHA) for use in x/tools. +package chautil + +import ( + "go/types" + + "golang.org/x/tools/go/ssa" + "golang.org/x/tools/go/types/typeutil" +) + +// LazyCallees returns a function that maps a call site (in a function in fns) +// to its callees within fns. The set of callees is computed using the CHA algorithm, +// i.e., on the entire implements relation between interfaces and concrete types +// in fns. Please see golang.org/x/tools/go/callgraph/cha for more information. +// +// The resulting function is not concurrency safe. +func LazyCallees(fns map[*ssa.Function]bool) func(site ssa.CallInstruction) []*ssa.Function { + // funcsBySig contains all functions, keyed by signature. It is + // the effective set of address-taken functions used to resolve + // a dynamic call of a particular signature. + var funcsBySig typeutil.Map // value is []*ssa.Function + + // methodsByID contains all methods, grouped by ID for efficient + // lookup. + // + // We must key by ID, not name, for correct resolution of interface + // calls to a type with two (unexported) methods spelled the same but + // from different packages. The fact that the concrete type implements + // the interface does not mean the call dispatches to both methods. + methodsByID := make(map[string][]*ssa.Function) + + // An imethod represents an interface method I.m. + // (There's no go/types object for it; + // a *types.Func may be shared by many interfaces due to interface embedding.) + type imethod struct { + I *types.Interface + id string + } + // methodsMemo records, for every abstract method call I.m on + // interface type I, the set of concrete methods C.m of all + // types C that satisfy interface I. + // + // Abstract methods may be shared by several interfaces, + // hence we must pass I explicitly, not guess from m. + // + // methodsMemo is just a cache, so it needn't be a typeutil.Map. + methodsMemo := make(map[imethod][]*ssa.Function) + lookupMethods := func(I *types.Interface, m *types.Func) []*ssa.Function { + id := m.Id() + methods, ok := methodsMemo[imethod{I, id}] + if !ok { + for _, f := range methodsByID[id] { + C := f.Signature.Recv().Type() // named or *named + if types.Implements(C, I) { + methods = append(methods, f) + } + } + methodsMemo[imethod{I, id}] = methods + } + return methods + } + + for f := range fns { + if f.Signature.Recv() == nil { + // Package initializers can never be address-taken. + if f.Name() == "init" && f.Synthetic == "package initializer" { + continue + } + funcs, _ := funcsBySig.At(f.Signature).([]*ssa.Function) + funcs = append(funcs, f) + funcsBySig.Set(f.Signature, funcs) + } else if obj := f.Object(); obj != nil { + id := obj.(*types.Func).Id() + methodsByID[id] = append(methodsByID[id], f) + } + } + + return func(site ssa.CallInstruction) []*ssa.Function { + call := site.Common() + if call.IsInvoke() { + tiface := call.Value.Type().Underlying().(*types.Interface) + return lookupMethods(tiface, call.Method) + } else if g := call.StaticCallee(); g != nil { + return []*ssa.Function{g} + } else if _, ok := call.Value.(*ssa.Builtin); !ok { + fns, _ := funcsBySig.At(call.Signature()).([]*ssa.Function) + return fns + } + return nil + } +} diff --git a/go/callgraph/rta/rta_test.go b/go/callgraph/rta/rta_test.go index 8552dc7b13c..49e21330738 100644 --- a/go/callgraph/rta/rta_test.go +++ b/go/callgraph/rta/rta_test.go @@ -12,7 +12,6 @@ package rta_test import ( "fmt" "go/ast" - "go/parser" "go/types" "sort" "strings" @@ -20,39 +19,60 @@ import ( "golang.org/x/tools/go/callgraph" "golang.org/x/tools/go/callgraph/rta" - "golang.org/x/tools/go/loader" + "golang.org/x/tools/go/packages" "golang.org/x/tools/go/ssa" "golang.org/x/tools/go/ssa/ssautil" "golang.org/x/tools/internal/aliases" + "golang.org/x/tools/internal/testenv" + "golang.org/x/tools/internal/testfiles" + "golang.org/x/tools/txtar" ) -// TestRTA runs RTA on each testdata/*.go file and compares the -// results with the expectations expressed in the WANT comment. +// TestRTA runs RTA on each testdata/*.txtar file containing a single +// go file in a single package or multiple files in different packages, +// and compares the results with the expectations expressed in the WANT +// comment. func TestRTA(t *testing.T) { - filenames := []string{ - "testdata/func.go", - "testdata/generics.go", - "testdata/iface.go", - "testdata/reflectcall.go", - "testdata/rtype.go", + archivePaths := []string{ + "testdata/func.txtar", + "testdata/generics.txtar", + "testdata/iface.txtar", + "testdata/reflectcall.txtar", + "testdata/rtype.txtar", + "testdata/multipkgs.txtar", } - for _, filename := range filenames { - t.Run(filename, func(t *testing.T) { - // Load main program and build SSA. - // TODO(adonovan): use go/packages instead. - conf := loader.Config{ParserMode: parser.ParseComments} - f, err := conf.ParseFile(filename, nil) - if err != nil { - t.Fatal(err) + for _, archive := range archivePaths { + t.Run(archive, func(t *testing.T) { + pkgs := loadPackages(t, archive) + + // find the file which contains the expected result + var f *ast.File + for _, p := range pkgs { + // We assume the packages have a single file or + // the wanted result is in the first file of the main package. + if p.Name == "main" { + f = p.Syntax[0] + } } - conf.CreateFromFiles("main", f) - lprog, err := conf.Load() - if err != nil { - t.Fatal(err) + if f == nil { + t.Fatalf("failed to find the file with expected result within main package %s", archive) } - prog := ssautil.CreateProgram(lprog, ssa.InstantiateGenerics) + + prog, spkgs := ssautil.Packages(pkgs, ssa.SanityCheckFunctions|ssa.InstantiateGenerics) + + // find the main package to get functions for rta analysis + var mainPkg *ssa.Package + for _, sp := range spkgs { + if sp.Pkg.Name() == "main" { + mainPkg = sp + break + } + } + if mainPkg == nil { + t.Fatalf("failed to find main ssa package %s", archive) + } + prog.Build() - mainPkg := prog.Package(lprog.Created[0].Pkg) res := rta.Analyze([]*ssa.Function{ mainPkg.Func("main"), @@ -64,6 +84,42 @@ func TestRTA(t *testing.T) { } } +// loadPackages unpacks the archive to a temporary directory and loads all packages within it. +func loadPackages(t *testing.T, archive string) []*packages.Package { + testenv.NeedsGoPackages(t) + + ar, err := txtar.ParseFile(archive) + if err != nil { + t.Fatal(err) + } + + fs, err := txtar.FS(ar) + if err != nil { + t.Fatal(err) + } + dir := testfiles.CopyToTmp(t, fs) + + var baseConfig = &packages.Config{ + Mode: packages.NeedSyntax | + packages.NeedTypesInfo | + packages.NeedDeps | + packages.NeedName | + packages.NeedFiles | + packages.NeedImports | + packages.NeedCompiledGoFiles | + packages.NeedTypes, + Dir: dir, + } + pkgs, err := packages.Load(baseConfig, "./...") + if err != nil { + t.Fatal(err) + } + if num := packages.PrintErrors(pkgs); num > 0 { + t.Fatalf("packages contained %d errors", num) + } + return pkgs +} + // check tests the RTA analysis results against the test expectations // defined by a comment starting with a line "WANT:". // diff --git a/go/callgraph/rta/testdata/func.go b/go/callgraph/rta/testdata/func.txtar similarity index 88% rename from go/callgraph/rta/testdata/func.go rename to go/callgraph/rta/testdata/func.txtar index bcdcb6ebf90..57930a40cb3 100644 --- a/go/callgraph/rta/testdata/func.go +++ b/go/callgraph/rta/testdata/func.txtar @@ -1,6 +1,8 @@ -//go:build ignore -// +build ignore +-- go.mod -- +module example.com +go 1.18 +-- func.go -- package main // Test of dynamic function calls. @@ -36,4 +38,4 @@ func main() { // reachable init$1 // reachable init$2 // !reachable B -// reachable main +// reachable main \ No newline at end of file diff --git a/go/callgraph/rta/testdata/generics.go b/go/callgraph/rta/testdata/generics.txtar similarity index 56% rename from go/callgraph/rta/testdata/generics.go rename to go/callgraph/rta/testdata/generics.txtar index 17ed6b58e0c..b8039742110 100644 --- a/go/callgraph/rta/testdata/generics.go +++ b/go/callgraph/rta/testdata/generics.txtar @@ -1,6 +1,8 @@ -//go:build ignore -// +build ignore +-- go.mod -- +module example.com +go 1.18 +-- generics.go -- package main // Test of generic function calls. @@ -53,27 +55,27 @@ func lambda[X I]() func() func() { // // edge (*C).Foo --static method call--> (C).Foo // edge (A).Foo$bound --static method call--> (A).Foo -// edge instantiated[main.A] --static method call--> (A).Foo -// edge instantiated[main.B] --static method call--> (B).Foo +// edge instantiated[example.com.A] --static method call--> (A).Foo +// edge instantiated[example.com.B] --static method call--> (B).Foo // edge main --dynamic method call--> (*C).Foo // edge main --dynamic function call--> (A).Foo$bound // edge main --dynamic method call--> (C).Foo -// edge main --static function call--> instantiated[main.A] -// edge main --static function call--> instantiated[main.B] -// edge main --static function call--> lambda[main.A] -// edge main --dynamic function call--> lambda[main.A]$1 -// edge main --static function call--> local[main.C] +// edge main --static function call--> instantiated[example.com.A] +// edge main --static function call--> instantiated[example.com.B] +// edge main --static function call--> lambda[example.com.A] +// edge main --dynamic function call--> lambda[example.com.A]$1 +// edge main --static function call--> local[example.com.C] // // reachable (*C).Foo // reachable (A).Foo // reachable (A).Foo$bound // reachable (B).Foo // reachable (C).Foo -// reachable instantiated[main.A] -// reachable instantiated[main.B] -// reachable lambda[main.A] -// reachable lambda[main.A]$1 -// reachable local[main.C] +// reachable instantiated[example.com.A] +// reachable instantiated[example.com.B] +// reachable lambda[example.com.A] +// reachable lambda[example.com.A]$1 +// reachable local[example.com.C] // // rtype *C // rtype C diff --git a/go/callgraph/rta/testdata/iface.go b/go/callgraph/rta/testdata/iface.txtar similarity index 96% rename from go/callgraph/rta/testdata/iface.go rename to go/callgraph/rta/testdata/iface.txtar index c559204581e..ceb0140a238 100644 --- a/go/callgraph/rta/testdata/iface.go +++ b/go/callgraph/rta/testdata/iface.txtar @@ -1,6 +1,8 @@ -//go:build ignore -// +build ignore +-- go.mod -- +module example.com +go 1.18 +-- iface.go -- package main // Test of interface calls. diff --git a/go/callgraph/rta/testdata/multipkgs.txtar b/go/callgraph/rta/testdata/multipkgs.txtar new file mode 100644 index 00000000000..908fea00563 --- /dev/null +++ b/go/callgraph/rta/testdata/multipkgs.txtar @@ -0,0 +1,106 @@ +-- go.mod -- +module example.com +go 1.18 + +-- iface.go -- +package main + +import ( + "example.com/subpkg" +) + +func use(interface{}) + +// Test of interface calls. + +func main() { + use(subpkg.A(0)) + use(new(subpkg.B)) + use(subpkg.B2(0)) + + var i interface { + F() + } + + // assign an interface type with a function return interface value + i = subpkg.NewInterfaceF() + + i.F() +} + +func dead() { + use(subpkg.D(0)) +} + +// WANT: +// +// edge (*example.com/subpkg.A).F --static method call--> (example.com/subpkg.A).F +// edge (*example.com/subpkg.B2).F --static method call--> (example.com/subpkg.B2).F +// edge (*example.com/subpkg.C).F --static method call--> (example.com/subpkg.C).F +// edge init --static function call--> example.com/subpkg.init +// edge main --dynamic method call--> (*example.com/subpkg.A).F +// edge main --dynamic method call--> (*example.com/subpkg.B).F +// edge main --dynamic method call--> (*example.com/subpkg.B2).F +// edge main --dynamic method call--> (*example.com/subpkg.C).F +// edge main --dynamic method call--> (example.com/subpkg.A).F +// edge main --dynamic method call--> (example.com/subpkg.B2).F +// edge main --dynamic method call--> (example.com/subpkg.C).F +// edge main --static function call--> example.com/subpkg.NewInterfaceF +// edge main --static function call--> use +// +// reachable (*example.com/subpkg.A).F +// reachable (*example.com/subpkg.B).F +// reachable (*example.com/subpkg.B2).F +// reachable (*example.com/subpkg.C).F +// reachable (example.com/subpkg.A).F +// !reachable (example.com/subpkg.B).F +// reachable (example.com/subpkg.B2).F +// reachable (example.com/subpkg.C).F +// reachable example.com/subpkg.NewInterfaceF +// reachable example.com/subpkg.init +// !reachable (*example.com/subpkg.D).F +// !reachable (example.com/subpkg.D).F +// reachable init +// reachable main +// reachable use +// +// rtype *example.com/subpkg.A +// rtype *example.com/subpkg.B +// rtype *example.com/subpkg.B2 +// rtype *example.com/subpkg.C +// rtype example.com/subpkg.B +// rtype example.com/subpkg.A +// rtype example.com/subpkg.B2 +// rtype example.com/subpkg.C +// !rtype example.com/subpkg.D + +-- subpkg/impl.go -- +package subpkg + +type InterfaceF interface { + F() +} + +type A byte // instantiated but not a reflect type + +func (A) F() {} // reachable: exported method of reflect type + +type B int // a reflect type + +func (*B) F() {} // reachable: exported method of reflect type + +type B2 int // a reflect type, and *B2 also + +func (B2) F() {} // reachable: exported method of reflect type + +type C string + +func (C) F() {} // reachable: exported by NewInterfaceF + +func NewInterfaceF() InterfaceF { + return C("") +} + +type D uint // instantiated only in dead code + +func (*D) F() {} // unreachable \ No newline at end of file diff --git a/go/callgraph/rta/testdata/reflectcall.go b/go/callgraph/rta/testdata/reflectcall.txtar similarity index 95% rename from go/callgraph/rta/testdata/reflectcall.go rename to go/callgraph/rta/testdata/reflectcall.txtar index 8f71fb58303..67cd290d479 100644 --- a/go/callgraph/rta/testdata/reflectcall.go +++ b/go/callgraph/rta/testdata/reflectcall.txtar @@ -1,6 +1,8 @@ -//go:build ignore -// +build ignore +-- go.mod -- +module example.com +go 1.18 +-- reflectcall.go -- // Test of a reflective call to an address-taken function. // // Dynamically, this program executes both print statements. diff --git a/go/callgraph/rta/testdata/rtype.go b/go/callgraph/rta/testdata/rtype.txtar similarity index 92% rename from go/callgraph/rta/testdata/rtype.go rename to go/callgraph/rta/testdata/rtype.txtar index 6d84e0342bf..377bc1f7c8c 100644 --- a/go/callgraph/rta/testdata/rtype.go +++ b/go/callgraph/rta/testdata/rtype.txtar @@ -1,6 +1,8 @@ -//go:build ignore -// +build ignore +-- go.mod -- +module example.com +go 1.18 +-- rtype.go -- package main // Test of runtime types (types for which descriptors are needed). diff --git a/go/callgraph/static/static.go b/go/callgraph/static/static.go index 62d2364bf2c..948ce9a3241 100644 --- a/go/callgraph/static/static.go +++ b/go/callgraph/static/static.go @@ -4,32 +4,95 @@ // Package static computes the call graph of a Go program containing // only static call edges. -package static // import "golang.org/x/tools/go/callgraph/static" - -// TODO(zpavlinovic): update static for how it handles generic function bodies. +package static import ( + "go/types" + "golang.org/x/tools/go/callgraph" "golang.org/x/tools/go/ssa" - "golang.org/x/tools/go/ssa/ssautil" ) -// CallGraph computes the call graph of the specified program -// considering only static calls. +// CallGraph computes the static call graph of the specified program. +// +// The resulting graph includes: +// - all package-level functions; +// - all methods of package-level non-parameterized non-interface types; +// - pointer wrappers (*C).F for source-level methods C.F; +// - and all functions reachable from them following only static calls. +// +// It does not consider exportedness, nor treat main packages specially. func CallGraph(prog *ssa.Program) *callgraph.Graph { - cg := callgraph.New(nil) // TODO(adonovan) eliminate concept of rooted callgraph - - // TODO(adonovan): opt: use only a single pass over the ssa.Program. - // TODO(adonovan): opt: this is slower than RTA (perhaps because - // the lower precision means so many edges are allocated)! - for f := range ssautil.AllFunctions(prog) { - fnode := cg.CreateNode(f) - for _, b := range f.Blocks { - for _, instr := range b.Instrs { - if site, ok := instr.(ssa.CallInstruction); ok { - if g := site.Common().StaticCallee(); g != nil { - gnode := cg.CreateNode(g) - callgraph.AddEdge(fnode, site, gnode) + cg := callgraph.New(nil) + + // Recursively follow all static calls. + seen := make(map[int]bool) // node IDs already seen + var visit func(fnode *callgraph.Node) + visit = func(fnode *callgraph.Node) { + if !seen[fnode.ID] { + seen[fnode.ID] = true + + for _, b := range fnode.Func.Blocks { + for _, instr := range b.Instrs { + if site, ok := instr.(ssa.CallInstruction); ok { + if g := site.Common().StaticCallee(); g != nil { + gnode := cg.CreateNode(g) + callgraph.AddEdge(fnode, site, gnode) + visit(gnode) + } + } + } + } + } + } + + // If we were ever to redesign this function, we should allow + // the caller to provide the set of root functions and just + // perform the reachability step. This would allow them to + // work forwards from main entry points: + // + // rootNames := []string{"init", "main"} + // for _, main := range ssautil.MainPackages(prog.AllPackages()) { + // for _, rootName := range rootNames { + // visit(cg.CreateNode(main.Func(rootName))) + // } + // } + // + // or to control whether to include non-exported + // functions/methods, wrapper methods, and so on. + // Unfortunately that's not consistent with its historical + // behavior and existing tests. + // + // The logic below is a slight simplification and + // rationalization of ssautil.AllFunctions. (Having to include + // (*T).F wrapper methods is unfortunate--they are not source + // functions, and if they're reachable, they'll be in the + // graph--but the existing tests will break without it.) + + methodsOf := func(T types.Type) { + if !types.IsInterface(T) { + mset := prog.MethodSets.MethodSet(T) + for i := 0; i < mset.Len(); i++ { + visit(cg.CreateNode(prog.MethodValue(mset.At(i)))) + } + } + } + + // Start from package-level symbols. + for _, pkg := range prog.AllPackages() { + for _, mem := range pkg.Members { + switch mem := mem.(type) { + case *ssa.Function: + // package-level function + visit(cg.CreateNode(mem)) + + case *ssa.Type: + // methods of package-level non-interface non-parameterized types + if !types.IsInterface(mem.Type()) { + if named, ok := mem.Type().(*types.Named); ok && + named.TypeParams() == nil { + methodsOf(named) // T + methodsOf(types.NewPointer(named)) // *T } } } diff --git a/go/callgraph/static/static_test.go b/go/callgraph/static/static_test.go index 4b61dbffa27..cf8392d2f7b 100644 --- a/go/callgraph/static/static_test.go +++ b/go/callgraph/static/static_test.go @@ -18,7 +18,7 @@ import ( "golang.org/x/tools/go/ssa/ssautil" ) -const input = `package P +const input = `package main type C int func (C) f() @@ -46,6 +46,9 @@ func g() { func h() var unknown bool + +func main() { +} ` const genericsInput = `package P diff --git a/go/callgraph/vta/graph.go b/go/callgraph/vta/graph.go index be117f6b736..1a9ed7cb321 100644 --- a/go/callgraph/vta/graph.go +++ b/go/callgraph/vta/graph.go @@ -9,7 +9,6 @@ import ( "go/token" "go/types" - "golang.org/x/tools/go/callgraph" "golang.org/x/tools/go/ssa" "golang.org/x/tools/go/types/typeutil" "golang.org/x/tools/internal/aliases" @@ -172,6 +171,26 @@ func (f function) String() string { return fmt.Sprintf("Function(%s)", f.f.Name()) } +// resultVar represents the result +// variable of a function, whether +// named or not. +type resultVar struct { + f *ssa.Function + index int // valid index into result var tuple +} + +func (o resultVar) Type() types.Type { + return o.f.Signature.Results().At(o.index).Type() +} + +func (o resultVar) String() string { + v := o.f.Signature.Results().At(o.index) + if n := v.Name(); n != "" { + return fmt.Sprintf("Return(%s[%s])", o.f.Name(), n) + } + return fmt.Sprintf("Return(%s[%d])", o.f.Name(), o.index) +} + // nestedPtrInterface node represents all references and dereferences // of locals and globals that have a nested pointer to interface type. // We merge such constructs into a single node for simplicity and without @@ -235,25 +254,27 @@ func (r recoverReturn) String() string { return "Recover" } +type empty = struct{} + // vtaGraph remembers for each VTA node the set of its successors. // Tailored for VTA, hence does not support singleton (sub)graphs. -type vtaGraph map[node]map[node]bool +type vtaGraph map[node]map[node]empty // addEdge adds an edge x->y to the graph. func (g vtaGraph) addEdge(x, y node) { succs, ok := g[x] if !ok { - succs = make(map[node]bool) + succs = make(map[node]empty) g[x] = succs } - succs[y] = true + succs[y] = empty{} } // typePropGraph builds a VTA graph for a set of `funcs` and initial // `callgraph` needed to establish interprocedural edges. Returns the // graph and a map for unique type representatives. -func typePropGraph(funcs map[*ssa.Function]bool, callgraph *callgraph.Graph) (vtaGraph, *typeutil.Map) { - b := builder{graph: make(vtaGraph), callGraph: callgraph} +func typePropGraph(funcs map[*ssa.Function]bool, callees calleesFunc) (vtaGraph, *typeutil.Map) { + b := builder{graph: make(vtaGraph), callees: callees} b.visit(funcs) return b.graph, &b.canon } @@ -261,8 +282,8 @@ func typePropGraph(funcs map[*ssa.Function]bool, callgraph *callgraph.Graph) (vt // Data structure responsible for linearly traversing the // code and building a VTA graph. type builder struct { - graph vtaGraph - callGraph *callgraph.Graph // initial call graph for creating flows at unresolved call sites. + graph vtaGraph + callees calleesFunc // initial call graph for creating flows at unresolved call sites. // Specialized type map for canonicalization of types.Type. // Semantically equivalent types can have different implementations, @@ -576,8 +597,26 @@ func (b *builder) call(c ssa.CallInstruction) { return } - siteCallees(c, b.callGraph)(func(f *ssa.Function) bool { + siteCallees(c, b.callees)(func(f *ssa.Function) bool { addArgumentFlows(b, c, f) + + site, ok := c.(ssa.Value) + if !ok { + return true // go or defer + } + + results := f.Signature.Results() + if results.Len() == 1 { + // When there is only one return value, the destination register does not + // have a tuple type. + b.addInFlowEdge(resultVar{f: f, index: 0}, b.nodeFromVal(site)) + } else { + tup := site.Type().(*types.Tuple) + for i := 0; i < results.Len(); i++ { + local := indexedLocal{val: site, typ: tup.At(i).Type(), index: i} + b.addInFlowEdge(resultVar{f: f, index: i}, local) + } + } return true }) } @@ -622,37 +661,11 @@ func addArgumentFlows(b *builder, c ssa.CallInstruction, f *ssa.Function) { } } -// rtrn produces flows between values of r and c where -// c is a call instruction that resolves to the enclosing -// function of r based on b.callGraph. +// rtrn creates flow edges from the operands of the return +// statement to the result variables of the enclosing function. func (b *builder) rtrn(r *ssa.Return) { - n := b.callGraph.Nodes[r.Parent()] - // n != nil when b.callgraph is sound, but the client can - // pass any callgraph, including an underapproximate one. - if n == nil { - return - } - - for _, e := range n.In { - if cv, ok := e.Site.(ssa.Value); ok { - addReturnFlows(b, r, cv) - } - } -} - -func addReturnFlows(b *builder, r *ssa.Return, site ssa.Value) { - results := r.Results - if len(results) == 1 { - // When there is only one return value, the destination register does not - // have a tuple type. - b.addInFlowEdge(b.nodeFromVal(results[0]), b.nodeFromVal(site)) - return - } - - tup := site.Type().(*types.Tuple) - for i, r := range results { - local := indexedLocal{val: site, typ: tup.At(i).Type(), index: i} - b.addInFlowEdge(b.nodeFromVal(r), local) + for i, rs := range r.Results { + b.addInFlowEdge(b.nodeFromVal(rs), resultVar{f: r.Parent(), index: i}) } } @@ -793,7 +806,7 @@ func (b *builder) representative(n node) node { return field{StructType: canonicalize(i.StructType, &b.canon), index: i.index} case indexedLocal: return indexedLocal{typ: t, val: i.val, index: i.index} - case local, global, panicArg, recoverReturn, function: + case local, global, panicArg, recoverReturn, function, resultVar: return n default: panic(fmt.Errorf("canonicalizing unrecognized node %v", n)) diff --git a/go/callgraph/vta/graph_test.go b/go/callgraph/vta/graph_test.go index ed3c1dbe81f..b32da4f54a6 100644 --- a/go/callgraph/vta/graph_test.go +++ b/go/callgraph/vta/graph_test.go @@ -24,6 +24,7 @@ func TestNodeInterface(t *testing.T) { // - basic type int // - struct X with two int fields a and b // - global variable "gl" + // - "foo" function // - "main" function and its // - first register instruction t0 := *gl prog, _, err := testProg("testdata/src/simple.go", ssa.BuilderMode(0)) @@ -33,6 +34,7 @@ func TestNodeInterface(t *testing.T) { pkg := prog.AllPackages()[0] main := pkg.Func("main") + foo := pkg.Func("foo") reg := firstRegInstr(main) // t0 := *gl X := pkg.Type("X").Type() gl := pkg.Var("gl") @@ -64,6 +66,7 @@ func TestNodeInterface(t *testing.T) { {local{val: reg}, "Local(t0)", bint}, {indexedLocal{val: reg, typ: X, index: 0}, "Local(t0[0])", X}, {function{f: main}, "Function(main)", voidFunc}, + {resultVar{f: foo, index: 0}, "Return(foo[r])", bint}, {nestedPtrInterface{typ: i}, "PtrInterface(interface{})", i}, {nestedPtrFunction{typ: voidFunc}, "PtrFunction(func())", voidFunc}, {panicArg{}, "Panic", nil}, @@ -111,9 +114,9 @@ func TestVtaGraph(t *testing.T) { g.addEdge(n1, n3) want := vtaGraph{ - n1: map[node]bool{n3: true}, - n2: map[node]bool{n3: true, n4: true}, - n3: map[node]bool{n4: true}, + n1: map[node]empty{n3: empty{}}, + n2: map[node]empty{n3: empty{}, n4: empty{}}, + n3: map[node]empty{n4: empty{}}, } if !reflect.DeepEqual(want, g) { @@ -202,11 +205,21 @@ func TestVTAGraphConstruction(t *testing.T) { t.Fatalf("couldn't find want in `%s`", file) } - g, _ := typePropGraph(ssautil.AllFunctions(prog), cha.CallGraph(prog)) + fs := ssautil.AllFunctions(prog) + + // First test propagation with lazy-CHA initial call graph. + g, _ := typePropGraph(fs, makeCalleesFunc(fs, nil)) got := vtaGraphStr(g) if diff := setdiff(want, got); len(diff) > 0 { t.Errorf("`%s`: want superset of %v;\n got %v\ndiff: %v", file, want, got, diff) } + + // Repeat the test with explicit CHA initial call graph. + g, _ = typePropGraph(fs, makeCalleesFunc(fs, cha.CallGraph(prog))) + got = vtaGraphStr(g) + if diff := setdiff(want, got); len(diff) > 0 { + t.Errorf("`%s`: want superset of %v;\n got %v\ndiff: %v", file, want, got, diff) + } }) } } diff --git a/go/callgraph/vta/initial.go b/go/callgraph/vta/initial.go new file mode 100644 index 00000000000..4dddc4eee6d --- /dev/null +++ b/go/callgraph/vta/initial.go @@ -0,0 +1,37 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vta + +import ( + "golang.org/x/tools/go/callgraph" + "golang.org/x/tools/go/callgraph/internal/chautil" + "golang.org/x/tools/go/ssa" +) + +// calleesFunc abstracts call graph in one direction, +// from call sites to callees. +type calleesFunc func(ssa.CallInstruction) []*ssa.Function + +// makeCalleesFunc returns an initial call graph for vta as a +// calleesFunc. If c is not nil, returns callees as given by c. +// Otherwise, it returns chautil.LazyCallees over fs. +func makeCalleesFunc(fs map[*ssa.Function]bool, c *callgraph.Graph) calleesFunc { + if c == nil { + return chautil.LazyCallees(fs) + } + return func(call ssa.CallInstruction) []*ssa.Function { + node := c.Nodes[call.Parent()] + if node == nil { + return nil + } + var cs []*ssa.Function + for _, edge := range node.Out { + if edge.Site == call { + cs = append(cs, edge.Callee.Func) + } + } + return cs + } +} diff --git a/go/callgraph/vta/propagation.go b/go/callgraph/vta/propagation.go index b15f3290e50..4274f482d10 100644 --- a/go/callgraph/vta/propagation.go +++ b/go/callgraph/vta/propagation.go @@ -147,10 +147,10 @@ func propagate(graph vtaGraph, canon *typeutil.Map) propTypeMap { } for i := len(sccs) - 1; i >= 0; i-- { - nextSccs := make(map[int]struct{}) + nextSccs := make(map[int]empty) for _, node := range sccs[i] { for succ := range graph[node] { - nextSccs[nodeToScc[succ]] = struct{}{} + nextSccs[nodeToScc[succ]] = empty{} } } // Propagate types to all successor SCCs. diff --git a/go/callgraph/vta/propagation_test.go b/go/callgraph/vta/propagation_test.go index f22518e0a56..87b80a20db7 100644 --- a/go/callgraph/vta/propagation_test.go +++ b/go/callgraph/vta/propagation_test.go @@ -199,42 +199,42 @@ func testSuite() map[string]vtaGraph { setName(f4, "F4") graphs := make(map[string]vtaGraph) - graphs["no-cycles"] = map[node]map[node]bool{ - newLocal("t0", a): {newLocal("t1", b): true}, - newLocal("t1", b): {newLocal("t2", c): true}, + graphs["no-cycles"] = map[node]map[node]empty{ + newLocal("t0", a): {newLocal("t1", b): empty{}}, + newLocal("t1", b): {newLocal("t2", c): empty{}}, } - graphs["trivial-cycle"] = map[node]map[node]bool{ - newLocal("t0", a): {newLocal("t0", a): true}, - newLocal("t1", b): {newLocal("t1", b): true}, + graphs["trivial-cycle"] = map[node]map[node]empty{ + newLocal("t0", a): {newLocal("t0", a): empty{}}, + newLocal("t1", b): {newLocal("t1", b): empty{}}, } - graphs["circle-cycle"] = map[node]map[node]bool{ - newLocal("t0", a): {newLocal("t1", a): true}, - newLocal("t1", a): {newLocal("t2", b): true}, - newLocal("t2", b): {newLocal("t0", a): true}, + graphs["circle-cycle"] = map[node]map[node]empty{ + newLocal("t0", a): {newLocal("t1", a): empty{}}, + newLocal("t1", a): {newLocal("t2", b): empty{}}, + newLocal("t2", b): {newLocal("t0", a): empty{}}, } - graphs["fully-connected"] = map[node]map[node]bool{ - newLocal("t0", a): {newLocal("t1", b): true, newLocal("t2", c): true}, - newLocal("t1", b): {newLocal("t0", a): true, newLocal("t2", c): true}, - newLocal("t2", c): {newLocal("t0", a): true, newLocal("t1", b): true}, + graphs["fully-connected"] = map[node]map[node]empty{ + newLocal("t0", a): {newLocal("t1", b): empty{}, newLocal("t2", c): empty{}}, + newLocal("t1", b): {newLocal("t0", a): empty{}, newLocal("t2", c): empty{}}, + newLocal("t2", c): {newLocal("t0", a): empty{}, newLocal("t1", b): empty{}}, } - graphs["subsumed-scc"] = map[node]map[node]bool{ - newLocal("t0", a): {newLocal("t1", b): true}, - newLocal("t1", b): {newLocal("t2", b): true}, - newLocal("t2", b): {newLocal("t1", b): true, newLocal("t3", a): true}, - newLocal("t3", a): {newLocal("t0", a): true}, + graphs["subsumed-scc"] = map[node]map[node]empty{ + newLocal("t0", a): {newLocal("t1", b): empty{}}, + newLocal("t1", b): {newLocal("t2", b): empty{}}, + newLocal("t2", b): {newLocal("t1", b): empty{}, newLocal("t3", a): empty{}}, + newLocal("t3", a): {newLocal("t0", a): empty{}}, } - graphs["more-realistic"] = map[node]map[node]bool{ - newLocal("t0", a): {newLocal("t0", a): true}, - newLocal("t1", a): {newLocal("t2", b): true}, - newLocal("t2", b): {newLocal("t1", a): true, function{f1}: true}, - function{f1}: {function{f2}: true, function{f3}: true}, - function{f2}: {function{f3}: true}, - function{f3}: {function{f1}: true, function{f4}: true}, + graphs["more-realistic"] = map[node]map[node]empty{ + newLocal("t0", a): {newLocal("t0", a): empty{}}, + newLocal("t1", a): {newLocal("t2", b): empty{}}, + newLocal("t2", b): {newLocal("t1", a): empty{}, function{f1}: empty{}}, + function{f1}: {function{f2}: empty{}, function{f3}: empty{}}, + function{f2}: {function{f3}: empty{}}, + function{f3}: {function{f1}: empty{}, function{f4}: empty{}}, } return graphs diff --git a/go/callgraph/vta/testdata/src/callgraph_range_over_func.go b/go/callgraph/vta/testdata/src/callgraph_range_over_func.go new file mode 100644 index 00000000000..fdc7e87ebaa --- /dev/null +++ b/go/callgraph/vta/testdata/src/callgraph_range_over_func.go @@ -0,0 +1,96 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go:build ignore + +package testdata + +type I interface { + Foo() +} + +type A struct{} + +func (a A) Foo() {} + +type B struct{} + +func (b B) Foo() {} + +type C struct{} + +func (c C) Foo() {} // Test that this is not called. + +type iset []I + +func (i iset) All() func(func(I) bool) { + return func(yield func(I) bool) { + for _, v := range i { + if !yield(v) { + return + } + } + } +} + +var x = iset([]I{A{}, B{}}) + +func X() { + for i := range x.All() { + i.Foo() + } +} + +func Y() I { + for i := range x.All() { + return i + } + return nil +} + +func Bar() { + X() + y := Y() + y.Foo() +} + +// Relevant SSA: +//func X$1(I) bool: +// t0 = *jump$1 +// t1 = t0 == 0:int +// if t1 goto 1 else 2 +//1: +// *jump$1 = -1:int +// t2 = invoke arg0.Foo() +// *jump$1 = 0:int +// return true:bool +//2: +// t3 = make interface{} <- string ("yield function ca...":string) interface{} +// panic t3 +// +//func All$1(yield func(I) bool): +// t0 = *i +// t1 = len(t0) +// jump 1 +//1: +// t2 = phi [0: -1:int, 2: t3] #rangeindex +// t3 = t2 + 1:int +// t4 = t3 < t1 +// if t4 goto 2 else 3 +//2: +// t5 = &t0[t3] +// t6 = *t5 +// t7 = yield(t6) +// if t7 goto 1 else 4 +// +//func Bar(): +// t0 = X() +// t1 = Y() +// t2 = invoke t1.Foo() +// return + +// WANT: +// Bar: X() -> X; Y() -> Y; invoke t1.Foo() -> A.Foo, B.Foo +// X$1: invoke arg0.Foo() -> A.Foo, B.Foo +// All$1: yield(t6) -> X$1, Y$1 diff --git a/go/callgraph/vta/testdata/src/callgraph_type_aliases.go b/go/callgraph/vta/testdata/src/callgraph_type_aliases.go index ded3158b874..9b32109a828 100644 --- a/go/callgraph/vta/testdata/src/callgraph_type_aliases.go +++ b/go/callgraph/vta/testdata/src/callgraph_type_aliases.go @@ -6,6 +6,7 @@ // This file is the same as callgraph_interfaces.go except for // types J, X, Y, and Z aliasing types I, A, B, and C, resp. +// This test requires GODEBUG=gotypesalias=1 (the default in go1.23). package testdata @@ -57,11 +58,11 @@ func Baz(b bool) { // func Do(b bool) I: // ... -// t1 = (C).Foo(struct{}{}:C) +// t1 = (C).Foo(struct{}{}:Z) // t2 = NewY() // t3 = make I <- B (t2) // return t3 // WANT: // Baz: Do(b) -> Do; invoke t0.Foo() -> A.Foo, B.Foo -// Do: (C).Foo(struct{}{}:C) -> C.Foo; NewY() -> NewY +// Do: (C).Foo(struct{}{}:Z) -> C.Foo; NewY() -> NewY diff --git a/go/callgraph/vta/testdata/src/dynamic_calls.go b/go/callgraph/vta/testdata/src/dynamic_calls.go index f8f88983dce..da37a0d55d3 100644 --- a/go/callgraph/vta/testdata/src/dynamic_calls.go +++ b/go/callgraph/vta/testdata/src/dynamic_calls.go @@ -43,6 +43,8 @@ var g *B = &B{} // ensure *B.foo is created. // type flow that gets merged together during stringification. // WANT: +// Return(doWork[0]) -> Local(t2) +// Return(close[0]) -> Local(t2) // Local(t0) -> Local(ai), Local(ai), Local(bi), Local(bi) -// Constant(testdata.I) -> Local(t2) +// Constant(testdata.I) -> Return(close[0]), Return(doWork[0]) // Local(x) -> Local(t0) diff --git a/go/callgraph/vta/testdata/src/maps.go b/go/callgraph/vta/testdata/src/maps.go index f5f51a3d687..69709b56e36 100644 --- a/go/callgraph/vta/testdata/src/maps.go +++ b/go/callgraph/vta/testdata/src/maps.go @@ -41,5 +41,5 @@ func Baz(m map[I]I, b1, b2 B, n map[string]*J) *J { // Local(b2) -> Local(t1) // Local(t1) -> MapValue(testdata.I) // Local(t0) -> MapKey(testdata.I) -// Local(t3) -> MapValue(*testdata.J) +// Local(t3) -> MapValue(*testdata.J), Return(Baz[0]) // MapValue(*testdata.J) -> Local(t3) diff --git a/go/callgraph/vta/testdata/src/returns.go b/go/callgraph/vta/testdata/src/returns.go index b11b4321ba7..27bc418851e 100644 --- a/go/callgraph/vta/testdata/src/returns.go +++ b/go/callgraph/vta/testdata/src/returns.go @@ -51,7 +51,9 @@ func Baz(i I) *I { // WANT: // Local(i) -> Local(ii), Local(j) // Local(ii) -> Local(iii) -// Local(iii) -> Local(t0[0]), Local(t0[1]) -// Local(t1) -> Local(t0[0]) -// Local(t2) -> Local(t0[1]) -// Local(t0) -> Local(t1) +// Local(iii) -> Return(Foo[0]), Return(Foo[1]) +// Local(t1) -> Return(Baz[0]) +// Local(t1) -> Return(Bar[0]) +// Local(t2) -> Return(Bar[1]) +// Local(t0) -> Return(Do[0]) +// Return(Do[0]) -> Local(t1) diff --git a/go/callgraph/vta/testdata/src/simple.go b/go/callgraph/vta/testdata/src/simple.go index d3bfbe79284..71ddbe37163 100644 --- a/go/callgraph/vta/testdata/src/simple.go +++ b/go/callgraph/vta/testdata/src/simple.go @@ -16,3 +16,5 @@ type X struct { func main() { print(gl) } + +func foo() (r int) { return gl } diff --git a/go/callgraph/vta/testdata/src/static_calls.go b/go/callgraph/vta/testdata/src/static_calls.go index 74a27c166ad..e44ab68979d 100644 --- a/go/callgraph/vta/testdata/src/static_calls.go +++ b/go/callgraph/vta/testdata/src/static_calls.go @@ -38,4 +38,6 @@ func Baz(inp I) { // Local(inp) -> Local(i) // Local(t1) -> Local(iii) // Local(t2) -> Local(ii) -// Local(i) -> Local(t0[0]), Local(t0[1]) +// Local(i) -> Return(foo[0]), Return(foo[1]) +// Return(foo[0]) -> Local(t0[0]) +// Return(foo[1]) -> Local(t0[1]) diff --git a/go/callgraph/vta/utils.go b/go/callgraph/vta/utils.go index 27923362f1a..141eb077f9c 100644 --- a/go/callgraph/vta/utils.go +++ b/go/callgraph/vta/utils.go @@ -7,7 +7,6 @@ package vta import ( "go/types" - "golang.org/x/tools/go/callgraph" "golang.org/x/tools/go/ssa" "golang.org/x/tools/internal/aliases" "golang.org/x/tools/internal/typeparams" @@ -149,22 +148,14 @@ func sliceArrayElem(t types.Type) types.Type { } } -// siteCallees returns a go1.23 iterator for the callees for call site `c` -// given program `callgraph`. -func siteCallees(c ssa.CallInstruction, callgraph *callgraph.Graph) func(yield func(*ssa.Function) bool) { +// siteCallees returns a go1.23 iterator for the callees for call site `c`. +func siteCallees(c ssa.CallInstruction, callees calleesFunc) func(yield func(*ssa.Function) bool) { // TODO: when x/tools uses go1.23, change callers to use range-over-func // (https://go.dev/issue/65237). - node := callgraph.Nodes[c.Parent()] return func(yield func(*ssa.Function) bool) { - if node == nil { - return - } - - for _, edge := range node.Out { - if edge.Site == c { - if !yield(edge.Callee.Func) { - return - } + for _, callee := range callees(c) { + if !yield(callee) { + return } } } diff --git a/go/callgraph/vta/vta.go b/go/callgraph/vta/vta.go index 1e21d055473..56fce13725f 100644 --- a/go/callgraph/vta/vta.go +++ b/go/callgraph/vta/vta.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // Package vta computes the call graph of a Go program using the Variable -// Type Analysis (VTA) algorithm originally described in “Practical Virtual +// Type Analysis (VTA) algorithm originally described in "Practical Virtual // Method Call Resolution for Java," Vijay Sundaresan, Laurie Hendren, // Chrislain Razafimahefa, Raja Vallée-Rai, Patrick Lam, Etienne Gagnon, and // Charles Godin. @@ -65,17 +65,20 @@ import ( // CallGraph uses the VTA algorithm to compute call graph for all functions // f:true in funcs. VTA refines the results of initial call graph and uses it -// to establish interprocedural type flow. The resulting graph does not have -// a root node. +// to establish interprocedural type flow. If initial is nil, VTA uses a more +// efficient approach to construct a CHA call graph. +// +// The resulting graph does not have a root node. // // CallGraph does not make any assumptions on initial types global variables // and function/method inputs can have. CallGraph is then sound, modulo use of // reflection and unsafe, if the initial call graph is sound. func CallGraph(funcs map[*ssa.Function]bool, initial *callgraph.Graph) *callgraph.Graph { - vtaG, canon := typePropGraph(funcs, initial) + callees := makeCalleesFunc(funcs, initial) + vtaG, canon := typePropGraph(funcs, callees) types := propagate(vtaG, canon) - c := &constructor{types: types, initial: initial, cache: make(methodCache)} + c := &constructor{types: types, callees: callees, cache: make(methodCache)} return c.construct(funcs) } @@ -85,7 +88,7 @@ func CallGraph(funcs map[*ssa.Function]bool, initial *callgraph.Graph) *callgrap type constructor struct { types propTypeMap cache methodCache - initial *callgraph.Graph + callees calleesFunc } func (c *constructor) construct(funcs map[*ssa.Function]bool) *callgraph.Graph { @@ -101,15 +104,15 @@ func (c *constructor) construct(funcs map[*ssa.Function]bool) *callgraph.Graph { func (c *constructor) constrct(g *callgraph.Graph, f *ssa.Function) { caller := g.CreateNode(f) for _, call := range calls(f) { - for _, c := range c.callees(call) { + for _, c := range c.resolves(call) { callgraph.AddEdge(caller, call, g.CreateNode(c)) } } } -// callees computes the set of functions to which VTA resolves `c`. The resolved -// functions are intersected with functions to which `initial` resolves `c`. -func (c *constructor) callees(call ssa.CallInstruction) []*ssa.Function { +// resolves computes the set of functions to which VTA resolves `c`. The resolved +// functions are intersected with functions to which `c.initial` resolves `c`. +func (c *constructor) resolves(call ssa.CallInstruction) []*ssa.Function { cc := call.Common() if cc.StaticCallee() != nil { return []*ssa.Function{cc.StaticCallee()} @@ -123,7 +126,7 @@ func (c *constructor) callees(call ssa.CallInstruction) []*ssa.Function { // Cover the case of dynamic higher-order and interface calls. var res []*ssa.Function resolved := resolve(call, c.types, c.cache) - siteCallees(call, c.initial)(func(f *ssa.Function) bool { + siteCallees(call, c.callees)(func(f *ssa.Function) bool { if _, ok := resolved[f]; ok { res = append(res, f) } @@ -134,18 +137,12 @@ func (c *constructor) callees(call ssa.CallInstruction) []*ssa.Function { // resolve returns a set of functions `c` resolves to based on the // type propagation results in `types`. -func resolve(c ssa.CallInstruction, types propTypeMap, cache methodCache) (fns map[*ssa.Function]struct{}) { +func resolve(c ssa.CallInstruction, types propTypeMap, cache methodCache) map[*ssa.Function]empty { + fns := make(map[*ssa.Function]empty) n := local{val: c.Common().Value} types.propTypes(n)(func(p propType) bool { - pfs := propFunc(p, c, cache) - if len(pfs) == 0 { - return true - } - if fns == nil { - fns = make(map[*ssa.Function]struct{}) - } - for _, f := range pfs { - fns[f] = struct{}{} + for _, f := range propFunc(p, c, cache) { + fns[f] = empty{} } return true }) @@ -171,9 +168,6 @@ func propFunc(p propType, c ssa.CallInstruction, cache methodCache) []*ssa.Funct // ssa.Program.MethodSets and ssa.Program.MethodValue // APIs. The cache is used to speed up querying of // methods of a type as the mentioned APIs are expensive. -// -// TODO(adonovan): Program.MethodValue already does this kind of -// caching. Is this really necessary? type methodCache map[types.Type]map[string][]*ssa.Function // methods returns methods of a type `t` named `name`. First consults diff --git a/go/callgraph/vta/vta_test.go b/go/callgraph/vta/vta_test.go index b190149edcc..1780bf6568a 100644 --- a/go/callgraph/vta/vta_test.go +++ b/go/callgraph/vta/vta_test.go @@ -2,21 +2,34 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:debug gotypesalias=1 + package vta import ( + "strings" "testing" + "github.com/google/go-cmp/cmp" "golang.org/x/tools/go/analysis" "golang.org/x/tools/go/analysis/analysistest" "golang.org/x/tools/go/analysis/passes/buildssa" "golang.org/x/tools/go/callgraph/cha" "golang.org/x/tools/go/ssa" "golang.org/x/tools/go/ssa/ssautil" + "golang.org/x/tools/internal/testenv" ) func TestVTACallGraph(t *testing.T) { - for _, file := range []string{ + errDiff := func(want, got, missing []string) { + t.Errorf("got:\n%s\n\nwant:\n%s\n\nmissing:\n%s\n\ndiff:\n%s", + strings.Join(got, "\n"), + strings.Join(want, "\n"), + strings.Join(missing, "\n"), + cmp.Diff(got, want)) // to aid debugging + } + + files := []string{ "testdata/src/callgraph_static.go", "testdata/src/callgraph_ho.go", "testdata/src/callgraph_interfaces.go", @@ -27,8 +40,13 @@ func TestVTACallGraph(t *testing.T) { "testdata/src/callgraph_recursive_types.go", "testdata/src/callgraph_issue_57756.go", "testdata/src/callgraph_comma_maps.go", - "testdata/src/callgraph_type_aliases.go", - } { + "testdata/src/callgraph_type_aliases.go", // https://github.com/golang/go/issues/68799 + } + if testenv.Go1Point() >= 23 { + files = append(files, "testdata/src/callgraph_range_over_func.go") + } + + for _, file := range files { t.Run(file, func(t *testing.T) { prog, want, err := testProg(file, ssa.BuilderMode(0)) if err != nil { @@ -38,10 +56,18 @@ func TestVTACallGraph(t *testing.T) { t.Fatalf("couldn't find want in `%s`", file) } - g := CallGraph(ssautil.AllFunctions(prog), cha.CallGraph(prog)) + // First test VTA with lazy-CHA initial call graph. + g := CallGraph(ssautil.AllFunctions(prog), nil) got := callGraphStr(g) - if diff := setdiff(want, got); len(diff) > 0 { - t.Errorf("computed callgraph %v\nshould contain\n%v\n(diff: %v)", got, want, diff) + if missing := setdiff(want, got); len(missing) > 0 { + errDiff(want, got, missing) + } + + // Repeat the test with explicit CHA initial call graph. + g = CallGraph(ssautil.AllFunctions(prog), cha.CallGraph(prog)) + got = callGraphStr(g) + if missing := setdiff(want, got); len(missing) > 0 { + errDiff(want, got, missing) } }) } @@ -156,7 +182,7 @@ func TestVTACallGraphGo117(t *testing.T) { t.Fatalf("couldn't find want in `%s`", file) } - g, _ := typePropGraph(ssautil.AllFunctions(prog), cha.CallGraph(prog)) + g, _ := typePropGraph(ssautil.AllFunctions(prog), makeCalleesFunc(nil, cha.CallGraph(prog))) got := vtaGraphStr(g) if diff := setdiff(want, got); len(diff) != 0 { t.Errorf("`%s`: want superset of %v;\n got %v", file, want, got) diff --git a/go/packages/external.go b/go/packages/external.go index c2b4b711b59..8f7afcb5dfb 100644 --- a/go/packages/external.go +++ b/go/packages/external.go @@ -82,7 +82,7 @@ type DriverResponse struct { type driver func(cfg *Config, patterns ...string) (*DriverResponse, error) // findExternalDriver returns the file path of a tool that supplies -// the build system package structure, or "" if not found." +// the build system package structure, or "" if not found. // If GOPACKAGESDRIVER is set in the environment findExternalTool returns its // value, otherwise it searches for a binary named gopackagesdriver on the PATH. func findExternalDriver(cfg *Config) driver { diff --git a/go/ssa/interp/interp_test.go b/go/ssa/interp/interp_test.go index 2ad6a9a0982..c55fe36c425 100644 --- a/go/ssa/interp/interp_test.go +++ b/go/ssa/interp/interp_test.go @@ -117,7 +117,7 @@ var testdataTests = []string{ "deepequal.go", "defer.go", "fieldprom.go", - "forvarlifetime_old.go", + // "forvarlifetime_old.go", Disabled for golang.org/cl/603895. Fix and re-enable. "ifaceconv.go", "ifaceprom.go", "initorder.go", @@ -129,7 +129,7 @@ var testdataTests = []string{ "slice2arrayptr.go", "static.go", "width32.go", - "rangevarlifetime_old.go", + // "rangevarlifetime_old.go", Disabled for golang.org/cl/603895. Fix and re-enable. "fixedbugs/issue52342.go", "fixedbugs/issue55115.go", "fixedbugs/issue52835.go", diff --git a/go/ssa/stdlib_test.go b/go/ssa/stdlib_test.go index 03c88510840..e56d6a98156 100644 --- a/go/ssa/stdlib_test.go +++ b/go/ssa/stdlib_test.go @@ -41,6 +41,7 @@ func bytesAllocated() uint64 { // returned by the 'std' query, the set is essentially transitively // closed, so marginal per-dependency costs are invisible. func TestStdlib(t *testing.T) { + t.Skip("broken; see https://go.dev/issues/69287") testLoad(t, 500, "std", "cmd") } diff --git a/go/ssa/subst.go b/go/ssa/subst.go index 4dcb871572d..631515882d3 100644 --- a/go/ssa/subst.go +++ b/go/ssa/subst.go @@ -365,19 +365,19 @@ func (subst *subster) alias(t *aliases.Alias) types.Type { rhs := subst.typ(aliases.Rhs(t)) // Create the fresh alias. - obj := aliases.NewAlias(true, tname.Pos(), tname.Pkg(), tname.Name(), rhs) - fresh := obj.Type() - if fresh, ok := fresh.(*aliases.Alias); ok { - // TODO: assume ok when aliases are always materialized (go1.27). - aliases.SetTypeParams(fresh, newTParams) - } + // + // Until 1.27, the result of aliases.NewAlias(...).Type() cannot guarantee it is a *types.Alias. + // However, as t is an *alias.Alias and t is well-typed, then aliases must have been enabled. + // Follow this decision, and always enable aliases here. + const enabled = true + obj := aliases.NewAlias(enabled, tname.Pos(), tname.Pkg(), tname.Name(), rhs, newTParams) // Substitute into all of the constraints after they are created. for i, ntp := range newTParams { bound := tparams.At(i).Constraint() ntp.SetConstraint(subst.typ(bound)) } - return fresh + return obj.Type() } // t is declared within the function origin and has type arguments. diff --git a/gopls/doc/features/passive.md b/gopls/doc/features/passive.md index 4d814acca66..92ae929ad5e 100644 --- a/gopls/doc/features/passive.md +++ b/gopls/doc/features/passive.md @@ -135,6 +135,9 @@ select any one member, gopls will highlight the complete set: More than one of these rules may be activated by a single selection, for example, by an identifier that is also a return operand. +Different occurrences of the same identifier may be color-coded to distinguish +"read" from "write" references to a given variable symbol. + Client support: @@ -204,6 +207,8 @@ a portion of it. The client may use this information to provide syntax highlighting that conveys semantic distinctions between, for example, functions and types, constants and variables, or library functions and built-ins. +Gopls also reports a modifier for the top-level constructor of each symbols's type, one of: +`interface`, `struct`, `signature`, `pointer`, `array`, `map`, `slice`, `chan`, `string`, `number`, `bool`, `invalid`. The client specifies the sets of types and modifiers it is interested in. Settings: diff --git a/gopls/doc/features/transformation.md b/gopls/doc/features/transformation.md index 061889227bb..579c14818fa 100644 --- a/gopls/doc/features/transformation.md +++ b/gopls/doc/features/transformation.md @@ -80,6 +80,18 @@ for the current selection. recognized by gopls that enables corresponding logic in the server's ApplyFix command handler. --> + + Caveats: - Many of gopls code transformations are limited by Go's syntax tree representation, which currently records comments not in the tree diff --git a/gopls/doc/generate/generate.go b/gopls/doc/generate/generate.go index c55f5fc28b6..3fd3e58e6ed 100644 --- a/gopls/doc/generate/generate.go +++ b/gopls/doc/generate/generate.go @@ -452,7 +452,7 @@ func loadLenses(settingsPkg *packages.Package, defaults map[settings.CodeLensSou return nil, fmt.Errorf("%s: declare one CodeLensSource per line", posn) } lit, ok := spec.Values[0].(*ast.BasicLit) - if !ok && lit.Kind != token.STRING { + if !ok || lit.Kind != token.STRING { return nil, fmt.Errorf("%s: CodeLensSource value is not a string literal", posn) } value, _ := strconv.Unquote(lit.Value) // ignore error: AST is well-formed @@ -539,13 +539,6 @@ func lowerFirst(x string) string { return strings.ToLower(x[:1]) + x[1:] } -func upperFirst(x string) string { - if x == "" { - return x - } - return strings.ToUpper(x[:1]) + x[1:] -} - func fileForPos(pkg *packages.Package, pos token.Pos) (*ast.File, error) { fset := pkg.Fset for _, f := range pkg.Syntax { diff --git a/gopls/doc/release/v0.17.0.md b/gopls/doc/release/v0.17.0.md index 65b835d6737..dba85fef46c 100644 --- a/gopls/doc/release/v0.17.0.md +++ b/gopls/doc/release/v0.17.0.md @@ -28,3 +28,10 @@ of paritial selection of a declration cannot invoke this code action. Hovering over a standard library symbol now displays information about the first Go release containing the symbol. For example, hovering over `errors.As` shows "Added in go1.13". + +## Semantic token modifiers of top-level constructor of types +The semantic tokens response now includes additional modifiers for the top-level +constructor of the type of each symbol: +`interface`, `struct`, `signature`, `pointer`, `array`, `map`, `slice`, `chan`, `string`, `number`, `bool`, and `invalid`. +Editors may use this for syntax coloring. + diff --git a/gopls/doc/semantictokens.md b/gopls/doc/semantictokens.md index 761d94a02d1..f17ea7f06d8 100644 --- a/gopls/doc/semantictokens.md +++ b/gopls/doc/semantictokens.md @@ -72,9 +72,9 @@ The references to *object* refer to the 1. __`keyword`__ All Go [keywords](https://golang.org/ref/spec#Keywords) are marked `keyword`. 1. __`namespace`__ All package names are marked `namespace`. In an import, if there is an alias, it would be marked. Otherwise the last component of the import path is marked. -1. __`type`__ Objects of type ```types.TypeName``` are marked `type`. -If they are also ```types.Basic``` -the modifier is `defaultLibrary`. (And in ```type B struct{C}```, ```B``` has modifier `definition`.) +1. __`type`__ Objects of type ```types.TypeName``` are marked `type`. It also reports +a modifier for the top-level constructor of the object's type, one of: +`interface`, `struct`, `signature`, `pointer`, `array`, `map`, `slice`, `chan`, `string`, `number`, `bool`, `invalid`. 1. __`parameter`__ The formal arguments in ```ast.FuncDecl``` and ```ast.FuncType``` nodes are marked `parameter`. 1. __`variable`__ Identifiers in the scope of ```const``` are modified with `readonly`. ```nil``` is usually a `variable` modified with both @@ -121,4 +121,4 @@ While a file is being edited it may temporarily contain either parsing errors or type errors. In this case gopls cannot determine some (or maybe any) of the semantic tokens. To avoid weird flickering it is the responsibility of clients to maintain the semantic token information -in the unedited part of the file, and they do. \ No newline at end of file +in the unedited part of the file, and they do. diff --git a/gopls/go.mod b/gopls/go.mod index 64a6aba7c69..a6128333050 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -1,14 +1,16 @@ module golang.org/x/tools/gopls -go 1.19 // => default GODEBUG has gotypesalias=0 +// go 1.23.1 fixes some bugs in go/types Alias support. +// (golang/go#68894 and golang/go#68905). +go 1.23.1 require ( github.com/google/go-cmp v0.6.0 github.com/jba/templatecheck v0.7.0 - golang.org/x/mod v0.20.0 + golang.org/x/mod v0.21.0 golang.org/x/sync v0.8.0 - golang.org/x/telemetry v0.0.0-20240712210958-268b4a8ec2d7 - golang.org/x/text v0.17.0 + golang.org/x/telemetry v0.0.0-20240829154258-f29ab539cc98 + golang.org/x/text v0.18.0 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d golang.org/x/vuln v1.0.4 gopkg.in/yaml.v3 v3.0.1 @@ -21,7 +23,7 @@ require ( github.com/BurntSushi/toml v1.2.1 // indirect github.com/google/safehtml v0.1.0 // indirect golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/sys v0.25.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/gopls/go.sum b/gopls/go.sum index 95a5c1d8585..c380cc75d5d 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -1,6 +1,7 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/safehtml v0.1.0 h1:EwLKo8qawTKfsi0orxcQAZzu07cICaBeFMegAU9eaT8= @@ -8,20 +9,23 @@ github.com/google/safehtml v0.1.0/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZj github.com/jba/templatecheck v0.7.0 h1:wjTb/VhGgSFeim5zjWVePBdaMo28X74bGLSABZV+zIA= github.com/jba/templatecheck v0.7.0/go.mod h1:n1Etw+Rrw1mDDD8dDRsEKTwMZsJ98EkktgNJC6wLUGo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 h1:2O2DON6y3XMJiQRAS1UWU+54aec2uopH3x7MAiqGW6Y= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= @@ -29,19 +33,19 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= -golang.org/x/telemetry v0.0.0-20240712210958-268b4a8ec2d7 h1:nU8/tAV/21mkPrCjACUeSibjhynTovgRMXc32+Y1Aec= -golang.org/x/telemetry v0.0.0-20240712210958-268b4a8ec2d7/go.mod h1:amNmu/SBSm2GAF3X+9U2C0epLocdh+r5Z+7oMYO5cLM= +golang.org/x/telemetry v0.0.0-20240829154258-f29ab539cc98 h1:Wm3cG5X6sZ0RSVRc/H1/sciC4AT6HAKgLCSH2lbpR/c= +golang.org/x/telemetry v0.0.0-20240829154258-f29ab539cc98/go.mod h1:m7R/r+o5h7UvF2JD9n2iLSGY4v8v+zNSyTJ6xynLrqs= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/vuln v1.0.4 h1:SP0mPeg2PmGCu03V+61EcQiOjmpri2XijexKdzv8Z1I= golang.org/x/vuln v1.0.4/go.mod h1:NbJdUQhX8jY++FtuhrXs2Eyx0yePo9pF7nPlIjo9aaQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/gopls/internal/cache/cache.go b/gopls/internal/cache/cache.go index a6a166aab58..9f85846165f 100644 --- a/gopls/internal/cache/cache.go +++ b/gopls/internal/cache/cache.go @@ -14,6 +14,52 @@ import ( "golang.org/x/tools/internal/memoize" ) +// ballast is a 100MB unused byte slice that exists only to reduce garbage +// collector CPU in small workspaces and at startup. +// +// The redesign of gopls described at https://go.dev/blog/gopls-scalability +// moved gopls to a model where it has a significantly smaller heap, yet still +// allocates many short-lived data structures during parsing and type checking. +// As a result, for some workspaces, particularly when opening a low-level +// package, the steady-state heap may be a small fraction of total allocation +// while rechecking the workspace, paradoxically causing the GC to consume much +// more CPU. For example, in one benchmark that analyzes the starlark +// repository, the steady-state heap was ~10MB, and the process of diagnosing +// the workspace allocated 100-200MB. +// +// The reason for this paradoxical behavior is that GC pacing +// (https://tip.golang.org/doc/gc-guide#GOGC) causes the collector to trigger +// at some multiple of the steady-state heap size, so a small steady-state heap +// causes GC to trigger sooner and more often when allocating the ephemeral +// structures. +// +// Allocating a 100MB ballast avoids this problem by ensuring a minimum heap +// size. The value of 100MB was chosen to be proportional to the in-memory +// cache in front the filecache package, and the throughput of type checking. +// Gopls already requires hundreds of megabytes of RAM to function. +// +// Note that while other use cases for a ballast were made obsolete by +// GOMEMLIMIT, ours is not. GOMEMLIMIT helps in cases where you have a +// containerized service and want to optimize its latency and throughput by +// taking advantage of available memory. However, in our case gopls is running +// on the developer's machine alongside other applications, and can have a wide +// range of memory footprints depending on the size of the user's workspace. +// Setting GOMEMLIMIT to too low a number would make gopls perform poorly on +// large repositories, and setting it to too high a number would make gopls a +// badly behaved tenant. Short of calibrating GOMEMLIMIT based on the user's +// workspace (an intractible problem), there is no way for gopls to use +// GOMEMLIMIT to solve its GC CPU problem. +// +// Because this allocation is large and occurs early, there is a good chance +// that rather than being recycled, it comes directly from the OS already +// zeroed, and since it is never accessed, the memory region may avoid being +// backed by pages of RAM. But see +// https://groups.google.com/g/golang-nuts/c/66d0cItfkjY/m/3NvgzL_sAgAJ +// +// For more details on this technique, see: +// https://blog.twitch.tv/en/2019/04/10/go-memory-ballast-how-i-learnt-to-stop-worrying-and-love-the-heap/ +var ballast = make([]byte, 100*1e6) + // New Creates a new cache for gopls operation results, using the given file // set, shared store, and session options. // diff --git a/gopls/internal/cache/check.go b/gopls/internal/cache/check.go index ea91d282456..d9c75100443 100644 --- a/gopls/internal/cache/check.go +++ b/gopls/internal/cache/check.go @@ -494,17 +494,17 @@ func (b *typeCheckBatch) getImportPackage(ctx context.Context, id PackageID) (pk // type-checking was short-circuited by the pre- func. } - // unsafe cannot be imported or type-checked. - if id == "unsafe" { - return types.Unsafe, nil - } - ph := b.handles[id] - // Do a second check for "unsafe" defensively, due to golang/go#60890. + // "unsafe" cannot be imported or type-checked. + // + // We check PkgPath, not id, as the structure of the ID + // depends on the build system (in particular, + // Bazel+gopackagesdriver appears to use something other than + // "unsafe", though we aren't sure what; even 'go list' can + // use "p [q.test]" for testing or if PGO is enabled. + // See golang/go#60890. if ph.mp.PkgPath == "unsafe" { - // (This assertion is reached.) - bug.Reportf("encountered \"unsafe\" as %s (golang/go#60890)", id) return types.Unsafe, nil } @@ -611,10 +611,6 @@ func storePackageResults(ctx context.Context, ph *packageHandle, p *Package) { } else { toCache[exportDataKind] = exportData } - } else if p.metadata.ID != "unsafe" { - // golang/go#60890: we should only ever see one variant of the "unsafe" - // package. - bug.Reportf("encountered \"unsafe\" as %s (golang/go#60890)", p.metadata.ID) } for kind, data := range toCache { @@ -973,11 +969,6 @@ func (s *Snapshot) getPackageHandles(ctx context.Context, ids []PackageID) (map[ for _, v := range b.nodes { assert(v.ph != nil, "nil handle") handles[v.mp.ID] = v.ph - - // debugging #60890 - if v.ph.mp.PkgPath == "unsafe" && v.mp.ID != "unsafe" { - bug.Reportf("PackagePath \"unsafe\" with ID %q", v.mp.ID) - } } return handles, nil diff --git a/gopls/internal/cache/load.go b/gopls/internal/cache/load.go index 3bf79cb1615..36aeddcd9e0 100644 --- a/gopls/internal/cache/load.go +++ b/gopls/internal/cache/load.go @@ -359,11 +359,6 @@ func buildMetadata(updates map[PackageID]*metadata.Package, pkg *packages.Packag pkgPath := PackagePath(pkg.PkgPath) id := PackageID(pkg.ID) - // debugging #60890 - if pkg.PkgPath == "unsafe" && pkg.ID != "unsafe" { - bug.Reportf("PackagePath \"unsafe\" with ID %q", pkg.ID) - } - if metadata.IsCommandLineArguments(id) { var f string // file to use as disambiguating suffix if len(pkg.CompiledGoFiles) > 0 { @@ -415,11 +410,6 @@ func buildMetadata(updates map[PackageID]*metadata.Package, pkg *packages.Packag Standalone: standalone, } - // debugging #60890 - if mp.PkgPath == "unsafe" && mp.ID != "unsafe" { - bug.Reportf("PackagePath \"unsafe\" with ID %q", mp.ID) - } - updates[id] = mp for _, filename := range pkg.CompiledGoFiles { diff --git a/gopls/internal/cache/snapshot.go b/gopls/internal/cache/snapshot.go index d575ae63b61..9014817bdff 100644 --- a/gopls/internal/cache/snapshot.go +++ b/gopls/internal/cache/snapshot.go @@ -964,7 +964,7 @@ func (s *Snapshot) watchSubdirs() bool { // requirements that client names do not change. We should update the VS // Code extension to set a default value of "subdirWatchPatterns" to "on", // so that this workaround is only temporary. - if s.Options().ClientInfo != nil && s.Options().ClientInfo.Name == "Visual Studio Code" { + if s.Options().ClientInfo.Name == "Visual Studio Code" { return true } return false diff --git a/gopls/internal/cache/view.go b/gopls/internal/cache/view.go index f1a13e358da..7ff3e7b0c8b 100644 --- a/gopls/internal/cache/view.go +++ b/gopls/internal/cache/view.go @@ -706,7 +706,7 @@ func (s *Snapshot) initialize(ctx context.Context, firstAttempt bool) { } case len(modDiagnostics) > 0: initialErr = &InitializationError{ - MainError: fmt.Errorf(modDiagnostics[0].Message), + MainError: errors.New(modDiagnostics[0].Message), } } diff --git a/gopls/internal/clonetest/clonetest.go b/gopls/internal/clonetest/clonetest.go new file mode 100644 index 00000000000..3542476ae09 --- /dev/null +++ b/gopls/internal/clonetest/clonetest.go @@ -0,0 +1,152 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package clonetest provides utility functions for testing Clone operations. +// +// The [NonZero] helper may be used to construct a type in which fields are +// recursively set to a non-zero value. This value can then be cloned, and the +// [ZeroOut] helper can set values stored in the clone to zero, recursively. +// Doing so should not mutate the original. +package clonetest + +import ( + "fmt" + "reflect" +) + +// NonZero returns a T set to some appropriate nonzero value: +// - Values of basic type are set to an arbitrary non-zero value. +// - Struct fields are set to a non-zero value. +// - Array indices are set to a non-zero value. +// - Pointers point to a non-zero value. +// - Maps and slices are given a non-zero element. +// - Chan, Func, Interface, UnsafePointer are all unsupported. +// +// NonZero breaks cycles by returning a zero value for recursive types. +func NonZero[T any]() T { + var x T + t := reflect.TypeOf(x) + if t == nil { + panic("untyped nil") + } + v := nonZeroValue(t, nil) + return v.Interface().(T) +} + +// nonZeroValue returns a non-zero, addressable value of the given type. +func nonZeroValue(t reflect.Type, seen []reflect.Type) reflect.Value { + for _, t2 := range seen { + if t == t2 { + // Cycle: return the zero value. + return reflect.Zero(t) + } + } + seen = append(seen, t) + v := reflect.New(t).Elem() + switch t.Kind() { + case reflect.Bool: + v.SetBool(true) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + v.SetInt(1) + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + v.SetUint(1) + + case reflect.Float32, reflect.Float64: + v.SetFloat(1) + + case reflect.Complex64, reflect.Complex128: + v.SetComplex(1) + + case reflect.Array: + for i := 0; i < v.Len(); i++ { + v.Index(i).Set(nonZeroValue(t.Elem(), seen)) + } + + case reflect.Map: + v2 := reflect.MakeMap(t) + v2.SetMapIndex(nonZeroValue(t.Key(), seen), nonZeroValue(t.Elem(), seen)) + v.Set(v2) + + case reflect.Pointer: + v2 := nonZeroValue(t.Elem(), seen) + v.Set(v2.Addr()) + + case reflect.Slice: + v2 := reflect.Append(v, nonZeroValue(t.Elem(), seen)) + v.Set(v2) + + case reflect.String: + v.SetString(".") + + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + v.Field(i).Set(nonZeroValue(t.Field(i).Type, seen)) + } + + default: // Chan, Func, Interface, UnsafePointer + panic(fmt.Sprintf("reflect kind %v not supported", t.Kind())) + } + return v +} + +// ZeroOut recursively sets values contained in t to zero. +// Values of king Chan, Func, Interface, UnsafePointer are all unsupported. +// +// No attempt is made to handle cyclic values. +func ZeroOut[T any](t *T) { + v := reflect.ValueOf(t).Elem() + zeroOutValue(v) +} + +func zeroOutValue(v reflect.Value) { + if v.IsZero() { + return // nothing to do; this also handles untyped nil values + } + + switch v.Kind() { + case reflect.Bool, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, + reflect.Complex64, reflect.Complex128, + reflect.String: + + v.Set(reflect.Zero(v.Type())) + + case reflect.Array: + for i := 0; i < v.Len(); i++ { + zeroOutValue(v.Index(i)) + } + + case reflect.Map: + iter := v.MapRange() + for iter.Next() { + mv := iter.Value() + if mv.CanAddr() { + zeroOutValue(mv) + } else { + mv = reflect.New(mv.Type()).Elem() + } + v.SetMapIndex(iter.Key(), mv) + } + + case reflect.Pointer: + zeroOutValue(v.Elem()) + + case reflect.Slice: + for i := 0; i < v.Len(); i++ { + zeroOutValue(v.Index(i)) + } + + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + zeroOutValue(v.Field(i)) + } + + default: + panic(fmt.Sprintf("reflect kind %v not supported", v.Kind())) + } +} diff --git a/gopls/internal/clonetest/clonetest_test.go b/gopls/internal/clonetest/clonetest_test.go new file mode 100644 index 00000000000..bbb803f2447 --- /dev/null +++ b/gopls/internal/clonetest/clonetest_test.go @@ -0,0 +1,74 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package clonetest_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "golang.org/x/tools/gopls/internal/clonetest" +) + +func Test(t *testing.T) { + doTest(t, true, false) + type B bool + doTest(t, B(true), false) + doTest(t, 1, 0) + doTest(t, int(1), 0) + doTest(t, int8(1), 0) + doTest(t, int16(1), 0) + doTest(t, int32(1), 0) + doTest(t, int64(1), 0) + doTest(t, uint(1), 0) + doTest(t, uint8(1), 0) + doTest(t, uint16(1), 0) + doTest(t, uint32(1), 0) + doTest(t, uint64(1), 0) + doTest(t, uintptr(1), 0) + doTest(t, float32(1), 0) + doTest(t, float64(1), 0) + doTest(t, complex64(1), 0) + doTest(t, complex128(1), 0) + doTest(t, [3]int{1, 1, 1}, [3]int{0, 0, 0}) + doTest(t, ".", "") + m1, m2 := map[string]int{".": 1}, map[string]int{".": 0} + doTest(t, m1, m2) + doTest(t, &m1, &m2) + doTest(t, []int{1}, []int{0}) + i, j := 1, 0 + doTest(t, &i, &j) + k, l := &i, &j + doTest(t, &k, &l) + + s1, s2 := []int{1}, []int{0} + doTest(t, &s1, &s2) + + type S struct { + Field int + } + doTest(t, S{1}, S{0}) + + doTest(t, []*S{{1}}, []*S{{0}}) + + // An arbitrary recursive type. + type LinkedList[T any] struct { + V T + Next *LinkedList[T] + } + doTest(t, &LinkedList[int]{V: 1}, &LinkedList[int]{V: 0}) +} + +// doTest checks that the result of NonZero matches the nonzero argument, and +// that zeroing out that result matches the zero argument. +func doTest[T any](t *testing.T, nonzero, zero T) { + got := clonetest.NonZero[T]() + if diff := cmp.Diff(nonzero, got); diff != "" { + t.Fatalf("NonZero() returned unexpected diff (-want +got):\n%s", diff) + } + clonetest.ZeroOut(&got) + if diff := cmp.Diff(zero, got); diff != "" { + t.Errorf("ZeroOut() returned unexpected diff (-want +got):\n%s", diff) + } +} diff --git a/gopls/internal/cmd/integration_test.go b/gopls/internal/cmd/integration_test.go index f4d76b90b27..0bc066b02e0 100644 --- a/gopls/internal/cmd/integration_test.go +++ b/gopls/internal/cmd/integration_test.go @@ -819,7 +819,7 @@ const c = 0 want := ` /*⇒7,keyword,[]*/package /*⇒1,namespace,[]*/a /*⇒4,keyword,[]*/func /*⇒1,function,[definition]*/f() -/*⇒3,keyword,[]*/var /*⇒1,variable,[definition]*/v /*⇒3,type,[defaultLibrary]*/int +/*⇒3,keyword,[]*/var /*⇒1,variable,[definition]*/v /*⇒3,type,[defaultLibrary number]*/int /*⇒5,keyword,[]*/const /*⇒1,variable,[definition readonly]*/c = /*⇒1,number,[]*/0 `[1:] if got != want { diff --git a/gopls/internal/filecache/filecache.go b/gopls/internal/filecache/filecache.go index 31a76efe3ae..243e9547128 100644 --- a/gopls/internal/filecache/filecache.go +++ b/gopls/internal/filecache/filecache.go @@ -53,7 +53,7 @@ func Start() { // As an optimization, use a 100MB in-memory LRU cache in front of filecache // operations. This reduces I/O for operations such as diagnostics or // implementations that repeatedly access the same cache entries. -var memCache = lru.New(100 * 1e6) +var memCache = lru.New[memKey, []byte](100 * 1e6) type memKey struct { kind string @@ -69,8 +69,8 @@ func Get(kind string, key [32]byte) ([]byte, error) { // First consult the read-through memory cache. // Note that memory cache hits do not update the times // used for LRU eviction of the file-based cache. - if value := memCache.Get(memKey{kind, key}); value != nil { - return value.([]byte), nil + if value, ok := memCache.Get(memKey{kind, key}); ok { + return value, nil } iolimit <- struct{}{} // acquire a token diff --git a/gopls/internal/golang/completion/statements.go b/gopls/internal/golang/completion/statements.go index 3ac130c4e21..ce80cfb08ce 100644 --- a/gopls/internal/golang/completion/statements.go +++ b/gopls/internal/golang/completion/statements.go @@ -408,7 +408,7 @@ func (c *completer) addReturnZeroValues() { snip.WritePlaceholder(func(b *snippet.Builder) { b.WriteText(zero) }) - fmt.Fprintf(&label, zero) + fmt.Fprint(&label, zero) } c.items = append(c.items, CompletionItem{ diff --git a/gopls/internal/golang/definition.go b/gopls/internal/golang/definition.go index 8b1adbfe561..438fe2a3949 100644 --- a/gopls/internal/golang/definition.go +++ b/gopls/internal/golang/definition.go @@ -173,6 +173,10 @@ func builtinDecl(ctx context.Context, snapshot *cache.Snapshot, obj types.Object if obj.Pkg() == types.Unsafe { // package "unsafe": // parse $GOROOT/src/unsafe/unsafe.go + // + // (Strictly, we shouldn't assume that the ID of a std + // package is its PkgPath, but no Bazel+gopackagesdriver + // users have complained about this yet.) unsafe := snapshot.Metadata("unsafe") if unsafe == nil { // If the type checker somehow resolved 'unsafe', we must have metadata diff --git a/gopls/internal/golang/highlight.go b/gopls/internal/golang/highlight.go index ea8a493041e..863c09f7974 100644 --- a/gopls/internal/golang/highlight.go +++ b/gopls/internal/golang/highlight.go @@ -19,7 +19,7 @@ import ( "golang.org/x/tools/internal/event" ) -func Highlight(ctx context.Context, snapshot *cache.Snapshot, fh file.Handle, position protocol.Position) ([]protocol.Range, error) { +func Highlight(ctx context.Context, snapshot *cache.Snapshot, fh file.Handle, position protocol.Position) ([]protocol.DocumentHighlight, error) { ctx, done := event.Start(ctx, "golang.Highlight") defer done() @@ -54,28 +54,31 @@ func Highlight(ctx context.Context, snapshot *cache.Snapshot, fh file.Handle, po if err != nil { return nil, err } - var ranges []protocol.Range - for rng := range result { + var ranges []protocol.DocumentHighlight + for rng, kind := range result { rng, err := pgf.PosRange(rng.start, rng.end) if err != nil { return nil, err } - ranges = append(ranges, rng) + ranges = append(ranges, protocol.DocumentHighlight{ + Range: rng, + Kind: kind, + }) } return ranges, nil } // highlightPath returns ranges to highlight for the given enclosing path, // which should be the result of astutil.PathEnclosingInterval. -func highlightPath(path []ast.Node, file *ast.File, info *types.Info) (map[posRange]struct{}, error) { - result := make(map[posRange]struct{}) +func highlightPath(path []ast.Node, file *ast.File, info *types.Info) (map[posRange]protocol.DocumentHighlightKind, error) { + result := make(map[posRange]protocol.DocumentHighlightKind) switch node := path[0].(type) { case *ast.BasicLit: // Import path string literal? if len(path) > 1 { if imp, ok := path[1].(*ast.ImportSpec); ok { highlight := func(n ast.Node) { - result[posRange{start: n.Pos(), end: n.End()}] = struct{}{} + highlightNode(result, n, protocol.Text) } // Highlight the import itself... @@ -124,10 +127,8 @@ func highlightPath(path []ast.Node, file *ast.File, info *types.Info) (map[posRa highlightLoopControlFlow(path, info, result) } } - default: - // If the cursor is in an unidentified area, return empty results. - return nil, nil } + return result, nil } @@ -145,7 +146,7 @@ type posRange struct { // // As a special case, if the cursor is within a complicated expression, control // flow highlighting is disabled, as it would highlight too much. -func highlightFuncControlFlow(path []ast.Node, result map[posRange]unit) { +func highlightFuncControlFlow(path []ast.Node, result map[posRange]protocol.DocumentHighlightKind) { var ( funcType *ast.FuncType // type of enclosing func, or nil @@ -211,10 +212,7 @@ findEnclosingFunc: if highlightAll { // Add the "func" part of the func declaration. - result[posRange{ - start: funcType.Func, - end: funcEnd, - }] = unit{} + highlightRange(result, funcType.Func, funcEnd, protocol.Text) } else if returnStmt == nil && !inResults { return // nothing to highlight } else { @@ -242,7 +240,7 @@ findEnclosingFunc: for _, field := range funcType.Results.List { for j, name := range field.Names { if inNode(name) || highlightIndexes[i+j] { - result[posRange{name.Pos(), name.End()}] = unit{} + highlightNode(result, name, protocol.Text) highlightIndexes[i+j] = true break findField // found/highlighted the specific name } @@ -257,7 +255,7 @@ findEnclosingFunc: // ...where it would make more sense to highlight only y. But we don't // reach this function if not in a func, return, ident, or basiclit. if inNode(field) || highlightIndexes[i] { - result[posRange{field.Pos(), field.End()}] = unit{} + highlightNode(result, field, protocol.Text) highlightIndexes[i] = true if inNode(field) { for j := range field.Names { @@ -286,12 +284,12 @@ findEnclosingFunc: case *ast.ReturnStmt: if highlightAll { // Add the entire return statement. - result[posRange{n.Pos(), n.End()}] = unit{} + highlightNode(result, n, protocol.Text) } else { // Add the highlighted indexes. for i, expr := range n.Results { if highlightIndexes[i] { - result[posRange{expr.Pos(), expr.End()}] = unit{} + highlightNode(result, expr, protocol.Text) } } } @@ -304,7 +302,7 @@ findEnclosingFunc: } // highlightUnlabeledBreakFlow highlights the innermost enclosing for/range/switch or swlect -func highlightUnlabeledBreakFlow(path []ast.Node, info *types.Info, result map[posRange]struct{}) { +func highlightUnlabeledBreakFlow(path []ast.Node, info *types.Info, result map[posRange]protocol.DocumentHighlightKind) { // Reverse walk the path until we find closest loop, select, or switch. for _, n := range path { switch n.(type) { @@ -323,7 +321,7 @@ func highlightUnlabeledBreakFlow(path []ast.Node, info *types.Info, result map[p // highlightLabeledFlow highlights the enclosing labeled for, range, // or switch statement denoted by a labeled break or continue stmt. -func highlightLabeledFlow(path []ast.Node, info *types.Info, stmt *ast.BranchStmt, result map[posRange]struct{}) { +func highlightLabeledFlow(path []ast.Node, info *types.Info, stmt *ast.BranchStmt, result map[posRange]protocol.DocumentHighlightKind) { use := info.Uses[stmt.Label] if use == nil { return @@ -350,7 +348,7 @@ func labelFor(path []ast.Node) *ast.Ident { return nil } -func highlightLoopControlFlow(path []ast.Node, info *types.Info, result map[posRange]struct{}) { +func highlightLoopControlFlow(path []ast.Node, info *types.Info, result map[posRange]protocol.DocumentHighlightKind) { var loop ast.Node var loopLabel *ast.Ident stmtLabel := labelFor(path) @@ -372,11 +370,9 @@ Outer: } // Add the for statement. - rng := posRange{ - start: loop.Pos(), - end: loop.Pos() + token.Pos(len("for")), - } - result[rng] = struct{}{} + rngStart := loop.Pos() + rngEnd := loop.Pos() + token.Pos(len("for")) + highlightRange(result, rngStart, rngEnd, protocol.Text) // Traverse AST to find branch statements within the same for-loop. ast.Inspect(loop, func(n ast.Node) bool { @@ -391,7 +387,7 @@ Outer: return true } if b.Label == nil || info.Uses[b.Label] == info.Defs[loopLabel] { - result[posRange{start: b.Pos(), end: b.End()}] = struct{}{} + highlightNode(result, b, protocol.Text) } return true }) @@ -404,7 +400,7 @@ Outer: } if n, ok := n.(*ast.BranchStmt); ok && n.Tok == token.CONTINUE { - result[posRange{start: n.Pos(), end: n.End()}] = struct{}{} + highlightNode(result, n, protocol.Text) } return true }) @@ -422,13 +418,13 @@ Outer: } // statement with labels that matches the loop if b.Label != nil && info.Uses[b.Label] == info.Defs[loopLabel] { - result[posRange{start: b.Pos(), end: b.End()}] = struct{}{} + highlightNode(result, b, protocol.Text) } return true }) } -func highlightSwitchFlow(path []ast.Node, info *types.Info, result map[posRange]struct{}) { +func highlightSwitchFlow(path []ast.Node, info *types.Info, result map[posRange]protocol.DocumentHighlightKind) { var switchNode ast.Node var switchNodeLabel *ast.Ident stmtLabel := labelFor(path) @@ -450,11 +446,9 @@ Outer: } // Add the switch statement. - rng := posRange{ - start: switchNode.Pos(), - end: switchNode.Pos() + token.Pos(len("switch")), - } - result[rng] = struct{}{} + rngStart := switchNode.Pos() + rngEnd := switchNode.Pos() + token.Pos(len("switch")) + highlightRange(result, rngStart, rngEnd, protocol.Text) // Traverse AST to find break statements within the same switch. ast.Inspect(switchNode, func(n ast.Node) bool { @@ -471,7 +465,7 @@ Outer: } if b.Label == nil || info.Uses[b.Label] == info.Defs[switchNodeLabel] { - result[posRange{start: b.Pos(), end: b.End()}] = struct{}{} + highlightNode(result, b, protocol.Text) } return true }) @@ -489,37 +483,115 @@ Outer: } if b.Label != nil && info.Uses[b.Label] == info.Defs[switchNodeLabel] { - result[posRange{start: b.Pos(), end: b.End()}] = struct{}{} + highlightNode(result, b, protocol.Text) } return true }) } -func highlightIdentifier(id *ast.Ident, file *ast.File, info *types.Info, result map[posRange]struct{}) { - highlight := func(n ast.Node) { - result[posRange{start: n.Pos(), end: n.End()}] = struct{}{} +func highlightNode(result map[posRange]protocol.DocumentHighlightKind, n ast.Node, kind protocol.DocumentHighlightKind) { + highlightRange(result, n.Pos(), n.End(), kind) +} + +func highlightRange(result map[posRange]protocol.DocumentHighlightKind, pos, end token.Pos, kind protocol.DocumentHighlightKind) { + rng := posRange{pos, end} + // Order of traversal is important: some nodes (e.g. identifiers) are + // visited more than once, but the kind set during the first visitation "wins". + if _, exists := result[rng]; !exists { + result[rng] = kind } +} + +func highlightIdentifier(id *ast.Ident, file *ast.File, info *types.Info, result map[posRange]protocol.DocumentHighlightKind) { // obj may be nil if the Ident is undefined. // In this case, the behavior expected by tests is // to match other undefined Idents of the same name. obj := info.ObjectOf(id) + highlightIdent := func(n *ast.Ident, kind protocol.DocumentHighlightKind) { + if n.Name == id.Name && info.ObjectOf(n) == obj { + highlightNode(result, n, kind) + } + } + // highlightWriteInExpr is called for expressions that are + // logically on the left side of an assignment. + // We follow the behavior of VSCode+Rust and GoLand, which differs + // slightly from types.TypeAndValue.Assignable: + // *ptr = 1 // ptr write + // *ptr.field = 1 // ptr read, field write + // s.field = 1 // s read, field write + // array[i] = 1 // array read + var highlightWriteInExpr func(expr ast.Expr) + highlightWriteInExpr = func(expr ast.Expr) { + switch expr := expr.(type) { + case *ast.Ident: + highlightIdent(expr, protocol.Write) + case *ast.SelectorExpr: + highlightIdent(expr.Sel, protocol.Write) + case *ast.StarExpr: + highlightWriteInExpr(expr.X) + case *ast.ParenExpr: + highlightWriteInExpr(expr.X) + } + } + ast.Inspect(file, func(n ast.Node) bool { switch n := n.(type) { + case *ast.AssignStmt: + for _, s := range n.Lhs { + highlightWriteInExpr(s) + } + case *ast.GenDecl: + if n.Tok == token.CONST || n.Tok == token.VAR { + for _, spec := range n.Specs { + if spec, ok := spec.(*ast.ValueSpec); ok { + for _, ele := range spec.Names { + highlightWriteInExpr(ele) + } + } + } + } + case *ast.IncDecStmt: + highlightWriteInExpr(n.X) + case *ast.SendStmt: + highlightWriteInExpr(n.Chan) + case *ast.CompositeLit: + t := info.TypeOf(n) + if ptr, ok := t.Underlying().(*types.Pointer); ok { + t = ptr.Elem() + } + if _, ok := t.Underlying().(*types.Struct); ok { + for _, expr := range n.Elts { + if expr, ok := (expr).(*ast.KeyValueExpr); ok { + highlightWriteInExpr(expr.Key) + } + } + } + case *ast.RangeStmt: + highlightWriteInExpr(n.Key) + highlightWriteInExpr(n.Value) + case *ast.Field: + for _, name := range n.Names { + highlightIdent(name, protocol.Text) + } case *ast.Ident: - if n.Name == id.Name && info.ObjectOf(n) == obj { - highlight(n) + // This case is reached for all Idents, + // including those also visited by highlightWriteInExpr. + if is[*types.Var](info.ObjectOf(n)) { + highlightIdent(n, protocol.Read) + } else { + // kind of idents in PkgName, etc. is Text + highlightIdent(n, protocol.Text) } - case *ast.ImportSpec: pkgname, ok := typesutil.ImportedPkgName(info, n) if ok && pkgname == obj { if n.Name != nil { - highlight(n.Name) + highlightNode(result, n.Name, protocol.Text) } else { - highlight(n) + highlightNode(result, n, protocol.Text) } } } diff --git a/gopls/internal/golang/hover.go b/gopls/internal/golang/hover.go index 2edb8a99d34..b315b7383d4 100644 --- a/gopls/internal/golang/hover.go +++ b/gopls/internal/golang/hover.go @@ -1279,7 +1279,7 @@ func StdSymbolOf(obj types.Object) *stdlib.Symbol { // If pkgURL is non-nil, it should be used to generate doc links. func formatLink(h *hoverJSON, options *settings.Options, pkgURL func(path PackagePath, fragment string) protocol.URI) string { - if options.LinksInHover == false || h.LinkPath == "" { + if options.LinksInHover == settings.LinksInHover_None || h.LinkPath == "" { return "" } var url protocol.URI diff --git a/gopls/internal/golang/references.go b/gopls/internal/golang/references.go index 954748fbc36..52d02543a33 100644 --- a/gopls/internal/golang/references.go +++ b/gopls/internal/golang/references.go @@ -31,7 +31,6 @@ import ( "golang.org/x/tools/gopls/internal/cache/parsego" "golang.org/x/tools/gopls/internal/file" "golang.org/x/tools/gopls/internal/protocol" - "golang.org/x/tools/gopls/internal/util/bug" "golang.org/x/tools/gopls/internal/util/safetoken" "golang.org/x/tools/internal/event" ) @@ -508,7 +507,10 @@ func expandMethodSearch(ctx context.Context, snapshot *cache.Snapshot, workspace // Compute the method-set fingerprint used as a key to the global search. key, hasMethods := methodsets.KeyOf(recv) if !hasMethods { - return bug.Errorf("KeyOf(%s)={} yet %s is a method", recv, method) + // The query object was method T.m, but methodset(T)={}: + // this indicates that ill-typed T has conflicting fields and methods. + // Rather than bug-report (#67978), treat the empty method set at face value. + return nil } // Search the methodset index of each package in the workspace. indexes, err := snapshot.MethodSets(ctx, workspaceIDs...) diff --git a/gopls/internal/golang/semtok.go b/gopls/internal/golang/semtok.go index 99a0ba335f7..9fd093fe5fc 100644 --- a/gopls/internal/golang/semtok.go +++ b/gopls/internal/golang/semtok.go @@ -210,18 +210,18 @@ func (tv *tokenVisitor) comment(c *ast.Comment, importByName map[string]*types.P } } - tokenTypeByObject := func(obj types.Object) semtok.TokenType { + tokenTypeByObject := func(obj types.Object) (semtok.TokenType, []string) { switch obj.(type) { case *types.PkgName: - return semtok.TokNamespace + return semtok.TokNamespace, nil case *types.Func: - return semtok.TokFunction + return semtok.TokFunction, nil case *types.TypeName: - return semtok.TokType + return semtok.TokType, appendTypeModifiers(nil, obj) case *types.Const, *types.Var: - return semtok.TokVariable + return semtok.TokVariable, nil default: - return semtok.TokComment + return semtok.TokComment, nil } } @@ -244,7 +244,8 @@ func (tv *tokenVisitor) comment(c *ast.Comment, importByName map[string]*types.P } id, rest, _ := strings.Cut(name, ".") name = rest - tv.token(offset, len(id), tokenTypeByObject(obj), nil) + tok, mods := tokenTypeByObject(obj) + tv.token(offset, len(id), tok, mods) offset += token.Pos(len(id)) } last = idx[3] @@ -483,6 +484,46 @@ func (tv *tokenVisitor) inspect(n ast.Node) (descend bool) { return true } +// appendTypeModifiers appends optional modifiers that describe the top-level +// type constructor of obj.Type(): "pointer", "map", etc. +func appendTypeModifiers(mods []string, obj types.Object) []string { + switch t := obj.Type().Underlying().(type) { + case *types.Interface: + mods = append(mods, "interface") + case *types.Struct: + mods = append(mods, "struct") + case *types.Signature: + mods = append(mods, "signature") + case *types.Pointer: + mods = append(mods, "pointer") + case *types.Array: + mods = append(mods, "array") + case *types.Map: + mods = append(mods, "map") + case *types.Slice: + mods = append(mods, "slice") + case *types.Chan: + mods = append(mods, "chan") + case *types.Basic: + mods = append(mods, "defaultLibrary") + switch t.Kind() { + case types.Invalid: + mods = append(mods, "invalid") + case types.String: + mods = append(mods, "string") + case types.Bool: + mods = append(mods, "bool") + case types.UnsafePointer: + mods = append(mods, "pointer") + default: + if t.Info()&types.IsNumeric != 0 { + mods = append(mods, "number") + } + } + } + return mods +} + func (tv *tokenVisitor) ident(id *ast.Ident) { var obj types.Object @@ -535,10 +576,8 @@ func (tv *tokenVisitor) ident(id *ast.Ident) { case *types.TypeName: // could be a TypeParam if is[*types.TypeParam](aliases.Unalias(obj.Type())) { emit(semtok.TokTypeParam) - } else if is[*types.Basic](obj.Type()) { - emit(semtok.TokType, "defaultLibrary") } else { - emit(semtok.TokType) + emit(semtok.TokType, appendTypeModifiers(nil, obj)...) } case *types.Var: if is[*types.Signature](aliases.Unalias(obj.Type())) { @@ -795,11 +834,11 @@ func (tv *tokenVisitor) definitionFor(id *ast.Ident, obj types.Object) (semtok.T if fld, ok := fldm.(*ast.Field); ok { // if len(fld.names) == 0 this is a semtok.TokType, being used if len(fld.Names) == 0 { - return semtok.TokType, nil + return semtok.TokType, appendTypeModifiers(nil, obj) } return semtok.TokVariable, modifiers } - return semtok.TokType, modifiers + return semtok.TokType, appendTypeModifiers(modifiers, obj) } } // can't happen diff --git a/gopls/internal/protocol/command/interface.go b/gopls/internal/protocol/command/interface.go index b39fc29852e..35e191eb413 100644 --- a/gopls/internal/protocol/command/interface.go +++ b/gopls/internal/protocol/command/interface.go @@ -282,10 +282,12 @@ type Interface interface { // Modules: Return information about modules within a directory // - // This command returns an empty result if there is no module, - // or if module mode is disabled. - // The result does not includes the modules that are not - // associated with any Views on the server yet. + // This command returns an empty result if there is no module, or if module + // mode is disabled. Modules will not cause any new views to be loaded and + // will only return modules associated with views that have already been + // loaded, regardless of how it is called. Given current usage (by the + // language server client), there should never be a case where Modules is + // called on a path that has not already been loaded. Modules(context.Context, ModulesArgs) (ModulesResult, error) } diff --git a/gopls/internal/protocol/semantic.go b/gopls/internal/protocol/semantic.go index 03407899b57..23356dd8ef2 100644 --- a/gopls/internal/protocol/semantic.go +++ b/gopls/internal/protocol/semantic.go @@ -52,5 +52,7 @@ var ( semanticModifiers = [...]string{ "declaration", "definition", "readonly", "static", "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary", + // Additional modifiers + "interface", "struct", "signature", "pointer", "array", "map", "slice", "chan", "string", "number", "bool", "invalid", } ) diff --git a/gopls/internal/server/command.go b/gopls/internal/server/command.go index 25a7f33372f..98e4bf92e32 100644 --- a/gopls/internal/server/command.go +++ b/gopls/internal/server/command.go @@ -71,12 +71,157 @@ type commandHandler struct { params *protocol.ExecuteCommandParams } -func (h *commandHandler) Modules(context.Context, command.ModulesArgs) (command.ModulesResult, error) { - panic("unimplemented") +func (h *commandHandler) Modules(ctx context.Context, args command.ModulesArgs) (command.ModulesResult, error) { + // keepModule filters modules based on the command args + keepModule := func(goMod protocol.DocumentURI) bool { + // Does the directory enclose the view's go.mod file? + if !args.Dir.Encloses(goMod) { + return false + } + + // Calculate the relative path + rel, err := filepath.Rel(args.Dir.Path(), goMod.Path()) + if err != nil { + return false // "can't happen" (see prior Encloses check) + } + + assert(filepath.Base(goMod.Path()) == "go.mod", fmt.Sprintf("invalid go.mod path: want go.mod, got %q", goMod.Path())) + + // Invariant: rel is a relative path without "../" segments and the last + // segment is "go.mod" + nparts := strings.Count(rel, string(filepath.Separator)) + return args.MaxDepth < 0 || nparts <= args.MaxDepth + } + + // Views may include: + // - go.work views containing one or more modules each; + // - go.mod views containing a single module each; + // - GOPATH and/or ad hoc views containing no modules. + // + // Retrieving a view via the request path would only work for a + // non-recursive query for a go.mod view, and even in that case + // [Session.SnapshotOf] doesn't work on directories. Thus we check every + // view. + var result command.ModulesResult + seen := map[protocol.DocumentURI]bool{} + for _, v := range h.s.session.Views() { + s, release, err := v.Snapshot() + if err != nil { + return command.ModulesResult{}, err + } + defer release() + + for _, modFile := range v.ModFiles() { + if !keepModule(modFile) { + continue + } + + // Deduplicate + if seen[modFile] { + continue + } + seen[modFile] = true + + fh, err := s.ReadFile(ctx, modFile) + if err != nil { + return command.ModulesResult{}, err + } + mod, err := s.ParseMod(ctx, fh) + if err != nil { + return command.ModulesResult{}, err + } + if mod.File.Module == nil { + continue // syntax contains errors + } + result.Modules = append(result.Modules, command.Module{ + Path: mod.File.Module.Mod.Path, + Version: mod.File.Module.Mod.Version, + GoMod: mod.URI, + }) + } + } + return result, nil } -func (h *commandHandler) Packages(context.Context, command.PackagesArgs) (command.PackagesResult, error) { - panic("unimplemented") +func (h *commandHandler) Packages(ctx context.Context, args command.PackagesArgs) (command.PackagesResult, error) { + wantTests := args.Mode&command.NeedTests != 0 + result := command.PackagesResult{ + Module: make(map[string]command.Module), + } + + keepPackage := func(pkg *metadata.Package) bool { + for _, file := range pkg.GoFiles { + for _, arg := range args.Files { + if file == arg || file.Dir() == arg { + return true + } + if args.Recursive && arg.Encloses(file) { + return true + } + } + } + return false + } + + buildPackage := func(snapshot *cache.Snapshot, meta *metadata.Package) (command.Package, command.Module) { + if wantTests { + // These will be used in the next CL to query tests + _, _ = ctx, snapshot + panic("unimplemented") + } + + pkg := command.Package{ + Path: string(meta.PkgPath), + } + if meta.Module == nil { + return pkg, command.Module{} + } + + mod := command.Module{ + Path: meta.Module.Path, + Version: meta.Module.Version, + GoMod: protocol.URIFromPath(meta.Module.GoMod), + } + pkg.ModulePath = mod.Path + return pkg, mod + } + + err := h.run(ctx, commandConfig{ + progress: "Packages", + }, func(ctx context.Context, _ commandDeps) error { + for _, view := range h.s.session.Views() { + snapshot, release, err := view.Snapshot() + if err != nil { + return err + } + defer release() + + metas, err := snapshot.WorkspaceMetadata(ctx) + if err != nil { + return err + } + + for _, meta := range metas { + if meta.IsIntermediateTestVariant() { + continue + } + if !keepPackage(meta) { + continue + } + + pkg, mod := buildPackage(snapshot, meta) + result.Packages = append(result.Packages, pkg) + + // Overwriting is ok + if mod.Path != "" { + result.Module[mod.Path] = mod + } + } + } + + return nil + }) + return result, err } func (h *commandHandler) MaybePromptForTelemetry(ctx context.Context) error { diff --git a/gopls/internal/server/debug.go b/gopls/internal/server/debug.go new file mode 100644 index 00000000000..734df8682a7 --- /dev/null +++ b/gopls/internal/server/debug.go @@ -0,0 +1,12 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package server + +// assert panics with the given msg if cond is not true. +func assert(cond bool, msg string) { + if !cond { + panic(msg) + } +} diff --git a/gopls/internal/server/highlight.go b/gopls/internal/server/highlight.go index f60f01e0dd0..35ffc2db2f5 100644 --- a/gopls/internal/server/highlight.go +++ b/gopls/internal/server/highlight.go @@ -33,19 +33,7 @@ func (s *server) DocumentHighlight(ctx context.Context, params *protocol.Documen if err != nil { event.Error(ctx, "no highlight", err) } - return toProtocolHighlight(rngs), nil + return rngs, nil } return nil, nil // empty result } - -func toProtocolHighlight(rngs []protocol.Range) []protocol.DocumentHighlight { - result := make([]protocol.DocumentHighlight, 0, len(rngs)) - kind := protocol.Text - for _, rng := range rngs { - result = append(result, protocol.DocumentHighlight{ - Kind: kind, - Range: rng, - }) - } - return result -} diff --git a/gopls/internal/server/hover.go b/gopls/internal/server/hover.go index 1470210c32e..80c35c09565 100644 --- a/gopls/internal/server/hover.go +++ b/gopls/internal/server/hover.go @@ -12,6 +12,7 @@ import ( "golang.org/x/tools/gopls/internal/label" "golang.org/x/tools/gopls/internal/mod" "golang.org/x/tools/gopls/internal/protocol" + "golang.org/x/tools/gopls/internal/settings" "golang.org/x/tools/gopls/internal/telemetry" "golang.org/x/tools/gopls/internal/template" "golang.org/x/tools/gopls/internal/work" @@ -38,7 +39,7 @@ func (s *server) Hover(ctx context.Context, params *protocol.HoverParams) (_ *pr return mod.Hover(ctx, snapshot, fh, params.Position) case file.Go: var pkgURL func(path golang.PackagePath, fragment string) protocol.URI - if snapshot.Options().LinksInHover == "gopls" { + if snapshot.Options().LinksInHover == settings.LinksInHover_Gopls { web, err := s.getWeb() if err != nil { event.Error(ctx, "failed to start web server", err) diff --git a/gopls/internal/server/prompt.go b/gopls/internal/server/prompt.go index cdd22048c3d..66329784a6f 100644 --- a/gopls/internal/server/prompt.go +++ b/gopls/internal/server/prompt.go @@ -113,15 +113,6 @@ func (s *server) maybePromptForTelemetry(ctx context.Context, enabled bool) { defer work.End(ctx, "Done.") } - if !enabled { // check this after the work progress message for testing. - return // prompt is disabled - } - - if s.telemetryMode() == "on" || s.telemetryMode() == "off" { - // Telemetry is already on or explicitly off -- nothing to ask about. - return - } - errorf := func(format string, args ...any) { err := fmt.Errorf(format, args...) event.Error(ctx, "telemetry prompt failed", err) @@ -134,12 +125,14 @@ func (s *server) maybePromptForTelemetry(ctx context.Context, enabled bool) { return } + // Read the current prompt file. + var ( promptDir = filepath.Join(configDir, "prompt") // prompt configuration directory promptFile = filepath.Join(promptDir, "telemetry") // telemetry prompt file ) - // prompt states, to be written to the prompt file + // prompt states, stored in the prompt file const ( pUnknown = "" // first time pNotReady = "-" // user is not asked yet (either not sampled or not past the grace period) @@ -177,17 +170,55 @@ func (s *server) maybePromptForTelemetry(ctx context.Context, enabled bool) { } else if !os.IsNotExist(err) { errorf("reading prompt file: %v", err) // Something went wrong. Since we don't know how many times we've asked the - // prompt, err on the side of not spamming. + // prompt, err on the side of not asking. + // + // But record this in telemetry, in case some users enable telemetry by + // other means. + counter.New("gopls/telemetryprompt/corrupted").Inc() return } - // Terminal conditions. - if state == pYes || state == pNo { - // Prompt has been answered. Nothing to do. + counter.New(fmt.Sprintf("gopls/telemetryprompt/attempts:%d", attempts)).Inc() + + // Check terminal conditions. + + if state == pYes { + // Prompt has been accepted. + // + // We record this counter for every gopls session, rather than when the + // prompt actually accepted below, because if we only recorded it in the + // counter file at the time telemetry is enabled, we'd never upload it, + // because we exclude any counter files that overlap with a time period + // that has telemetry uploading is disabled. + counter.New("gopls/telemetryprompt/accepted").Inc() + return + } + if state == pNo { + // Prompt has been declined. In most cases, this means we'll never see the + // counter below, but it's possible that the user may enable telemetry by + // other means later on. If we see a significant number of users that have + // accepted telemetry but declined the prompt, it may be an indication that + // the prompt is not working well. + counter.New("gopls/telemetryprompt/declined").Inc() return } if attempts >= 5 { // pPending or pFailed - // We've tried asking enough; give up. + // We've tried asking enough; give up. Record that the prompt expired, in + // case the user decides to enable telemetry by other means later on. + // (see also the pNo case). + counter.New("gopls/telemetryprompt/expired").Inc() + return + } + + // We only check enabled after (1) the work progress is started, and (2) the + // prompt file has been read. (1) is for testing purposes, and (2) is so that + // we record the "gopls/telemetryprompt/accepted" counter for every session. + if !enabled { + return // prompt is disabled + } + + if s.telemetryMode() == "on" || s.telemetryMode() == "off" { + // Telemetry is already on or explicitly off -- nothing to ask about. return } @@ -309,7 +340,6 @@ Would you like to enable Go telemetry? result = pYes if err := s.setTelemetryMode("on"); err == nil { message(protocol.Info, telemetryOnMessage(s.Options().LinkifyShowMessage)) - counter.New("gopls/telemetryprompt/accepted").Inc() } else { errorf("enabling telemetry failed: %v", err) msg := fmt.Sprintf("Failed to enable Go telemetry: %v\nTo enable telemetry manually, please run `go run golang.org/x/telemetry/cmd/gotelemetry@latest on`", err) diff --git a/gopls/internal/settings/codeactionkind.go b/gopls/internal/settings/codeactionkind.go index dea2e699700..7cc13229279 100644 --- a/gopls/internal/settings/codeactionkind.go +++ b/gopls/internal/settings/codeactionkind.go @@ -80,6 +80,6 @@ const ( GoAssembly protocol.CodeActionKind = "source.assembly" GoDoc protocol.CodeActionKind = "source.doc" GoFreeSymbols protocol.CodeActionKind = "source.freesymbols" - GoTest protocol.CodeActionKind = "goTest" // TODO(adonovan): rename "source.test" + GoTest protocol.CodeActionKind = "source.test" GoplsDocFeatures protocol.CodeActionKind = "gopls.doc.features" ) diff --git a/gopls/internal/settings/default.go b/gopls/internal/settings/default.go index 7b14d2a5d79..9641613cd5d 100644 --- a/gopls/internal/settings/default.go +++ b/gopls/internal/settings/default.go @@ -89,7 +89,7 @@ func DefaultOptions(overrides ...func(*Options)) *Options { DocumentationOptions: DocumentationOptions{ HoverKind: FullDocumentation, LinkTarget: "pkg.go.dev", - LinksInHover: true, + LinksInHover: LinksInHover_LinkTarget, }, NavigationOptions: NavigationOptions{ ImportShortcut: BothShortcuts, diff --git a/gopls/internal/settings/settings.go b/gopls/internal/settings/settings.go index 935eb103980..2cd504b2555 100644 --- a/gopls/internal/settings/settings.go +++ b/gopls/internal/settings/settings.go @@ -13,8 +13,8 @@ import ( "golang.org/x/tools/gopls/internal/file" "golang.org/x/tools/gopls/internal/protocol" + "golang.org/x/tools/gopls/internal/util/frob" "golang.org/x/tools/gopls/internal/util/maps" - "golang.org/x/tools/gopls/internal/util/slices" ) type Annotation string @@ -36,7 +36,8 @@ const ( // Options holds various configuration that affects Gopls execution, organized // by the nature or origin of the settings. // -// Options must be comparable with reflect.DeepEqual. +// Options must be comparable with reflect.DeepEqual, and serializable with +// [frob.Codec]. // // This type defines both the logic of LSP-supplied option parsing // (see [SetOptions]), and the public documentation of options in @@ -58,7 +59,7 @@ type Options struct { // // ClientOptions must be comparable with reflect.DeepEqual. type ClientOptions struct { - ClientInfo *protocol.ClientInfo + ClientInfo protocol.ClientInfo InsertTextFormat protocol.InsertTextFormat InsertReplaceSupported bool ConfigurationSupported bool @@ -371,7 +372,29 @@ type DocumentationOptions struct { // // Note: this type has special logic in loadEnums in generate.go. // Be sure to reflect enum and doc changes there! -type LinksInHoverEnum any +type LinksInHoverEnum int + +const ( + LinksInHover_None LinksInHoverEnum = iota + LinksInHover_LinkTarget + LinksInHover_Gopls +) + +// MarshalJSON implements the json.Marshaler interface, so that the default +// values are formatted correctly in documentation. (See [Options.setOne] for +// the flexible custom unmarshalling behavior). +func (l LinksInHoverEnum) MarshalJSON() ([]byte, error) { + switch l { + case LinksInHover_None: + return []byte("false"), nil + case LinksInHover_LinkTarget: + return []byte("true"), nil + case LinksInHover_Gopls: + return []byte(`"gopls"`), nil + default: + return nil, fmt.Errorf("invalid LinksInHover value %d", l) + } +} // Note: FormattingOptions must be comparable with reflect.DeepEqual. type FormattingOptions struct { @@ -798,7 +821,20 @@ func (o *Options) Set(value any) (errors []error) { case map[string]any: seen := make(map[string]struct{}) for name, value := range value { - if err := o.set(name, value, seen); err != nil { + // Use only the last segment of a dotted name such as + // ui.navigation.symbolMatcher. The other segments + // are discarded, even without validation (!). + // (They are supported to enable hierarchical names + // in the VS Code graphical configuration UI.) + split := strings.Split(name, ".") + name = split[len(split)-1] + + if _, ok := seen[name]; ok { + errors = append(errors, fmt.Errorf("duplicate value for %s", name)) + } + seen[name] = struct{}{} + + if err := o.setOne(name, value); err != nil { err := fmt.Errorf("setting option %q: %w", name, err) errors = append(errors, err) } @@ -809,8 +845,10 @@ func (o *Options) Set(value any) (errors []error) { return errors } -func (o *Options) ForClientCapabilities(clientName *protocol.ClientInfo, caps protocol.ClientCapabilities) { - o.ClientInfo = clientName +func (o *Options) ForClientCapabilities(clientInfo *protocol.ClientInfo, caps protocol.ClientCapabilities) { + if clientInfo != nil { + o.ClientInfo = *clientInfo + } if caps.Workspace.WorkspaceEdit != nil { o.SupportedResourceOperations = caps.Workspace.WorkspaceEdit.ResourceOperations } @@ -860,24 +898,13 @@ func (o *Options) ForClientCapabilities(clientName *protocol.ClientInfo, caps pr } } -func (o *Options) Clone() *Options { - // TODO(rfindley): has this function gone stale? It appears that there are - // settings that are incorrectly cloned here (such as TemplateExtensions). - result := &Options{ - ClientOptions: o.ClientOptions, - InternalOptions: o.InternalOptions, - ServerOptions: o.ServerOptions, - UserOptions: o.UserOptions, - } - // Fully clone any slice or map fields. Only UserOptions can be modified. - result.Analyses = maps.Clone(o.Analyses) - result.Codelenses = maps.Clone(o.Codelenses) - result.SetEnvSlice(o.EnvSlice()) - result.BuildFlags = slices.Clone(o.BuildFlags) - result.DirectoryFilters = slices.Clone(o.DirectoryFilters) - result.StandaloneTags = slices.Clone(o.StandaloneTags) +var codec = frob.CodecFor[*Options]() - return result +func (o *Options) Clone() *Options { + data := codec.Encode(o) + var clone *Options + codec.Decode(data, &clone) + return clone } // validateDirectoryFilter validates if the filter string @@ -904,23 +931,10 @@ func validateDirectoryFilter(ifilter string) (string, error) { return strings.TrimRight(filepath.FromSlash(filter), "/"), nil } -// set updates a field of o based on the name and value. +// setOne updates a field of o based on the name and value. // It returns an error if the value was invalid or duplicate. // It is the caller's responsibility to augment the error with 'name'. -func (o *Options) set(name string, value any, seen map[string]struct{}) error { - // Use only the last segment of a dotted name such as - // ui.navigation.symbolMatcher. The other segments - // are discarded, even without validation (!). - // (They are supported to enable hierarchical names - // in the VS Code graphical configuration UI.) - split := strings.Split(name, ".") - name = split[len(split)-1] - - if _, ok := seen[name]; ok { - return fmt.Errorf("duplicate value") - } - seen[name] = struct{}{} - +func (o *Options) setOne(name string, value any) error { switch name { case "env": env, ok := value.(map[string]any) @@ -1005,8 +1019,12 @@ func (o *Options) set(name string, value any, seen map[string]struct{}) error { case "linksInHover": switch value { - case false, true, "gopls": - o.LinksInHover = value + case false: + o.LinksInHover = LinksInHover_None + case true: + o.LinksInHover = LinksInHover_LinkTarget + case "gopls": + o.LinksInHover = LinksInHover_Gopls default: return fmt.Errorf(`invalid value %s; expect false, true, or "gopls"`, value) @@ -1334,7 +1352,6 @@ func setAnnotationMap(dest *map[Annotation]bool, value any) error { default: return err } - continue } m[a] = enabled } diff --git a/gopls/internal/settings/settings_test.go b/gopls/internal/settings/settings_test.go index aa566d8f0b4..e2375222639 100644 --- a/gopls/internal/settings/settings_test.go +++ b/gopls/internal/settings/settings_test.go @@ -2,12 +2,16 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package settings +package settings_test import ( "reflect" "testing" "time" + + "github.com/google/go-cmp/cmp" + "golang.org/x/tools/gopls/internal/clonetest" + . "golang.org/x/tools/gopls/internal/settings" ) func TestDefaultsEquivalence(t *testing.T) { @@ -18,7 +22,7 @@ func TestDefaultsEquivalence(t *testing.T) { } } -func TestSetOption(t *testing.T) { +func TestOptions_Set(t *testing.T) { type testCase struct { name string value any @@ -206,7 +210,7 @@ func TestSetOption(t *testing.T) { for _, test := range tests { var opts Options - err := opts.set(test.name, test.value, make(map[string]struct{})) + err := opts.Set(map[string]any{test.name: test.value}) if err != nil { if !test.wantError { t.Errorf("Options.set(%q, %v) failed: %v", @@ -225,3 +229,23 @@ func TestSetOption(t *testing.T) { } } } + +func TestOptions_Clone(t *testing.T) { + // Test that the Options.Clone actually performs a deep clone of the Options + // struct. + + golden := clonetest.NonZero[*Options]() + opts := clonetest.NonZero[*Options]() + opts2 := opts.Clone() + + // The clone should be equivalent to the original. + if diff := cmp.Diff(golden, opts2); diff != "" { + t.Errorf("Clone() does not match original (-want +got):\n%s", diff) + } + + // Mutating the clone should not mutate the original. + clonetest.ZeroOut(opts2) + if diff := cmp.Diff(golden, opts); diff != "" { + t.Errorf("Mutating clone mutated the original (-want +got):\n%s", diff) + } +} diff --git a/gopls/internal/test/integration/fake/editor.go b/gopls/internal/test/integration/fake/editor.go index 383f047aeab..ae41bd409fa 100644 --- a/gopls/internal/test/integration/fake/editor.go +++ b/gopls/internal/test/integration/fake/editor.go @@ -200,7 +200,8 @@ func (e *Editor) Exit(ctx context.Context) error { return nil } -// Close issues the shutdown and exit sequence an editor should. +// Close disconnects the LSP client session. +// TODO(rfindley): rename to 'Disconnect'. func (e *Editor) Close(ctx context.Context) error { if err := e.Shutdown(ctx); err != nil { return err @@ -353,6 +354,8 @@ func clientCapabilities(cfg EditorConfig) (protocol.ClientCapabilities, error) { capabilities.TextDocument.SemanticTokens.TokenModifiers = []string{ "declaration", "definition", "readonly", "static", "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary", + // Additional modifiers supported by this client: + "interface", "struct", "signature", "pointer", "array", "map", "slice", "chan", "string", "number", "bool", "invalid", } // The LSP tests have historically enabled this flag, // but really we should test both ways for older editors. @@ -633,9 +636,13 @@ func (e *Editor) sendDidClose(ctx context.Context, doc protocol.TextDocumentIden return nil } +func (e *Editor) DocumentURI(path string) protocol.DocumentURI { + return e.sandbox.Workdir.URI(path) +} + func (e *Editor) TextDocumentIdentifier(path string) protocol.TextDocumentIdentifier { return protocol.TextDocumentIdentifier{ - URI: e.sandbox.Workdir.URI(path), + URI: e.DocumentURI(path), } } @@ -656,7 +663,7 @@ func (e *Editor) SaveBufferWithoutActions(ctx context.Context, path string) erro defer e.mu.Unlock() buf, ok := e.buffers[path] if !ok { - return fmt.Errorf(fmt.Sprintf("unknown buffer: %q", path)) + return fmt.Errorf("unknown buffer: %q", path) } content := buf.text() includeText := false diff --git a/gopls/internal/test/integration/fake/workdir.go b/gopls/internal/test/integration/fake/workdir.go index 977bf5458c5..25b3cb5c557 100644 --- a/gopls/internal/test/integration/fake/workdir.go +++ b/gopls/internal/test/integration/fake/workdir.go @@ -19,6 +19,7 @@ import ( "time" "golang.org/x/tools/gopls/internal/protocol" + "golang.org/x/tools/gopls/internal/util/slices" "golang.org/x/tools/internal/robustio" ) @@ -333,8 +334,7 @@ func (w *Workdir) CheckForFileChanges(ctx context.Context) error { return nil } w.watcherMu.Lock() - watchers := make([]func(context.Context, []protocol.FileEvent), len(w.watchers)) - copy(watchers, w.watchers) + watchers := slices.Clone(w.watchers) w.watcherMu.Unlock() for _, w := range watchers { w(ctx, evts) diff --git a/gopls/internal/test/integration/misc/imports_test.go b/gopls/internal/test/integration/misc/imports_test.go index ebc1c0d50d2..15fbd87e0fd 100644 --- a/gopls/internal/test/integration/misc/imports_test.go +++ b/gopls/internal/test/integration/misc/imports_test.go @@ -6,6 +6,7 @@ package misc import ( "os" + "os/exec" "path/filepath" "strings" "testing" @@ -195,8 +196,7 @@ func main() { }) } -func TestGOMODCACHE(t *testing.T) { - const proxy = ` +const exampleProxy = ` -- example.com@v1.2.3/go.mod -- module example.com @@ -210,6 +210,8 @@ package y const Y = 2 ` + +func TestGOMODCACHE(t *testing.T) { const files = ` -- go.mod -- module mod.com @@ -217,9 +219,6 @@ module mod.com go 1.12 require example.com v1.2.3 --- go.sum -- -example.com v1.2.3 h1:6vTQqzX+pnwngZF1+5gcO3ZEWmix1jJ/h+pWS8wUxK0= -example.com v1.2.3/go.mod h1:Y2Rc5rVWjWur0h3pd9aEvK5Pof8YKDANh9gHA2Maujo= -- main.go -- package main @@ -227,14 +226,13 @@ import "example.com/x" var _, _ = x.X, y.Y ` - modcache, err := os.MkdirTemp("", "TestGOMODCACHE-modcache") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(modcache) + modcache := t.TempDir() + defer cleanModCache(t, modcache) // see doc comment of cleanModCache + WithOptions( EnvVars{"GOMODCACHE": modcache}, - ProxyFiles(proxy), + ProxyFiles(exampleProxy), + WriteGoSum("."), ).Run(t, files, func(t *testing.T, env *Env) { env.OpenFile("main.go") env.AfterChange(Diagnostics(env.AtRegexp("main.go", `y.Y`))) @@ -248,6 +246,58 @@ var _, _ = x.X, y.Y }) } +func TestRelativeReplace(t *testing.T) { + const files = ` +-- go.mod -- +module mod.com/a + +go 1.20 + +require ( + example.com v1.2.3 +) + +replace example.com/b => ../b +-- main.go -- +package main + +import "example.com/x" + +var _, _ = x.X, y.Y +` + modcache := t.TempDir() + base := filepath.Base(modcache) + defer cleanModCache(t, modcache) // see doc comment of cleanModCache + + // Construct a very unclean module cache whose length exceeds the length of + // the clean directory path, to reproduce the crash in golang/go#67156 + const sep = string(filepath.Separator) + modcache += strings.Repeat(sep+".."+sep+base, 10) + + WithOptions( + EnvVars{"GOMODCACHE": modcache}, + ProxyFiles(exampleProxy), + WriteGoSum("."), + ).Run(t, files, func(t *testing.T, env *Env) { + env.OpenFile("main.go") + env.AfterChange(Diagnostics(env.AtRegexp("main.go", `y.Y`))) + env.SaveBuffer("main.go") + env.AfterChange(NoDiagnostics(ForFile("main.go"))) + }) +} + +// TODO(rfindley): this is only necessary as the module cache cleaning of the +// sandbox does not respect GOMODCACHE set via EnvVars. We should fix this, but +// that is probably part of a larger refactoring of the sandbox that I'm not +// inclined to undertake. +func cleanModCache(t *testing.T, modcache string) { + cmd := exec.Command("go", "clean", "-modcache") + cmd.Env = append(os.Environ(), "GOMODCACHE="+modcache) + if err := cmd.Run(); err != nil { + t.Errorf("cleaning modcache: %v", err) + } +} + // Tests golang/go#40685. func TestAcceptImportsQuickFixTestVariant(t *testing.T) { const pkg = ` diff --git a/gopls/internal/test/integration/misc/prompt_test.go b/gopls/internal/test/integration/misc/prompt_test.go index 6eda9dabee3..b412d408d1c 100644 --- a/gopls/internal/test/integration/misc/prompt_test.go +++ b/gopls/internal/test/integration/misc/prompt_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" "golang.org/x/telemetry/counter" "golang.org/x/telemetry/counter/countertest" "golang.org/x/tools/gopls/internal/protocol" @@ -268,32 +269,63 @@ func main() { } ` - acceptanceCounterName := "gopls/telemetryprompt/accepted" - acceptanceCounter := counter.New(acceptanceCounterName) - // We must increment the acceptance counter in order for the initial read - // below to succeed. + var ( + acceptanceCounter = "gopls/telemetryprompt/accepted" + declinedCounter = "gopls/telemetryprompt/declined" + attempt1Counter = "gopls/telemetryprompt/attempts:1" + allCounters = []string{acceptanceCounter, declinedCounter, attempt1Counter} + ) + + // We must increment counters in order for the initial reads below to + // succeed. // // TODO(rfindley): ReadCounter should simply return 0 for uninitialized // counters. - acceptanceCounter.Inc() + for _, name := range allCounters { + counter.New(name).Inc() + } + + readCounts := func(t *testing.T) map[string]uint64 { + t.Helper() + counts := make(map[string]uint64) + for _, name := range allCounters { + count, err := countertest.ReadCounter(counter.New(name)) + if err != nil { + t.Fatalf("ReadCounter(%q) failed: %v", name, err) + } + counts[name] = count + } + return counts + } tests := []struct { - name string // subtest name - response string // response to choose for the telemetry dialog - wantMode string // resulting telemetry mode - wantMsg string // substring contained in the follow-up popup (if empty, no popup is expected) - wantInc uint64 // expected 'prompt accepted' counter increment + name string // subtest name + response string // response to choose for the telemetry dialog + wantMode string // resulting telemetry mode + wantMsg string // substring contained in the follow-up popup (if empty, no popup is expected) + wantInc uint64 // expected 'prompt accepted' counter increment + wantCounts map[string]uint64 }{ - {"yes", server.TelemetryYes, "on", "uploading is now enabled", 1}, - {"no", server.TelemetryNo, "", "", 0}, - {"empty", "", "", "", 0}, + {"yes", server.TelemetryYes, "on", "uploading is now enabled", 1, map[string]uint64{ + acceptanceCounter: 1, + declinedCounter: 0, + attempt1Counter: 1, + }}, + {"no", server.TelemetryNo, "", "", 0, map[string]uint64{ + acceptanceCounter: 0, + declinedCounter: 1, + attempt1Counter: 1, + }}, + {"empty", "", "", "", 0, map[string]uint64{ + acceptanceCounter: 0, + declinedCounter: 0, + attempt1Counter: 1, + }}, } + for _, test := range tests { t.Run(test.name, func(t *testing.T) { - initialCount, err := countertest.ReadCounter(acceptanceCounter) - if err != nil { - t.Fatalf("ReadCounter(%q) failed: %v", acceptanceCounterName, err) - } + initialCounts := readCounts(t) modeFile := filepath.Join(t.TempDir(), "mode") telemetryStartTime := time.Now().Add(-8 * 24 * time.Hour) msgRE := regexp.MustCompile(".*Would you like to enable Go telemetry?") @@ -340,12 +372,22 @@ func main() { if gotMode != test.wantMode { t.Errorf("after prompt, mode=%s, want %s", gotMode, test.wantMode) } - finalCount, err := countertest.ReadCounter(acceptanceCounter) - if err != nil { - t.Fatalf("ReadCounter(%q) failed: %v", acceptanceCounterName, err) + + // We increment the acceptance counter when checking the prompt file + // before prompting, so start a second, transient gopls session and + // verify that the acceptance counter is incremented. + env2 := ConnectGoplsEnv(t, env.Ctx, env.Sandbox, env.Editor.Config(), env.Server) + env2.Await(CompletedWork(server.TelemetryPromptWorkTitle, 1, true)) + if err := env2.Editor.Close(env2.Ctx); err != nil { + t.Errorf("closing second editor: %v", err) + } + + gotCounts := readCounts(t) + for k := range gotCounts { + gotCounts[k] -= initialCounts[k] } - if gotInc := finalCount - initialCount; gotInc != test.wantInc { - t.Errorf("%q mismatch: got %d, want %d", acceptanceCounterName, gotInc, test.wantInc) + if diff := cmp.Diff(test.wantCounts, gotCounts); diff != "" { + t.Errorf("counter mismatch (-want +got):\n%s", diff) } }) }) diff --git a/gopls/internal/test/integration/misc/semantictokens_test.go b/gopls/internal/test/integration/misc/semantictokens_test.go index e688be50946..b8d8729c63a 100644 --- a/gopls/internal/test/integration/misc/semantictokens_test.go +++ b/gopls/internal/test/integration/misc/semantictokens_test.go @@ -57,7 +57,7 @@ func TestSemantic_2527(t *testing.T) { {Token: "func", TokenType: "keyword"}, {Token: "Add", TokenType: "function", Mod: "definition deprecated"}, {Token: "T", TokenType: "typeParameter", Mod: "definition"}, - {Token: "int", TokenType: "type", Mod: "defaultLibrary"}, + {Token: "int", TokenType: "type", Mod: "defaultLibrary number"}, {Token: "target", TokenType: "parameter", Mod: "definition"}, {Token: "T", TokenType: "typeParameter"}, {Token: "l", TokenType: "parameter", Mod: "definition"}, diff --git a/gopls/internal/test/integration/misc/shared_test.go b/gopls/internal/test/integration/misc/shared_test.go index b91dde2d282..b0bbcaa030a 100644 --- a/gopls/internal/test/integration/misc/shared_test.go +++ b/gopls/internal/test/integration/misc/shared_test.go @@ -8,7 +8,6 @@ import ( "testing" . "golang.org/x/tools/gopls/internal/test/integration" - "golang.org/x/tools/gopls/internal/test/integration/fake" ) // Smoke test that simultaneous editing sessions in the same workspace works. @@ -32,19 +31,7 @@ func main() { ).Run(t, sharedProgram, func(t *testing.T, env1 *Env) { // Create a second test session connected to the same workspace and server // as the first. - awaiter := NewAwaiter(env1.Sandbox.Workdir) - editor, err := fake.NewEditor(env1.Sandbox, env1.Editor.Config()).Connect(env1.Ctx, env1.Server, awaiter.Hooks()) - if err != nil { - t.Fatal(err) - } - env2 := &Env{ - T: t, - Ctx: env1.Ctx, - Sandbox: env1.Sandbox, - Server: env1.Server, - Editor: editor, - Awaiter: awaiter, - } + env2 := ConnectGoplsEnv(t, env1.Ctx, env1.Sandbox, env1.Editor.Config(), env1.Server) env2.Await(InitialWorkspaceLoad) // In editor #1, break fmt.Println as before. env1.OpenFile("main.go") diff --git a/gopls/internal/test/integration/runner.go b/gopls/internal/test/integration/runner.go index fff5e77300a..7b3b757536f 100644 --- a/gopls/internal/test/integration/runner.go +++ b/gopls/internal/test/integration/runner.go @@ -215,19 +215,7 @@ func (r *Runner) Run(t *testing.T, files string, test TestFunc, opts ...RunOptio framer = ls.framer(jsonrpc2.NewRawStream) ts := servertest.NewPipeServer(ss, framer) - awaiter := NewAwaiter(sandbox.Workdir) - editor, err := fake.NewEditor(sandbox, config.editor).Connect(ctx, ts, awaiter.Hooks()) - if err != nil { - t.Fatal(err) - } - env := &Env{ - T: t, - Ctx: ctx, - Sandbox: sandbox, - Editor: editor, - Server: ts, - Awaiter: awaiter, - } + env := ConnectGoplsEnv(t, ctx, sandbox, config.editor, ts) defer func() { if t.Failed() && r.PrintGoroutinesOnFailure { pprof.Lookup("goroutine").WriteTo(os.Stderr, 1) @@ -242,7 +230,7 @@ func (r *Runner) Run(t *testing.T, files string, test TestFunc, opts ...RunOptio // the editor: in general we want to clean up before proceeding to the // next test, and if there is a deadlock preventing closing it will // eventually be handled by the `go test` timeout. - if err := editor.Close(xcontext.Detach(ctx)); err != nil { + if err := env.Editor.Close(xcontext.Detach(ctx)); err != nil { t.Errorf("closing editor: %v", err) } }() @@ -253,6 +241,28 @@ func (r *Runner) Run(t *testing.T, files string, test TestFunc, opts ...RunOptio } } +// ConnectGoplsEnv creates a new Gopls environment for the given sandbox, +// editor config, and server connector. +// +// TODO(rfindley): significantly refactor the way testing environments are +// constructed. +func ConnectGoplsEnv(t testing.TB, ctx context.Context, sandbox *fake.Sandbox, config fake.EditorConfig, connector servertest.Connector) *Env { + awaiter := NewAwaiter(sandbox.Workdir) + editor, err := fake.NewEditor(sandbox, config).Connect(ctx, connector, awaiter.Hooks()) + if err != nil { + t.Fatal(err) + } + env := &Env{ + T: t, + Ctx: ctx, + Sandbox: sandbox, + Server: connector, + Editor: editor, + Awaiter: awaiter, + } + return env +} + // longBuilders maps builders that are skipped when -short is set to a // (possibly empty) justification. var longBuilders = map[string]string{ diff --git a/gopls/internal/test/integration/workspace/modules_test.go b/gopls/internal/test/integration/workspace/modules_test.go new file mode 100644 index 00000000000..a3e8122bc4b --- /dev/null +++ b/gopls/internal/test/integration/workspace/modules_test.go @@ -0,0 +1,164 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package workspace + +import ( + "sort" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "golang.org/x/tools/gopls/internal/protocol" + "golang.org/x/tools/gopls/internal/protocol/command" + . "golang.org/x/tools/gopls/internal/test/integration" +) + +func TestModulesCmd(t *testing.T) { + const goModView = ` +-- go.mod -- +module foo + +-- pkg/pkg.go -- +package pkg +func Pkg() + +-- bar/bar.go -- +package bar +func Bar() + +-- bar/baz/go.mod -- +module baz + +-- bar/baz/baz.go -- +package baz +func Baz() +` + + const goWorkView = ` +-- go.work -- +use ./foo +use ./bar + +-- foo/go.mod -- +module foo + +-- foo/foo.go -- +package foo +func Foo() + +-- bar/go.mod -- +module bar + +-- bar/bar.go -- +package bar +func Bar() +` + + t.Run("go.mod view", func(t *testing.T) { + // If baz isn't loaded, it will not be included + t.Run("unloaded", func(t *testing.T) { + Run(t, goModView, func(t *testing.T, env *Env) { + checkModules(t, env, env.Editor.DocumentURI(""), -1, []command.Module{ + { + Path: "foo", + GoMod: env.Editor.DocumentURI("go.mod"), + }, + }) + }) + }) + + // With baz loaded and recursion enabled, baz will be included + t.Run("recurse", func(t *testing.T) { + Run(t, goModView, func(t *testing.T, env *Env) { + env.OpenFile("bar/baz/baz.go") + checkModules(t, env, env.Editor.DocumentURI(""), -1, []command.Module{ + { + Path: "baz", + GoMod: env.Editor.DocumentURI("bar/baz/go.mod"), + }, + { + Path: "foo", + GoMod: env.Editor.DocumentURI("go.mod"), + }, + }) + }) + }) + + // With recursion=1, baz will not be included + t.Run("depth", func(t *testing.T) { + Run(t, goModView, func(t *testing.T, env *Env) { + env.OpenFile("bar/baz/baz.go") + checkModules(t, env, env.Editor.DocumentURI(""), 1, []command.Module{ + { + Path: "foo", + GoMod: env.Editor.DocumentURI("go.mod"), + }, + }) + }) + }) + + // Baz will be included if it is requested specifically + t.Run("nested", func(t *testing.T) { + Run(t, goModView, func(t *testing.T, env *Env) { + env.OpenFile("bar/baz/baz.go") + checkModules(t, env, env.Editor.DocumentURI("bar/baz"), 0, []command.Module{ + { + Path: "baz", + GoMod: env.Editor.DocumentURI("bar/baz/go.mod"), + }, + }) + }) + }) + }) + + t.Run("go.work view", func(t *testing.T) { + t.Run("base", func(t *testing.T) { + Run(t, goWorkView, func(t *testing.T, env *Env) { + checkModules(t, env, env.Editor.DocumentURI(""), 0, nil) + }) + }) + + t.Run("recursive", func(t *testing.T) { + Run(t, goWorkView, func(t *testing.T, env *Env) { + checkModules(t, env, env.Editor.DocumentURI(""), -1, []command.Module{ + { + Path: "bar", + GoMod: env.Editor.DocumentURI("bar/go.mod"), + }, + { + Path: "foo", + GoMod: env.Editor.DocumentURI("foo/go.mod"), + }, + }) + }) + }) + }) +} + +func checkModules(t testing.TB, env *Env, dir protocol.DocumentURI, maxDepth int, want []command.Module) { + t.Helper() + + cmd, err := command.NewModulesCommand("Modules", command.ModulesArgs{Dir: dir, MaxDepth: maxDepth}) + if err != nil { + t.Fatal(err) + } + var result command.ModulesResult + env.ExecuteCommand(&protocol.ExecuteCommandParams{ + Command: command.Modules.String(), + Arguments: cmd.Arguments, + }, &result) + + // The ordering of results is undefined and modules from a go.work view are + // retrieved from a map, so sort the results to ensure consistency + sort.Slice(result.Modules, func(i, j int) bool { + a, b := result.Modules[i], result.Modules[j] + return strings.Compare(a.Path, b.Path) < 0 + }) + + diff := cmp.Diff(want, result.Modules) + if diff != "" { + t.Errorf("Modules(%v) returned unexpected diff (-want +got):\n%s", dir, diff) + } +} diff --git a/gopls/internal/test/integration/workspace/packages_test.go b/gopls/internal/test/integration/workspace/packages_test.go new file mode 100644 index 00000000000..ebeb518c644 --- /dev/null +++ b/gopls/internal/test/integration/workspace/packages_test.go @@ -0,0 +1,139 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package workspace + +import ( + "sort" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "golang.org/x/tools/gopls/internal/protocol" + "golang.org/x/tools/gopls/internal/protocol/command" + . "golang.org/x/tools/gopls/internal/test/integration" +) + +func TestPackages(t *testing.T) { + const goModView = ` +-- go.mod -- +module foo + +-- foo.go -- +package foo +func Foo() + +-- bar/bar.go -- +package bar +func Bar() + +-- baz/go.mod -- +module baz + +-- baz/baz.go -- +package baz +func Baz() +` + + t.Run("file", func(t *testing.T) { + Run(t, goModView, func(t *testing.T, env *Env) { + checkPackages(t, env, []protocol.DocumentURI{env.Editor.DocumentURI("foo.go")}, false, []command.Package{ + { + Path: "foo", + ModulePath: "foo", + }, + }, map[string]command.Module{ + "foo": { + Path: "foo", + GoMod: env.Editor.DocumentURI("go.mod"), + }, + }) + }) + }) + + t.Run("package", func(t *testing.T) { + Run(t, goModView, func(t *testing.T, env *Env) { + checkPackages(t, env, []protocol.DocumentURI{env.Editor.DocumentURI("bar")}, false, []command.Package{ + { + Path: "foo/bar", + ModulePath: "foo", + }, + }, map[string]command.Module{ + "foo": { + Path: "foo", + GoMod: env.Editor.DocumentURI("go.mod"), + }, + }) + }) + }) + + t.Run("workspace", func(t *testing.T) { + Run(t, goModView, func(t *testing.T, env *Env) { + checkPackages(t, env, []protocol.DocumentURI{env.Editor.DocumentURI("")}, true, []command.Package{ + { + Path: "foo", + ModulePath: "foo", + }, + { + Path: "foo/bar", + ModulePath: "foo", + }, + }, map[string]command.Module{ + "foo": { + Path: "foo", + GoMod: env.Editor.DocumentURI("go.mod"), + }, + }) + }) + }) + + t.Run("nested module", func(t *testing.T) { + Run(t, goModView, func(t *testing.T, env *Env) { + // Load the nested module + env.OpenFile("baz/baz.go") + + // Request packages using the URI of the nested module _directory_ + checkPackages(t, env, []protocol.DocumentURI{env.Editor.DocumentURI("baz")}, true, []command.Package{ + { + Path: "baz", + ModulePath: "baz", + }, + }, map[string]command.Module{ + "baz": { + Path: "baz", + GoMod: env.Editor.DocumentURI("baz/go.mod"), + }, + }) + }) + }) +} + +func checkPackages(t testing.TB, env *Env, files []protocol.DocumentURI, recursive bool, wantPkg []command.Package, wantModule map[string]command.Module) { + t.Helper() + + cmd, err := command.NewPackagesCommand("Packages", command.PackagesArgs{Files: files, Recursive: recursive}) + if err != nil { + t.Fatal(err) + } + var result command.PackagesResult + env.ExecuteCommand(&protocol.ExecuteCommandParams{ + Command: command.Packages.String(), + Arguments: cmd.Arguments, + }, &result) + + // The ordering of packages is undefined so sort the results to ensure + // consistency + sort.Slice(result.Packages, func(i, j int) bool { + a, b := result.Packages[i], result.Packages[j] + return strings.Compare(a.Path, b.Path) < 0 + }) + + if diff := cmp.Diff(wantPkg, result.Packages); diff != "" { + t.Errorf("Packages(%v) returned unexpected packages (-want +got):\n%s", files, diff) + } + + if diff := cmp.Diff(wantModule, result.Module); diff != "" { + t.Errorf("Packages(%v) returned unexpected modules (-want +got):\n%s", files, diff) + } +} diff --git a/gopls/internal/test/marker/doc.go b/gopls/internal/test/marker/doc.go index 5acc4ab12e7..f81604c913d 100644 --- a/gopls/internal/test/marker/doc.go +++ b/gopls/internal/test/marker/doc.go @@ -157,9 +157,21 @@ The following markers are supported within marker tests: source. If the formatting request fails, the golden file must contain the error message. - - highlight(src location, dsts ...location): makes a + - highlightall(all ...documentHighlight): makes a textDocument/highlight + request at each location in "all" and checks that the result is "all". + In other words, given highlightall(X1, X2, ..., Xn), it checks that + highlight(X1) = highlight(X2) = ... = highlight(Xn) = {X1, X2, ..., Xn}. + In general, highlight sets are not equivalence classes; for asymmetric + cases, use @highlight instead. + Each element of "all" is the label of a @hiloc marker. + + - highlight(src location, dsts ...documentHighlight): makes a textDocument/highlight request at the given src location, which should - highlight the provided dst locations. + highlight the provided dst locations and kinds. + + - hiloc(label, location, kind): defines a documentHighlight value of the + given location and kind. Use its label in a @highlightall marker to + indicate the expected result of a highlight query. - hover(src, dst location, sm stringMatcher): performs a textDocument/hover at the src location, and checks that the result is the dst location, with @@ -178,10 +190,10 @@ The following markers are supported within marker tests: (These locations are the declarations of the functions enclosing the calls, not the calls themselves.) - - item(label, details, kind): defines a completion item with the provided + - item(label, details, kind): defines a completionItem with the provided fields. This information is not positional, and therefore @item markers may occur anywhere in the source. Used in conjunction with @complete, - snippet, or rank. + @snippet, or @rank. TODO(rfindley): rethink whether floating @item annotations are the best way to specify completion results. @@ -220,28 +232,26 @@ The following markers are supported within marker tests: (Failures in the computation to offer a fix do not generally result in LSP errors, so this marker is not appropriate for testing them.) - - rank(location, ...completionItem): executes a textDocument/completion - request at the given location, and verifies that each expected - completion item occurs in the results, in the expected order. Other - unexpected completion items may occur in the results. - TODO(rfindley): this exists for compatibility with the old marker tests. - Replace this with rankl, and rename. - A "!" prefix on a label asserts that the symbol is not a + - rank(location, ...string OR completionItem): executes a + textDocument/completion request at the given location, and verifies that + each expected completion item occurs in the results, in the expected order. + Items may be specified as string literal completion labels, or as + references to a completion item created with the @item marker. + Other unexpected completion items are allowed to occur in the results, and + are ignored. A "!" prefix on a label asserts that the symbol is not a completion candidate. - - rankl(location, ...label): like rank, but only cares about completion - item labels. - - refs(location, want ...location): executes a textDocument/references request at the first location and asserts that the result is the set of 'want' locations. The first want location must be the declaration (assumedly unique). - - snippet(location, completionItem, snippet): executes a - textDocument/completion request at the location, and searches for a - result with label matching that of the provided completion item - (TODO(rfindley): accept a label rather than a completion item). Check - the result snippet matches the provided snippet. + - snippet(location, string OR completionItem, snippet): executes a + textDocument/completion request at the location, and searches for a result + with label matching that its second argument, which may be a string literal + or a reference to a completion item created by the @item marker (in which + case the item's label is used). It checks that the resulting snippet + matches the provided snippet. - symbol(golden): makes a textDocument/documentSymbol request for the enclosing file, formats the response with one symbol diff --git a/gopls/internal/test/marker/marker_test.go b/gopls/internal/test/marker/marker_test.go index c745686f9f2..d3a7685b4dd 100644 --- a/gopls/internal/test/marker/marker_test.go +++ b/gopls/internal/test/marker/marker_test.go @@ -29,6 +29,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "golang.org/x/tools/go/expect" "golang.org/x/tools/gopls/internal/cache" @@ -506,8 +507,9 @@ func is[T any](arg any) bool { // Supported value marker functions. See [valueMarkerFunc] for more details. var valueMarkerFuncs = map[string]func(marker){ - "loc": valueMarkerFunc(locMarker), - "item": valueMarkerFunc(completionItemMarker), + "loc": valueMarkerFunc(locMarker), + "item": valueMarkerFunc(completionItemMarker), + "hiloc": valueMarkerFunc(highlightLocationMarker), } // Supported action marker functions. See [actionMarkerFunc] for more details. @@ -524,6 +526,7 @@ var actionMarkerFuncs = map[string]func(marker){ "foldingrange": actionMarkerFunc(foldingRangeMarker), "format": actionMarkerFunc(formatMarker), "highlight": actionMarkerFunc(highlightMarker), + "highlightall": actionMarkerFunc(highlightAllMarker), "hover": actionMarkerFunc(hoverMarker), "hovererr": actionMarkerFunc(hoverErrMarker), "implementation": actionMarkerFunc(implementationMarker), @@ -532,7 +535,6 @@ var actionMarkerFuncs = map[string]func(marker){ "outgoingcalls": actionMarkerFunc(outgoingCallsMarker), "preparerename": actionMarkerFunc(prepareRenameMarker), "rank": actionMarkerFunc(rankMarker), - "rankl": actionMarkerFunc(ranklMarker), "refs": actionMarkerFunc(refsMarker), "rename": actionMarkerFunc(renameMarker), "renameerr": actionMarkerFunc(renameErrMarker), @@ -1032,19 +1034,24 @@ func (run *markerTestRun) fmtLocDetails(loc protocol.Location, includeTxtPos boo // ---- converters ---- -// converter is the signature of argument converters. -// A converter should return an error rather than calling marker.errorf(). -// -// type converter func(marker, any) (any, error) - -// Types with special conversions. +// Types with special handling. var ( goldenType = reflect.TypeOf(&Golden{}) - locationType = reflect.TypeOf(protocol.Location{}) markerType = reflect.TypeOf(marker{}) stringMatcherType = reflect.TypeOf(stringMatcher{}) ) +// Custom conversions. +// +// These functions are called after valueMarkerFuncs have run to convert +// arguments into the desired parameter types. +// +// Converters should return an error rather than calling marker.errorf(). +var customConverters = map[reflect.Type]func(marker, any) (any, error){ + reflect.TypeOf(protocol.Location{}): convertLocation, + reflect.TypeOf(completionLabel("")): convertCompletionLabel, +} + func convert(mark marker, arg any, paramType reflect.Type) (any, error) { // Handle stringMatcher and golden parameters before resolving identifiers, // because golden content lives in a separate namespace from other @@ -1060,15 +1067,16 @@ func convert(mark marker, arg any, paramType reflect.Type) (any, error) { return mark.run.test.getGolden(id), nil } if id, ok := arg.(expect.Identifier); ok { - if arg, ok := mark.run.values[id]; ok { - if !reflect.TypeOf(arg).AssignableTo(paramType) { - return nil, fmt.Errorf("cannot convert %v (%T) to %s", arg, arg, paramType) - } - return arg, nil + if arg2, ok := mark.run.values[id]; ok { + arg = arg2 } } - if paramType == locationType { - return convertLocation(mark, arg) + if converter, ok := customConverters[paramType]; ok { + arg2, err := converter(mark, arg) + if err != nil { + return nil, fmt.Errorf("converting for input type %T to %v: %v", arg, paramType, err) + } + arg = arg2 } if reflect.TypeOf(arg).AssignableTo(paramType) { return arg, nil // no conversion required @@ -1079,8 +1087,10 @@ func convert(mark marker, arg any, paramType reflect.Type) (any, error) { // convertLocation converts a string or regexp argument into the protocol // location corresponding to the first position of the string (or first match // of the regexp) in the line preceding the note. -func convertLocation(mark marker, arg any) (protocol.Location, error) { +func convertLocation(mark marker, arg any) (any, error) { switch arg := arg.(type) { + case protocol.Location: + return arg, nil case string: startOff, preceding, m, err := linePreceding(mark.run, mark.note.Pos) if err != nil { @@ -1095,7 +1105,32 @@ func convertLocation(mark marker, arg any) (protocol.Location, error) { case *regexp.Regexp: return findRegexpInLine(mark.run, mark.note.Pos, arg) default: - return protocol.Location{}, fmt.Errorf("cannot convert argument type %T to location (must be a string to match the preceding line)", arg) + return protocol.Location{}, fmt.Errorf("cannot convert argument type %T to location (must be a string or regexp to match the preceding line)", arg) + } +} + +// completionLabel is a special parameter type that may be converted from a +// string literal, or extracted from a completion item. +// +// See [convertCompletionLabel]. +type completionLabel string + +// convertCompletionLabel coerces an argument to a [completionLabel] parameter +// type. +// +// If the arg is a string, it is trivially converted. If the arg is a +// completionItem, its label is extracted. +// +// This allows us to stage a migration of the "snippet" marker to a simpler +// model where the completion label can just be listed explicitly. +func convertCompletionLabel(mark marker, arg any) (any, error) { + switch arg := arg.(type) { + case string: + return completionLabel(arg), nil + case completionItem: + return completionLabel(arg.Label), nil + default: + return "", fmt.Errorf("cannot convert argument type %T to completion label (must be a string or completion item)", arg) } } @@ -1315,11 +1350,11 @@ func completionItemMarker(mark marker, label string, other ...string) completion return item } -func rankMarker(mark marker, src protocol.Location, items ...completionItem) { +func rankMarker(mark marker, src protocol.Location, items ...completionLabel) { // Separate positive and negative items (expectations). - var pos, neg []completionItem + var pos, neg []completionLabel for _, item := range items { - if strings.HasPrefix(item.Label, "!") { + if strings.HasPrefix(string(item), "!") { neg = append(neg, item) } else { pos = append(pos, item) @@ -1331,13 +1366,13 @@ func rankMarker(mark marker, src protocol.Location, items ...completionItem) { var got []string for _, g := range list.Items { for _, w := range pos { - if g.Label == w.Label { + if g.Label == string(w) { got = append(got, g.Label) break } } for _, w := range neg { - if g.Label == w.Label[len("!"):] { + if g.Label == string(w[len("!"):]) { mark.errorf("got unwanted completion: %s", g.Label) break } @@ -1345,40 +1380,14 @@ func rankMarker(mark marker, src protocol.Location, items ...completionItem) { } var want []string for _, w := range pos { - want = append(want, w.Label) + want = append(want, string(w)) } if diff := cmp.Diff(want, got); diff != "" { mark.errorf("completion rankings do not match (-want +got):\n%s", diff) } } -func ranklMarker(mark marker, src protocol.Location, labels ...string) { - // Separate positive and negative labels (expectations). - var pos, neg []string - for _, label := range labels { - if strings.HasPrefix(label, "!") { - neg = append(neg, label[len("!"):]) - } else { - pos = append(pos, label) - } - } - - // Collect results that are present in items, preserving their order. - list := mark.run.env.Completion(src) - var got []string - for _, g := range list.Items { - if slices.Contains(pos, g.Label) { - got = append(got, g.Label) - } else if slices.Contains(neg, g.Label) { - mark.errorf("got unwanted completion: %s", g.Label) - } - } - if diff := cmp.Diff(pos, got); diff != "" { - mark.errorf("completion rankings do not match (-want +got):\n%s", diff) - } -} - -func snippetMarker(mark marker, src protocol.Location, item completionItem, want string) { +func snippetMarker(mark marker, src protocol.Location, label completionLabel, want string) { list := mark.run.env.Completion(src) var ( found bool @@ -1388,7 +1397,7 @@ func snippetMarker(mark marker, src protocol.Location, item completionItem, want items := filterBuiltinsAndKeywords(mark, list.Items) for _, i := range items { all = append(all, i.Label) - if i.Label == item.Label { + if i.Label == string(label) { found = true if i.TextEdit != nil { if edit, err := protocol.SelectCompletionTextEdit(i, false); err == nil { @@ -1399,7 +1408,7 @@ func snippetMarker(mark marker, src protocol.Location, item completionItem, want } } if !found { - mark.errorf("no completion item found matching %s (got: %v)", item.Label, all) + mark.errorf("no completion item found matching %s (got: %v)", label, all) return } if got != want { @@ -1593,28 +1602,60 @@ func formatMarker(mark marker, golden *Golden) { compareGolden(mark, got, golden) } -func highlightMarker(mark marker, src protocol.Location, dsts ...protocol.Location) { - highlights := mark.run.env.DocumentHighlight(src) - var got []protocol.Range - for _, h := range highlights { - got = append(got, h.Range) +func highlightLocationMarker(mark marker, loc protocol.Location, kindName expect.Identifier) protocol.DocumentHighlight { + var kind protocol.DocumentHighlightKind + switch kindName { + case "read": + kind = protocol.Read + case "write": + kind = protocol.Write + case "text": + kind = protocol.Text + default: + mark.errorf("invalid highlight kind: %q", kindName) } - var want []protocol.Range - for _, d := range dsts { - want = append(want, d.Range) + return protocol.DocumentHighlight{ + Range: loc.Range, + Kind: kind, } +} +func sortDocumentHighlights(s []protocol.DocumentHighlight) { + sort.Slice(s, func(i, j int) bool { + return protocol.CompareRange(s[i].Range, s[j].Range) < 0 + }) +} - sortRanges := func(s []protocol.Range) { - sort.Slice(s, func(i, j int) bool { - return protocol.CompareRange(s[i], s[j]) < 0 - }) +// highlightAllMarker makes textDocument/highlight +// requests at locations of equivalence classes. Given input +// highlightall(X1, X2, ..., Xn), the marker checks +// highlight(X1) = highlight(X2) = ... = highlight(Xn) = {X1, X2, ..., Xn}. +// It is not the general rule for all highlighting, and use @highlight +// for asymmetric cases. +// +// TODO(b/288111111): this is a bit of a hack. We should probably +// have a more general way of testing that a function is idempotent. +func highlightAllMarker(mark marker, all ...protocol.DocumentHighlight) { + sortDocumentHighlights(all) + for _, src := range all { + loc := protocol.Location{URI: mark.uri(), Range: src.Range} + got := mark.run.env.DocumentHighlight(loc) + sortDocumentHighlights(got) + + if d := cmp.Diff(all, got); d != "" { + mark.errorf("DocumentHighlight(%v) mismatch (-want +got):\n%s", loc, d) + } } +} - sortRanges(got) - sortRanges(want) +func highlightMarker(mark marker, src protocol.DocumentHighlight, dsts ...protocol.DocumentHighlight) { + loc := protocol.Location{URI: mark.uri(), Range: src.Range} + got := mark.run.env.DocumentHighlight(loc) - if diff := cmp.Diff(want, got); diff != "" { + sortDocumentHighlights(got) + sortDocumentHighlights(dsts) + + if diff := cmp.Diff(dsts, got, cmpopts.EquateEmpty()); diff != "" { mark.errorf("DocumentHighlight(%v) mismatch (-want +got):\n%s", src, diff) } } diff --git a/gopls/internal/test/marker/testdata/completion/imported-std.txt b/gopls/internal/test/marker/testdata/completion/imported-std.txt index 5f4520f6b6a..bb17a07d4f8 100644 --- a/gopls/internal/test/marker/testdata/completion/imported-std.txt +++ b/gopls/internal/test/marker/testdata/completion/imported-std.txt @@ -28,16 +28,16 @@ import "go/token" import "go/types" // package-level decl -var _ = types.Sat //@rankl("Sat", "Satisfies") -var _ = types.Ali //@rankl("Ali", "!Alias") +var _ = types.Sat //@rank("Sat", "Satisfies") +var _ = types.Ali //@rank("Ali", "!Alias") // field -var _ = new(types.Info).Use //@rankl("Use", "Uses") -var _ = new(types.Info).Fil //@rankl("Fil", "!FileVersions") +var _ = new(types.Info).Use //@rank("Use", "Uses") +var _ = new(types.Info).Fil //@rank("Fil", "!FileVersions") // method -var _ = new(types.Checker).Obje //@rankl("Obje", "ObjectOf") -var _ = new(types.Checker).PkgN //@rankl("PkgN", "!PkgNameOf") +var _ = new(types.Checker).Obje //@rank("Obje", "ObjectOf") +var _ = new(types.Checker).PkgN //@rank("PkgN", "!PkgNameOf") -- b/b.go -- //go:build go1.22 @@ -49,13 +49,13 @@ import "go/token" import "go/types" // package-level decl -var _ = types.Sat //@rankl("Sat", "Satisfies") -var _ = types.Ali //@rankl("Ali", "Alias") +var _ = types.Sat //@rank("Sat", "Satisfies") +var _ = types.Ali //@rank("Ali", "Alias") // field -var _ = new(types.Info).Use //@rankl("Use", "Uses") -var _ = new(types.Info).Fil //@rankl("Fil", "FileVersions") +var _ = new(types.Info).Use //@rank("Use", "Uses") +var _ = new(types.Info).Fil //@rank("Fil", "FileVersions") // method -var _ = new(types.Checker).Obje //@rankl("Obje", "ObjectOf") -var _ = new(types.Checker).PkgN //@rankl("PkgN", "PkgNameOf") +var _ = new(types.Checker).Obje //@rank("Obje", "ObjectOf") +var _ = new(types.Checker).PkgN //@rank("PkgN", "PkgNameOf") diff --git a/gopls/internal/test/marker/testdata/completion/issue62560.txt b/gopls/internal/test/marker/testdata/completion/issue62560.txt index 89763fe0221..b018bd7cdb8 100644 --- a/gopls/internal/test/marker/testdata/completion/issue62560.txt +++ b/gopls/internal/test/marker/testdata/completion/issue62560.txt @@ -10,7 +10,7 @@ module mod.test package foo func _() { - json.U //@rankl(re"U()", "Unmarshal", "InvalidUTF8Error"), diag("json", re"(undefined|undeclared)") + json.U //@rank(re"U()", "Unmarshal", "InvalidUTF8Error"), diag("json", re"(undefined|undeclared)") } -- bar/bar.go -- diff --git a/gopls/internal/test/marker/testdata/completion/range_func.txt b/gopls/internal/test/marker/testdata/completion/range_func.txt index f459cf630ca..638ef9ba1fd 100644 --- a/gopls/internal/test/marker/testdata/completion/range_func.txt +++ b/gopls/internal/test/marker/testdata/completion/range_func.txt @@ -12,12 +12,12 @@ func iter1(func(int) bool) {} func iter2(func(int, int) bool) func _() { - for range i { //@rankl(" {", "iter0", "iterNot"),rankl(" {", "iter1", "iterNot"),rankl(" {", "iter2", "iterNot") + for range i { //@rank(" {", "iter0", "iterNot"),rank(" {", "iter1", "iterNot"),rank(" {", "iter2", "iterNot") } - for k := range i { //@rankl(" {", "iter1", "iterNot"),rankl(" {", "iter1", "iter0"),rankl(" {", "iter2", "iter0") + for k := range i { //@rank(" {", "iter1", "iterNot"),rank(" {", "iter1", "iter0"),rank(" {", "iter2", "iter0") } - for k, v := range i { //@rankl(" {", "iter2", "iterNot"),rankl(" {", "iter2", "iter0"),rankl(" {", "iter2", "iter1") + for k, v := range i { //@rank(" {", "iter2", "iterNot"),rank(" {", "iter2", "iter0"),rank(" {", "iter2", "iter1") } } diff --git a/gopls/internal/test/marker/testdata/completion/type_mods.txt b/gopls/internal/test/marker/testdata/completion/type_mods.txt index de295c62e9a..3988a372b57 100644 --- a/gopls/internal/test/marker/testdata/completion/type_mods.txt +++ b/gopls/internal/test/marker/testdata/completion/type_mods.txt @@ -6,22 +6,22 @@ This test check completion snippets with type modifiers. -- typemods.go -- package typemods -func fooFunc() func() int { //@item(modFooFunc, "fooFunc", "func() func() int", "func") +func fooFunc() func() int { return func() int { return 0 } } -func fooPtr() *int { //@item(modFooPtr, "fooPtr", "func() *int", "func") +func fooPtr() *int { return nil } func _() { - var _ int = foo //@snippet(" //", modFooFunc, "fooFunc()()"),snippet(" //", modFooPtr, "*fooPtr()") + var _ int = foo //@snippet(" //", "fooFunc", "fooFunc()()"),snippet(" //", "fooPtr", "*fooPtr()") } func _() { - var m map[int][]chan int //@item(modMapChanPtr, "m", "map[int]chan *int", "var") + var m map[int][]chan int - var _ int = m //@snippet(" //", modMapChanPtr, "<-m[${1:}][${2:}]") + var _ int = m //@snippet(" //", "m", "<-m[${1:}][${2:}]") } diff --git a/gopls/internal/test/marker/testdata/completion/unimported-std.txt b/gopls/internal/test/marker/testdata/completion/unimported-std.txt index 5eb996a487e..3bedf6bc5bd 100644 --- a/gopls/internal/test/marker/testdata/completion/unimported-std.txt +++ b/gopls/internal/test/marker/testdata/completion/unimported-std.txt @@ -24,20 +24,20 @@ go 1.21 package a // package-level func -var _ = types.Sat //@rankl("Sat", "Satisfies") -var _ = types.Ali //@rankl("Ali", "!Alias") +var _ = types.Sat //@rank("Sat", "Satisfies") +var _ = types.Ali //@rank("Ali", "!Alias") // (We don't offer completions of methods // of types from unimported packages, so the fact that // we don't implement std version filtering isn't evident.) // field -var _ = new(types.Info).Use //@rankl("Use", "!Uses") -var _ = new(types.Info).Fil //@rankl("Fil", "!FileVersions") +var _ = new(types.Info).Use //@rank("Use", "!Uses") +var _ = new(types.Info).Fil //@rank("Fil", "!FileVersions") // method -var _ = new(types.Checker).Obje //@rankl("Obje", "!ObjectOf") -var _ = new(types.Checker).PkgN //@rankl("PkgN", "!PkgNameOf") +var _ = new(types.Checker).Obje //@rank("Obje", "!ObjectOf") +var _ = new(types.Checker).PkgN //@rank("PkgN", "!PkgNameOf") -- b/b.go -- //go:build go1.22 @@ -45,5 +45,5 @@ var _ = new(types.Checker).PkgN //@rankl("PkgN", "!PkgNameOf") package a // package-level decl -var _ = types.Sat //@rankl("Sat", "Satisfies") -var _ = types.Ali //@rankl("Ali", "Alias") +var _ = types.Sat //@rank("Sat", "Satisfies") +var _ = types.Ali //@rank("Ali", "Alias") diff --git a/gopls/internal/test/marker/testdata/highlight/controlflow.txt b/gopls/internal/test/marker/testdata/highlight/controlflow.txt index 25cc9394a47..c09f748a553 100644 --- a/gopls/internal/test/marker/testdata/highlight/controlflow.txt +++ b/gopls/internal/test/marker/testdata/highlight/controlflow.txt @@ -11,12 +11,12 @@ package p -- issue60589.go -- package p -// This test verifies that control flow lighlighting correctly +// This test verifies that control flow highlighting correctly // accounts for multi-name result parameters. // In golang/go#60589, it did not. -func _() (foo int, bar, baz string) { //@ loc(func, "func"), loc(foo, "foo"), loc(fooint, "foo int"), loc(int, "int"), loc(bar, "bar"), loc(beforebaz, " baz"), loc(baz, "baz"), loc(barbazstring, "bar, baz string"), loc(beforestring, re`() string`), loc(string, "string") - return 0, "1", "2" //@ loc(return, `return 0, "1", "2"`), loc(l0, "0"), loc(l1, `"1"`), loc(l2, `"2"`) +func _() (foo int, bar, baz string) { //@ hiloc(func, "func", text), hiloc(foo, "foo", text), hiloc(fooint, "foo int", text), hiloc(int, "int", text), hiloc(bar, "bar", text), hiloc(beforebaz, " baz", text), hiloc(baz, "baz", text), hiloc(barbazstring, "bar, baz string", text), hiloc(beforestring, re`() string`, text), hiloc(string, "string", text) + return 0, "1", "2" //@ hiloc(return, `return 0, "1", "2"`, text), hiloc(l0, "0", text), hiloc(l1, `"1"`, text), hiloc(l2, `"2"`, text) } // Assertions, expressed here to avoid clutter above. @@ -38,8 +38,8 @@ func _() (foo int, bar, baz string) { //@ loc(func, "func"), loc(foo, "foo"), lo // Check that duplicate result names do not cause // inaccurate highlighting. -func _() (x, x int32) { //@ loc(x1, re`\((x)`), loc(x2, re`(x) int`), diag(x1, re"redeclared"), diag(x2, re"redeclared") - return 1, 2 //@ loc(one, "1"), loc(two, "2") +func _() (x, x int32) { //@ loc(locx1, re`\((x)`), loc(locx2, re`(x) int`), hiloc(x1, re`\((x)`, text), hiloc(x2, re`(x) int`, text), diag(locx1, re"redeclared"), diag(locx2, re"redeclared") + return 1, 2 //@ hiloc(one, "1", text), hiloc(two, "2", text) } //@ highlight(one, one, x1) @@ -53,7 +53,8 @@ package p // This test checks that gopls doesn't crash while highlighting // functions with no body (golang/go#65516). -func Foo() (int, string) //@highlight("int", "int"), highlight("func", "func") +func Foo() (int, string) //@hiloc(noBodyInt, "int", text), hiloc(noBodyFunc, "func", text) +//@highlight(noBodyInt, noBodyInt), highlight(noBodyFunc, noBodyFunc) -- issue65952.go -- package p @@ -62,10 +63,12 @@ package p // return values in functions with no results. func _() { - return 0 //@highlight("0", "0"), diag("0", re"too many return") + return 0 //@hiloc(ret1, "0", text), diag("0", re"too many return") + //@highlight(ret1, ret1) } func _() () { // TODO(golang/go#65966): fix the triplicate diagnostics here. - return 0 //@highlight("0", "0"), diag("0", re"too many return"), diag("0", re"too many return"), diag("0", re"too many return") + return 0 //@hiloc(ret2, "0", text), diag("0", re"too many return"), diag("0", re"too many return"), diag("0", re"too many return") + //@highlight(ret2, ret2) } diff --git a/gopls/internal/test/marker/testdata/highlight/highlight.txt b/gopls/internal/test/marker/testdata/highlight/highlight.txt index 10b30259b10..68d13d1ee64 100644 --- a/gopls/internal/test/marker/testdata/highlight/highlight.txt +++ b/gopls/internal/test/marker/testdata/highlight/highlight.txt @@ -4,96 +4,96 @@ This test checks basic functionality of the textDocument/highlight request. package highlights import ( - "fmt" //@loc(fmtImp, "\"fmt\""),highlight(fmtImp, fmtImp, fmt1, fmt2, fmt3, fmt4) - h2 "net/http" //@loc(hImp, "h2"),highlight(hImp, hImp, hUse) + "fmt" //@hiloc(fmtImp, "\"fmt\"", text),highlightall(fmtImp, fmt1, fmt2, fmt3, fmt4) + h2 "net/http" //@hiloc(hImp, "h2", text),highlightall(hImp, hUse) "sort" ) -type F struct{ bar int } //@loc(barDeclaration, "bar"),highlight(barDeclaration, barDeclaration, bar1, bar2, bar3) +type F struct{ bar int } //@hiloc(barDeclaration, "bar", text),highlightall(barDeclaration, bar1, bar2, bar3) func _() F { return F{ - bar: 123, //@loc(bar1, "bar"),highlight(bar1, barDeclaration, bar1, bar2, bar3) + bar: 123, //@hiloc(bar1, "bar", write) } } -var foo = F{bar: 52} //@loc(fooDeclaration, "foo"),loc(bar2, "bar"),highlight(fooDeclaration, fooDeclaration, fooUse),highlight(bar2, barDeclaration, bar1, bar2, bar3) +var foo = F{bar: 52} //@hiloc(fooDeclaration, "foo", write),hiloc(bar2, "bar", write),highlightall(fooDeclaration, fooUse) -func Print() { //@loc(printFunc, "Print"),highlight(printFunc, printFunc, printTest) - _ = h2.Client{} //@loc(hUse, "h2"),highlight(hUse, hImp, hUse) +func Print() { //@hiloc(printFunc, "Print", text),highlightall(printFunc, printTest) + _ = h2.Client{} //@hiloc(hUse, "h2", text) - fmt.Println(foo) //@loc(fooUse, "foo"),highlight(fooUse, fooDeclaration, fooUse),loc(fmt1, "fmt"),highlight(fmt1, fmtImp, fmt1, fmt2, fmt3, fmt4) - fmt.Print("yo") //@loc(printSep, "Print"),highlight(printSep, printSep, print1, print2),loc(fmt2, "fmt"),highlight(fmt2, fmtImp, fmt1, fmt2, fmt3, fmt4) + fmt.Println(foo) //@hiloc(fooUse, "foo", read),hiloc(fmt1, "fmt", text) + fmt.Print("yo") //@hiloc(printSep, "Print", text),highlightall(printSep, print1, print2),hiloc(fmt2, "fmt", text) } -func (x *F) Inc() { //@loc(xRightDecl, "x"),loc(xLeftDecl, " *"),highlight(xRightDecl, xRightDecl, xUse),highlight(xLeftDecl, xRightDecl, xUse) - x.bar++ //@loc(xUse, "x"),loc(bar3, "bar"),highlight(xUse, xRightDecl, xUse),highlight(bar3, barDeclaration, bar1, bar2, bar3) +func (x *F) Inc() { //@hiloc(xRightDecl, "x", text),hiloc(xLeftDecl, " *", text),highlightall(xRightDecl, xUse),highlight(xLeftDecl, xRightDecl, xUse) + x.bar++ //@hiloc(xUse, "x", read),hiloc(bar3, "bar", write) } func testFunctions() { - fmt.Print("main start") //@loc(print1, "Print"),highlight(print1, printSep, print1, print2),loc(fmt3, "fmt"),highlight(fmt3, fmtImp, fmt1, fmt2, fmt3, fmt4) - fmt.Print("ok") //@loc(print2, "Print"),highlight(print2, printSep, print1, print2),loc(fmt4, "fmt"),highlight(fmt4, fmtImp, fmt1, fmt2, fmt3, fmt4) - Print() //@loc(printTest, "Print"),highlight(printTest, printFunc, printTest) + fmt.Print("main start") //@hiloc(print1, "Print", text),hiloc(fmt3, "fmt", text) + fmt.Print("ok") //@hiloc(print2, "Print", text),hiloc(fmt4, "fmt", text) + Print() //@hiloc(printTest, "Print", text) } // DocumentHighlight is undefined, so its uses below are type errors. // Nevertheless, document highlighting should still work. -//@diag(doc1, re"undefined|undeclared"), diag(doc2, re"undefined|undeclared"), diag(doc3, re"undefined|undeclared") +//@diag(locdoc1, re"undefined|undeclared"), diag(locdoc2, re"undefined|undeclared"), diag(locdoc3, re"undefined|undeclared") -func toProtocolHighlight(rngs []int) []DocumentHighlight { //@loc(doc1, "DocumentHighlight"),loc(docRet1, "[]DocumentHighlight"),highlight(doc1, docRet1, doc1, doc2, doc3, result) - result := make([]DocumentHighlight, 0, len(rngs)) //@loc(doc2, "DocumentHighlight"),highlight(doc2, doc1, doc2, doc3) +func toProtocolHighlight(rngs []int) []DocumentHighlight { //@loc(locdoc1, "DocumentHighlight"), hiloc(doc1, "DocumentHighlight", text),hiloc(docRet1, "[]DocumentHighlight", text),highlight(doc1, docRet1, doc1, doc2, doc3, result) + result := make([]DocumentHighlight, 0, len(rngs)) //@loc(locdoc2, "DocumentHighlight"), hiloc(doc2, "DocumentHighlight", text),highlight(doc2, doc1, doc2, doc3) for _, rng := range rngs { - result = append(result, DocumentHighlight{ //@loc(doc3, "DocumentHighlight"),highlight(doc3, doc1, doc2, doc3) + result = append(result, DocumentHighlight{ //@loc(locdoc3, "DocumentHighlight"), hiloc(doc3, "DocumentHighlight", text),highlight(doc3, doc1, doc2, doc3) Range: rng, }) } - return result //@loc(result, "result") + return result //@hiloc(result, "result", text) } func testForLoops() { - for i := 0; i < 10; i++ { //@loc(forDecl1, "for"),highlight(forDecl1, forDecl1, brk1, cont1) + for i := 0; i < 10; i++ { //@hiloc(forDecl1, "for", text),highlightall(forDecl1, brk1, cont1) if i > 8 { - break //@loc(brk1, "break"),highlight(brk1, forDecl1, brk1, cont1) + break //@hiloc(brk1, "break", text) } if i < 2 { - for j := 1; j < 10; j++ { //@loc(forDecl2, "for"),highlight(forDecl2, forDecl2, cont2) + for j := 1; j < 10; j++ { //@hiloc(forDecl2, "for", text),highlightall(forDecl2, cont2) if j < 3 { - for k := 1; k < 10; k++ { //@loc(forDecl3, "for"),highlight(forDecl3, forDecl3, cont3) + for k := 1; k < 10; k++ { //@hiloc(forDecl3, "for", text),highlightall(forDecl3, cont3) if k < 3 { - continue //@loc(cont3, "continue"),highlight(cont3, forDecl3, cont3) + continue //@hiloc(cont3, "continue", text) } } - continue //@loc(cont2, "continue"),highlight(cont2, forDecl2, cont2) + continue //@hiloc(cont2, "continue", text) } } - continue //@loc(cont1, "continue"),highlight(cont1, forDecl1, brk1, cont1) + continue //@hiloc(cont1, "continue", text) } } arr := []int{} - for i := range arr { //@loc(forDecl4, "for"),highlight(forDecl4, forDecl4, brk4, cont4) + for i := range arr { //@hiloc(forDecl4, "for", text),highlightall(forDecl4, brk4, cont4) if i > 8 { - break //@loc(brk4, "break"),highlight(brk4, forDecl4, brk4, cont4) + break //@hiloc(brk4, "break", text) } if i < 4 { - continue //@loc(cont4, "continue"),highlight(cont4, forDecl4, brk4, cont4) + continue //@hiloc(cont4, "continue", text) } } Outer: - for i := 0; i < 10; i++ { //@loc(forDecl5, "for"),highlight(forDecl5, forDecl5, brk5, brk6, brk8) - break //@loc(brk5, "break"),highlight(brk5, forDecl5, brk5, brk6, brk8) - for { //@loc(forDecl6, "for"),highlight(forDecl6, forDecl6, cont5), diag("for", re"unreachable") + for i := 0; i < 10; i++ { //@hiloc(forDecl5, "for", text),highlightall(forDecl5, brk5, brk6, brk8) + break //@hiloc(brk5, "break", text) + for { //@hiloc(forDecl6, "for", text),highlightall(forDecl6, cont5), diag("for", re"unreachable") if i == 1 { - break Outer //@loc(brk6, "break Outer"),highlight(brk6, forDecl5, brk5, brk6, brk8) + break Outer //@hiloc(brk6, "break Outer", text) } - switch i { //@loc(switch1, "switch"),highlight(switch1, switch1, brk7) + switch i { //@hiloc(switch1, "switch", text),highlightall(switch1, brk7) case 5: - break //@loc(brk7, "break"),highlight(brk7, switch1, brk7) + break //@hiloc(brk7, "break", text) case 6: - continue //@loc(cont5, "continue"),highlight(cont5, forDecl6, cont5) + continue //@hiloc(cont5, "continue", text) case 7: - break Outer //@loc(brk8, "break Outer"),highlight(brk8, forDecl5, brk5, brk6, brk8) + break Outer //@hiloc(brk8, "break Outer", text) } } } @@ -103,56 +103,56 @@ func testSwitch() { var i, j int L1: - for { //@loc(forDecl7, "for"),highlight(forDecl7, forDecl7, brk10, cont6) + for { //@hiloc(forDecl7, "for", text),highlightall(forDecl7, brk10, cont6) L2: - switch i { //@loc(switch2, "switch"),highlight(switch2, switch2, brk11, brk12, brk13) + switch i { //@hiloc(switch2, "switch", text),highlightall(switch2, brk11, brk12, brk13) case 1: - switch j { //@loc(switch3, "switch"),highlight(switch3, switch3, brk9) + switch j { //@hiloc(switch3, "switch", text),highlightall(switch3, brk9) case 1: - break //@loc(brk9, "break"),highlight(brk9, switch3, brk9) + break //@hiloc(brk9, "break", text) case 2: - break L1 //@loc(brk10, "break L1"),highlight(brk10, forDecl7, brk10, cont6) + break L1 //@hiloc(brk10, "break L1", text) case 3: - break L2 //@loc(brk11, "break L2"),highlight(brk11, switch2, brk11, brk12, brk13) + break L2 //@hiloc(brk11, "break L2", text) default: - continue //@loc(cont6, "continue"),highlight(cont6, forDecl7, brk10, cont6) + continue //@hiloc(cont6, "continue", text) } case 2: - break //@loc(brk12, "break"),highlight(brk12, switch2, brk11, brk12, brk13) + break //@hiloc(brk12, "break", text) default: - break L2 //@loc(brk13, "break L2"),highlight(brk13, switch2, brk11, brk12, brk13) + break L2 //@hiloc(brk13, "break L2", text) } } } -func testReturn() bool { //@loc(func1, "func"),loc(bool1, "bool"),highlight(func1, func1, fullRet11, fullRet12),highlight(bool1, bool1, false1, bool2, true1) +func testReturn() bool { //@hiloc(func1, "func", text),hiloc(bool1, "bool", text),highlight(func1, func1, fullRet11, fullRet12),highlight(bool1, bool1, false1, bool2, true1) if 1 < 2 { - return false //@loc(ret11, "return"),loc(fullRet11, "return false"),loc(false1, "false"),highlight(ret11, func1, fullRet11, fullRet12) + return false //@hiloc(ret11, "return", text),hiloc(fullRet11, "return false", text),hiloc(false1, "false", text),highlight(ret11, func1, fullRet11, fullRet12) } candidates := []int{} - sort.SliceStable(candidates, func(i, j int) bool { //@loc(func2, "func"),loc(bool2, "bool"),highlight(func2, func2, fullRet2) - return candidates[i] > candidates[j] //@loc(ret2, "return"),loc(fullRet2, "return candidates[i] > candidates[j]"),highlight(ret2, func2, fullRet2) + sort.SliceStable(candidates, func(i, j int) bool { //@hiloc(func2, "func", text),hiloc(bool2, "bool", text),highlight(func2, func2, fullRet2) + return candidates[i] > candidates[j] //@hiloc(ret2, "return", text),hiloc(fullRet2, "return candidates[i] > candidates[j]", text),highlight(ret2, func2, fullRet2) }) - return true //@loc(ret12, "return"),loc(fullRet12, "return true"),loc(true1, "true"),highlight(ret12, func1, fullRet11, fullRet12) + return true //@hiloc(ret12, "return", text),hiloc(fullRet12, "return true", text),hiloc(true1, "true", text),highlight(ret12, func1, fullRet11, fullRet12) } -func testReturnFields() float64 { //@loc(retVal1, "float64"),highlight(retVal1, retVal1, retVal11, retVal21) +func testReturnFields() float64 { //@hiloc(retVal1, "float64", text),highlight(retVal1, retVal1, retVal11, retVal21) if 1 < 2 { - return 20.1 //@loc(retVal11, "20.1"),highlight(retVal11, retVal1, retVal11, retVal21) + return 20.1 //@hiloc(retVal11, "20.1", text),highlight(retVal11, retVal1, retVal11, retVal21) } - z := 4.3 //@loc(zDecl, "z") - return z //@loc(retVal21, "z"),highlight(retVal21, retVal1, retVal11, zDecl, retVal21) + z := 4.3 //@hiloc(zDecl, "z", write) + return z //@hiloc(retVal21, "z", text),highlight(retVal21, retVal1, retVal11, zDecl, retVal21) } -func testReturnMultipleFields() (float32, string) { //@loc(retVal31, "float32"),loc(retVal32, "string"),highlight(retVal31, retVal31, retVal41, retVal51),highlight(retVal32, retVal32, retVal42, retVal52) - y := "im a var" //@loc(yDecl, "y"), +func testReturnMultipleFields() (float32, string) { //@hiloc(retVal31, "float32", text),hiloc(retVal32, "string", text),highlight(retVal31, retVal31, retVal41, retVal51),highlight(retVal32, retVal32, retVal42, retVal52) + y := "im a var" //@hiloc(yDecl, "y", write), if 1 < 2 { - return 20.1, y //@loc(retVal41, "20.1"),loc(retVal42, "y"),highlight(retVal41, retVal31, retVal41, retVal51),highlight(retVal42, retVal32, yDecl, retVal42, retVal52) + return 20.1, y //@hiloc(retVal41, "20.1", text),hiloc(retVal42, "y", text),highlight(retVal41, retVal31, retVal41, retVal51),highlight(retVal42, retVal32, yDecl, retVal42, retVal52) } - return 4.9, "test" //@loc(retVal51, "4.9"),loc(retVal52, "\"test\""),highlight(retVal51, retVal31, retVal41, retVal51),highlight(retVal52, retVal32, retVal42, retVal52) + return 4.9, "test" //@hiloc(retVal51, "4.9", text),hiloc(retVal52, "\"test\"", text),highlight(retVal51, retVal31, retVal41, retVal51),highlight(retVal52, retVal32, retVal42, retVal52) } -func testReturnFunc() int32 { //@loc(retCall, "int32") - mulch := 1 //@loc(mulchDec, "mulch"),highlight(mulchDec, mulchDec, mulchRet) - return int32(mulch) //@loc(mulchRet, "mulch"),loc(retFunc, "int32"),loc(retTotal, "int32(mulch)"),highlight(mulchRet, mulchDec, mulchRet),highlight(retFunc, retCall, retFunc, retTotal) +func testReturnFunc() int32 { //@hiloc(retCall, "int32", text) + mulch := 1 //@hiloc(mulchDec, "mulch", write),highlight(mulchDec, mulchDec, mulchRet) + return int32(mulch) //@hiloc(mulchRet, "mulch", read),hiloc(retFunc, "int32", text),hiloc(retTotal, "int32(mulch)", text),highlight(mulchRet, mulchDec, mulchRet),highlight(retFunc, retCall, retFunc, retTotal) } diff --git a/gopls/internal/test/marker/testdata/highlight/highlight_kind.txt b/gopls/internal/test/marker/testdata/highlight/highlight_kind.txt new file mode 100644 index 00000000000..bd059f77450 --- /dev/null +++ b/gopls/internal/test/marker/testdata/highlight/highlight_kind.txt @@ -0,0 +1,88 @@ +This test checks textDocument/highlight with highlight kinds. +For example, a use of a variable is reported as a "read", +and an assignment to a variable is reported as a "write". +(Note that the details don't align exactly with the Go +type-checker notions of values versus addressable variables). + + +-- highlight_kind.go -- +package a + +type Nest struct { + nest *Nest //@hiloc(fNest, "nest", text) +} +type MyMap map[string]string + +type NestMap map[Nest]Nest + +func highlightTest() { + const constIdent = 1 //@hiloc(constIdent, "constIdent", write) + //@highlightall(constIdent) + var varNoInit int //@hiloc(varNoInit, "varNoInit", write) + (varNoInit) = 1 //@hiloc(varNoInitAssign, "varNoInit", write) + _ = varNoInit //@hiloc(varNoInitRead, "varNoInit", read) + //@highlightall(varNoInit, varNoInitAssign, varNoInitRead) + + str, num := "hello", 2 //@hiloc(str, "str", write), hiloc(num, "num", write) + _, _ = str, num //@hiloc(strRead, "str", read), hiloc(numRead, "num", read) + //@highlightall(str, strRead, strMapKey, strMapVal, strMyMapKey, strMyMapVal, strMyMapSliceKey, strMyMapSliceVal, strMyMapPtrSliceKey, strMyMapPtrSliceVal) + //@highlightall(num, numRead, numAddr, numIncr, numMul) + nest := &Nest{nest: nil} //@hiloc(nest, "nest", write),hiloc(fNestComp, re`(nest):`, write) + nest.nest = &Nest{} //@hiloc(nestSelX, "nest", read), hiloc(fNestSel, re`(nest) =`, write) + *nest.nest = Nest{} //@hiloc(nestSelXStar, "nest", read), hiloc(fNestSelStar, re`(nest) =`, write) + //@highlightall(nest, nestSelX, nestSelXStar, nestMapVal) + //@highlightall(fNest, fNestComp, fNestSel, fNestSelStar, fNestSliceComp, fNestPtrSliceComp, fNestMapKey) + + pInt := &num //@hiloc(pInt, "pInt", write),hiloc(numAddr, "num", read) + // StarExpr is reported as "write" in GoLand and Rust Analyzer + *pInt = 3 //@hiloc(pIntStar, "pInt", write) + var ppInt **int = &pInt //@hiloc(ppInt, "ppInt", write),hiloc(pIntAddr, re`&(pInt)`, read) + **ppInt = 4 //@hiloc(ppIntStar, "ppInt", write) + *(*ppInt) = 4 //@hiloc(ppIntParen, "ppInt", write) + //@highlightall(pInt, pIntStar, pIntAddr) + //@highlightall(ppInt, ppIntStar, ppIntParen) + + num++ //@hiloc(numIncr, "num", write) + num *= 1 //@hiloc(numMul, "num", write) + + var ch chan int = make(chan int, 10) //@hiloc(ch, "ch", write) + ch <- 3 //@hiloc(chSend, "ch", write) + <-ch //@hiloc(chRecv, "ch", read) + //@highlightall(ch, chSend, chRecv) + + var nums []int = []int{1, 2} //@hiloc(nums, "nums", write) + // IndexExpr is reported as "read" in GoLand, Rust Analyzer and Java JDT + nums[0] = 1 //@hiloc(numsIndex, "nums", read) + //@highlightall(nums, numsIndex) + + mapLiteral := map[string]string{ //@hiloc(mapLiteral, "mapLiteral", write) + str: str, //@hiloc(strMapKey, "str", read),hiloc(strMapVal, re`(str),`, read) + } + for key, value := range mapLiteral { //@hiloc(mapKey, "key", write), hiloc(mapVal, "value", write), hiloc(mapLiteralRange, "mapLiteral", read) + _, _ = key, value //@hiloc(mapKeyRead, "key", read), hiloc(mapValRead, "value", read) + } + //@highlightall(mapLiteral, mapLiteralRange) + //@highlightall(mapKey, mapKeyRead) + //@highlightall(mapVal, mapValRead) + + nestSlice := []Nest{ + {nest: nil}, //@hiloc(fNestSliceComp, "nest", write) + } + nestPtrSlice := []*Nest{ + {nest: nil}, //@hiloc(fNestPtrSliceComp, "nest", write) + } + myMap := MyMap{ + str: str, //@hiloc(strMyMapKey, "str", read),hiloc(strMyMapVal, re`(str),`, read) + } + myMapSlice := []MyMap{ + {str: str}, //@hiloc(strMyMapSliceKey, "str", read),hiloc(strMyMapSliceVal, re`: (str)`, read) + } + myMapPtrSlice := []*MyMap{ + {str: str}, //@hiloc(strMyMapPtrSliceKey, "str", read),hiloc(strMyMapPtrSliceVal, re`: (str)`, read) + } + nestMap := NestMap{ + Nest{nest: nil}: *nest, //@hiloc(fNestMapKey, "nest", write), hiloc(nestMapVal, re`(nest),`, read) + } + + _, _, _, _, _, _ = myMap, nestSlice, nestPtrSlice, myMapSlice, myMapPtrSlice, nestMap +} diff --git a/gopls/internal/test/marker/testdata/highlight/issue60435.txt b/gopls/internal/test/marker/testdata/highlight/issue60435.txt index 324e4b85e77..0eef08029ee 100644 --- a/gopls/internal/test/marker/testdata/highlight/issue60435.txt +++ b/gopls/internal/test/marker/testdata/highlight/issue60435.txt @@ -7,9 +7,9 @@ such as httptest. package highlights import ( - "net/http" //@loc(httpImp, `"net/http"`) - "net/http/httptest" //@loc(httptestImp, `"net/http/httptest"`) + "net/http" //@hiloc(httpImp, `"net/http"`, text) + "net/http/httptest" //@hiloc(httptestImp, `"net/http/httptest"`, text) ) var _ = httptest.NewRequest -var _ = http.NewRequest //@loc(here, "http"), highlight(here, here, httpImp) +var _ = http.NewRequest //@hiloc(here, "http", text), highlight(here, here, httpImp) diff --git a/gopls/internal/test/marker/testdata/highlight/switchbreak.txt b/gopls/internal/test/marker/testdata/highlight/switchbreak.txt index b486ad1d80d..3893b4c502d 100644 --- a/gopls/internal/test/marker/testdata/highlight/switchbreak.txt +++ b/gopls/internal/test/marker/testdata/highlight/switchbreak.txt @@ -7,15 +7,15 @@ package a func _(x any) { for { // type switch - switch x.(type) { //@loc(tswitch, "switch") + switch x.(type) { //@hiloc(tswitch, "switch", text) default: - break //@highlight("break", tswitch, "break") + break //@hiloc(tbreak, "break", text),highlight(tbreak, tswitch, tbreak) } // value switch - switch { //@loc(vswitch, "switch") + switch { //@hiloc(vswitch, "switch", text) default: - break //@highlight("break", vswitch, "break") + break //@hiloc(vbreak, "break", text), highlight(vbreak, vswitch, vbreak) } } } diff --git a/gopls/internal/test/marker/testdata/references/issue67978.txt b/gopls/internal/test/marker/testdata/references/issue67978.txt new file mode 100644 index 00000000000..c214116e74d --- /dev/null +++ b/gopls/internal/test/marker/testdata/references/issue67978.txt @@ -0,0 +1,18 @@ + +This test exercises a references query on an exported method that +conflicts with a field name. This ill-typed input violates the +assumption that if type T has a method, then the method set of T is +nonempty, which led to a crash. + +See https://github.com/golang/go/issues/67978. + +-- a.go -- +package p + +type E struct { X int } //@ diag(re"()X", re"field.*same name") + +func (E) X() {} //@ loc(a, "X"), refs("X", a, b), diag(re"()X", re"method.*same name") + +var _ = new(E).X //@ loc(b, "X") + + diff --git a/gopls/internal/test/marker/testdata/token/comment.txt b/gopls/internal/test/marker/testdata/token/comment.txt index 082e95491dd..a5ce9139c4e 100644 --- a/gopls/internal/test/marker/testdata/token/comment.txt +++ b/gopls/internal/test/marker/testdata/token/comment.txt @@ -21,7 +21,7 @@ var B = 2 type Foo int -// [F] accept a [Foo], and print it. //@token("F", "function", ""),token("Foo", "type", "") +// [F] accept a [Foo], and print it. //@token("F", "function", ""),token("Foo", "type", "defaultLibrary number") func F(v Foo) { println(v) @@ -44,7 +44,7 @@ func F2(s string) { -- b.go -- package p -// [F3] accept [*Foo] //@token("F3", "function", ""),token("Foo", "type", "") +// [F3] accept [*Foo] //@token("F3", "function", ""),token("Foo", "type", "defaultLibrary number") func F3(v *Foo) { println(*v) } diff --git a/gopls/internal/util/lru/lru.go b/gopls/internal/util/lru/lru.go index b75fc852d2d..4ed8eafad76 100644 --- a/gopls/internal/util/lru/lru.go +++ b/gopls/internal/util/lru/lru.go @@ -11,45 +11,79 @@ import ( "sync" ) -// A Cache is a fixed-size in-memory LRU cache. -type Cache struct { - capacity int +// A Cache is a fixed-size in-memory LRU cache, storing values of type V keyed +// by keys of type K. +type Cache[K comparable, V any] struct { + impl *cache +} - mu sync.Mutex - used int // used capacity, in user-specified units - m map[any]*entry // k/v lookup - lru queue // min-atime priority queue of *entry - clock int64 // clock time, incremented whenever the cache is updated +// Get retrieves the value for the specified key. +// If the key is found, its access time is updated. +// +// The second result reports whether the key was found. +func (c *Cache[K, V]) Get(key K) (V, bool) { + v, ok := c.impl.get(key) + if !ok { + var zero V + return zero, false + } + // Handle untyped nil explicitly to avoid a panic in the type assertion + // below. + if v == nil { + var zero V + return zero, true + } + return v.(V), true } -type entry struct { - key any - value any - size int // caller-specified size - atime int64 // last access / set time - index int // index of entry in the heap slice +// Set stores a value for the specified key, using its given size to update the +// current cache size, evicting old entries as necessary to fit in the cache +// capacity. +// +// Size must be a non-negative value. If size is larger than the cache +// capacity, the value is not stored and the cache is not modified. +func (c *Cache[K, V]) Set(key K, value V, size int) { + c.impl.set(key, value, size) } // New creates a new Cache with the given capacity, which must be positive. // // The cache capacity uses arbitrary units, which are specified during the Set // operation. -func New(capacity int) *Cache { +func New[K comparable, V any](capacity int) *Cache[K, V] { if capacity == 0 { panic("zero capacity") } - return &Cache{ + return &Cache[K, V]{&cache{ capacity: capacity, m: make(map[any]*entry), - } + }} } -// Get retrieves the value for the specified key, or nil if the key is not -// found. +// cache is the non-generic implementation of [Cache]. // -// If the key is found, its access time is updated. -func (c *Cache) Get(key any) any { +// (Using a generic wrapper around a non-generic impl avoids unnecessary +// "stenciling" or code duplication.) +type cache struct { + capacity int + + mu sync.Mutex + used int // used capacity, in user-specified units + m map[any]*entry // k/v lookup + lru queue // min-atime priority queue of *entry + clock int64 // clock time, incremented whenever the cache is updated +} + +type entry struct { + key any + value any + size int // caller-specified size + atime int64 // last access / set time + index int // index of entry in the heap slice +} + +func (c *cache) get(key any) (any, bool) { c.mu.Lock() defer c.mu.Unlock() @@ -58,19 +92,13 @@ func (c *Cache) Get(key any) any { if e, ok := c.m[key]; ok { // cache hit e.atime = c.clock heap.Fix(&c.lru, e.index) - return e.value + return e.value, true } - return nil + return nil, false } -// Set stores a value for the specified key, using its given size to update the -// current cache size, evicting old entries as necessary to fit in the cache -// capacity. -// -// Size must be a non-negative value. If size is larger than the cache -// capacity, the value is not stored and the cache is not modified. -func (c *Cache) Set(key, value any, size int) { +func (c *cache) set(key, value any, size int) { if size < 0 { panic(fmt.Sprintf("size must be non-negative, got %d", size)) } diff --git a/gopls/internal/util/lru/lru_fuzz_test.go b/gopls/internal/util/lru/lru_fuzz_test.go index b82776b25ba..2f5f43cb9f5 100644 --- a/gopls/internal/util/lru/lru_fuzz_test.go +++ b/gopls/internal/util/lru/lru_fuzz_test.go @@ -22,14 +22,14 @@ func FuzzCache(f *testing.F) { ops = append(ops, op{data[0]%2 == 0, data[1], data[2]}) data = data[3:] } - cache := lru.New(100) + cache := lru.New[byte, byte](100) var reference [256]byte for _, op := range ops { if op.set { reference[op.key] = op.value cache.Set(op.key, op.value, 1) } else { - if v := cache.Get(op.key); v != nil && v != reference[op.key] { + if v, ok := cache.Get(op.key); ok && v != reference[op.key] { t.Fatalf("cache.Get(%d) = %d, want %d", op.key, v, reference[op.key]) } } diff --git a/gopls/internal/util/lru/lru_nil_test.go b/gopls/internal/util/lru/lru_nil_test.go new file mode 100644 index 00000000000..08ce910989c --- /dev/null +++ b/gopls/internal/util/lru/lru_nil_test.go @@ -0,0 +1,25 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lru_test + +// TODO(rfindley): uncomment once -lang is at least go1.20. +// Prior to that language version, interfaces did not satisfy comparable. +// Note that we can't simply use //go:build go1.20, because we need at least Go +// 1.21 in the go.mod file for file language versions support! +/* +import ( + "testing" + + "golang.org/x/tools/gopls/internal/util/lru" +) + +func TestSetUntypedNil(t *testing.T) { + cache := lru.New[any, any](100 * 1e6) + cache.Set(nil, nil, 1) + if got, ok := cache.Get(nil); !ok || got != nil { + t.Errorf("cache.Get(nil) = %v, %v, want nil, true", got, ok) + } +} +*/ diff --git a/gopls/internal/util/lru/lru_test.go b/gopls/internal/util/lru/lru_test.go index 9ffe346257d..bf96e8d31b7 100644 --- a/gopls/internal/util/lru/lru_test.go +++ b/gopls/internal/util/lru/lru_test.go @@ -20,7 +20,7 @@ import ( func TestCache(t *testing.T) { type get struct { key string - want any + want string } type set struct { key, value string @@ -31,8 +31,8 @@ func TestCache(t *testing.T) { steps []any }{ {"empty cache", []any{ - get{"a", nil}, - get{"b", nil}, + get{"a", ""}, + get{"b", ""}, }}, {"zero-length string", []any{ set{"a", ""}, @@ -48,7 +48,7 @@ func TestCache(t *testing.T) { set{"a", "123"}, set{"b", "456"}, set{"c", "78901"}, - get{"a", nil}, + get{"a", ""}, get{"b", "456"}, get{"c", "78901"}, }}, @@ -58,18 +58,18 @@ func TestCache(t *testing.T) { get{"a", "123"}, set{"c", "78901"}, get{"a", "123"}, - get{"b", nil}, + get{"b", ""}, get{"c", "78901"}, }}, } for _, test := range tests { t.Run(test.label, func(t *testing.T) { - c := lru.New(10) + c := lru.New[string, string](10) for i, step := range test.steps { switch step := step.(type) { case get: - if got := c.Get(step.key); got != step.want { + if got, _ := c.Get(step.key); got != step.want { t.Errorf("#%d: c.Get(%q) = %q, want %q", i, step.key, got, step.want) } case set: @@ -96,21 +96,20 @@ func TestConcurrency(t *testing.T) { } } - cache := lru.New(100 * 1e6) // 100MB cache + cache := lru.New[[32]byte, []byte](100 * 1e6) // 100MB cache // get calls Get and verifies that the cache entry // matches one of the values passed to Set. get := func(mustBeFound bool) error { - got := cache.Get(key) - if got == nil { + got, ok := cache.Get(key) + if !ok { if !mustBeFound { return nil } return fmt.Errorf("Get did not return a value") } - gotBytes := got.([]byte) for _, want := range values { - if bytes.Equal(want[:], gotBytes) { + if bytes.Equal(want[:], got) { return nil // a match } } diff --git a/gopls/internal/vulncheck/vulntest/db.go b/gopls/internal/vulncheck/vulntest/db.go index e661b83bc71..ee2a6923264 100644 --- a/gopls/internal/vulncheck/vulntest/db.go +++ b/gopls/internal/vulncheck/vulntest/db.go @@ -51,7 +51,7 @@ func NewDatabase(ctx context.Context, txtarReports []byte) (*DB, error) { // DB is a read-only vulnerability database on disk. // Users can use this database with golang.org/x/vuln APIs -// by setting the `VULNDB“ environment variable. +// by setting the `VULNDB` environment variable. type DB struct { disk string } diff --git a/internal/aliases/aliases.go b/internal/aliases/aliases.go index c24c2eee457..f7798e3354e 100644 --- a/internal/aliases/aliases.go +++ b/internal/aliases/aliases.go @@ -22,11 +22,17 @@ import ( // GODEBUG=gotypesalias=... by invoking the type checker. The Enabled // function is expensive and should be called once per task (e.g. // package import), not once per call to NewAlias. -func NewAlias(enabled bool, pos token.Pos, pkg *types.Package, name string, rhs types.Type) *types.TypeName { +// +// Precondition: enabled || len(tparams)==0. +// If materialized aliases are disabled, there must not be any type parameters. +func NewAlias(enabled bool, pos token.Pos, pkg *types.Package, name string, rhs types.Type, tparams []*types.TypeParam) *types.TypeName { if enabled { tname := types.NewTypeName(pos, pkg, name, nil) - newAlias(tname, rhs) + newAlias(tname, rhs, tparams) return tname } + if len(tparams) > 0 { + panic("cannot create an alias with type parameters when gotypesalias is not enabled") + } return types.NewTypeName(pos, pkg, name, rhs) } diff --git a/internal/aliases/aliases_go121.go b/internal/aliases/aliases_go121.go index 6652f7db0fb..a775fcc4bed 100644 --- a/internal/aliases/aliases_go121.go +++ b/internal/aliases/aliases_go121.go @@ -27,7 +27,9 @@ func Origin(alias *Alias) *Alias { panic("unreachabl // Unalias returns the type t for go <=1.21. func Unalias(t types.Type) types.Type { return t } -func newAlias(name *types.TypeName, rhs types.Type) *Alias { panic("unreachable") } +func newAlias(name *types.TypeName, rhs types.Type, tparams []*types.TypeParam) *Alias { + panic("unreachable") +} // Enabled reports whether [NewAlias] should create [types.Alias] types. // diff --git a/internal/aliases/aliases_go122.go b/internal/aliases/aliases_go122.go index 3ef1afeb403..31c159e42e6 100644 --- a/internal/aliases/aliases_go122.go +++ b/internal/aliases/aliases_go122.go @@ -70,10 +70,9 @@ func Unalias(t types.Type) types.Type { return types.Unalias(t) } // newAlias is an internal alias around types.NewAlias. // Direct usage is discouraged as the moment. // Try to use NewAlias instead. -func newAlias(tname *types.TypeName, rhs types.Type) *Alias { +func newAlias(tname *types.TypeName, rhs types.Type, tparams []*types.TypeParam) *Alias { a := types.NewAlias(tname, rhs) - // TODO(go.dev/issue/65455): Remove kludgy workaround to set a.actual as a side-effect. - Unalias(a) + SetTypeParams(a, tparams) return a } diff --git a/internal/aliases/aliases_test.go b/internal/aliases/aliases_test.go index d27fd6dfd57..d19afcc56c9 100644 --- a/internal/aliases/aliases_test.go +++ b/internal/aliases/aliases_test.go @@ -24,7 +24,7 @@ var _ func(*aliases.Alias) *types.TypeName = (*aliases.Alias).Obj // be an *aliases.Alias. func TestNewAlias(t *testing.T) { const source = ` - package P + package p type Named int ` @@ -35,7 +35,7 @@ func TestNewAlias(t *testing.T) { } var conf types.Config - pkg, err := conf.Check("P", fset, []*ast.File{f}, nil) + pkg, err := conf.Check("p", fset, []*ast.File{f}, nil) if err != nil { t.Fatal(err) } @@ -47,15 +47,18 @@ func TestNewAlias(t *testing.T) { } for _, godebug := range []string{ - // "", // The default is in transition; suppress this case for now + // The default gotypesalias value follows the x/tools/go.mod version + // The go.mod is at 1.19 so the default is gotypesalias=0. + // "", // Use the default GODEBUG value. "gotypesalias=0", - "gotypesalias=1"} { + "gotypesalias=1", + } { t.Run(godebug, func(t *testing.T) { t.Setenv("GODEBUG", godebug) enabled := aliases.Enabled() - A := aliases.NewAlias(enabled, token.NoPos, pkg, "A", tv.Type) + A := aliases.NewAlias(enabled, token.NoPos, pkg, "A", tv.Type, nil) if got, want := A.Name(), "A"; got != want { t.Errorf("Expected A.Name()==%q. got %q", want, got) } @@ -75,3 +78,79 @@ func TestNewAlias(t *testing.T) { }) } } + +// TestNewAlias tests that alias.NewAlias can create a parameterized alias +// A[T] of a type whose underlying and Unaliased type is *T. The test then +// instantiates A[Named] and checks that the underlying and Unaliased type +// of A[Named] is *Named. +// +// Requires gotypesalias GODEBUG and aliastypeparams GOEXPERIMENT. +func TestNewParameterizedAlias(t *testing.T) { + testenv.NeedsGoExperiment(t, "aliastypeparams") + + t.Setenv("GODEBUG", "gotypesalias=1") // needed until gotypesalias is removed (1.27). + enabled := aliases.Enabled() + if !enabled { + t.Fatal("Need materialized aliases enabled") + } + + const source = ` + package p + + type Named int + ` + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "hello.go", source, 0) + if err != nil { + t.Fatal(err) + } + + var conf types.Config + pkg, err := conf.Check("p", fset, []*ast.File{f}, nil) + if err != nil { + t.Fatal(err) + } + + // type A[T ~int] = *T + tparam := types.NewTypeParam( + types.NewTypeName(token.NoPos, pkg, "T", nil), + types.NewUnion([]*types.Term{types.NewTerm(true, types.Typ[types.Int])}), + ) + ptrT := types.NewPointer(tparam) + A := aliases.NewAlias(enabled, token.NoPos, pkg, "A", ptrT, []*types.TypeParam{tparam}) + if got, want := A.Name(), "A"; got != want { + t.Errorf("NewAlias: got %q, want %q", got, want) + } + + if got, want := A.Type().Underlying(), ptrT; !types.Identical(got, want) { + t.Errorf("A.Type().Underlying (%q) is not identical to %q", got, want) + } + if got, want := aliases.Unalias(A.Type()), ptrT; !types.Identical(got, want) { + t.Errorf("Unalias(A)==%q is not identical to %q", got, want) + } + + if _, ok := A.Type().(*aliases.Alias); !ok { + t.Errorf("Expected A.Type() to be a types.Alias(). got %q", A.Type()) + } + + pkg.Scope().Insert(A) // Add A to pkg so it is available to types.Eval. + + named, ok := pkg.Scope().Lookup("Named").(*types.TypeName) + if !ok { + t.Fatalf("Failed to Lookup(%q) in package %s", "Named", pkg) + } + ptrNamed := types.NewPointer(named.Type()) + + const expr = `A[Named]` + tv, err := types.Eval(fset, pkg, 0, expr) + if err != nil { + t.Fatalf("Eval(%s) failed: %v", expr, err) + } + + if got, want := tv.Type.Underlying(), ptrNamed; !types.Identical(got, want) { + t.Errorf("A[Named].Type().Underlying (%q) is not identical to %q", got, want) + } + if got, want := aliases.Unalias(tv.Type), ptrNamed; !types.Identical(got, want) { + t.Errorf("Unalias(A[Named])==%q is not identical to %q", got, want) + } +} diff --git a/internal/gcimporter/gcimporter_test.go b/internal/gcimporter/gcimporter_test.go index 95cc36c4d96..1a56af40323 100644 --- a/internal/gcimporter/gcimporter_test.go +++ b/internal/gcimporter/gcimporter_test.go @@ -956,6 +956,67 @@ func TestIssue58296(t *testing.T) { } } +func TestIssueAliases(t *testing.T) { + // This package only handles gc export data. + testenv.NeedsGo1Point(t, 24) + needsCompiler(t, "gc") + testenv.NeedsGoBuild(t) // to find stdlib export data in the build cache + testenv.NeedsGoExperiment(t, "aliastypeparams") + + t.Setenv("GODEBUG", fmt.Sprintf("gotypesalias=%d", 1)) + + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + testoutdir := filepath.Join(tmpdir, "testdata") + + apkg := filepath.Join(testoutdir, "a") + bpkg := filepath.Join(testoutdir, "b") + cpkg := filepath.Join(testoutdir, "c") + + // compile a, b and c into gc export data. + srcdir := filepath.Join("testdata", "aliases") + compilePkg(t, filepath.Join(srcdir, "a"), "a.go", testoutdir, nil, apkg) + compilePkg(t, filepath.Join(srcdir, "b"), "b.go", testoutdir, map[string]string{apkg: filepath.Join(testoutdir, "a.o")}, bpkg) + compilePkg(t, filepath.Join(srcdir, "c"), "c.go", testoutdir, + map[string]string{apkg: filepath.Join(testoutdir, "a.o"), bpkg: filepath.Join(testoutdir, "b.o")}, + cpkg, + ) + + // import c from gc export data using a and b. + pkg, err := Import(map[string]*types.Package{ + apkg: types.NewPackage(apkg, "a"), + bpkg: types.NewPackage(bpkg, "b"), + }, "./c", testoutdir, nil) + if err != nil { + t.Fatal(err) + } + + // Check c's objects and types. + var objs []string + for _, imp := range pkg.Scope().Names() { + obj := pkg.Scope().Lookup(imp) + s := fmt.Sprintf("%s : %s", obj.Name(), obj.Type()) + s = strings.ReplaceAll(s, testoutdir, "testdata") + objs = append(objs, s) + } + sort.Strings(objs) + + want := strings.Join([]string{ + "S : struct{F int}", + "T : struct{F int}", + "U : testdata/a.A[string]", + "V : testdata/a.A[int]", + "W : testdata/b.B[string]", + "X : testdata/b.B[int]", + "Y : testdata/c.c[string]", + "Z : testdata/c.c[int]", + "c : testdata/c.c", + }, ",") + if got := strings.Join(objs, ","); got != want { + t.Errorf("got imports %v for package c. wanted %v", objs, want) + } +} + // apkg returns the package "a" prefixed by (as a package) testoutdir func apkg(testoutdir string) string { apkg := testoutdir + "/a" diff --git a/internal/gcimporter/iexport.go b/internal/gcimporter/iexport.go index deeb67f315a..5f283281a25 100644 --- a/internal/gcimporter/iexport.go +++ b/internal/gcimporter/iexport.go @@ -2,9 +2,227 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Indexed binary package export. -// This file was derived from $GOROOT/src/cmd/compile/internal/gc/iexport.go; -// see that file for specification of the format. +// Indexed package export. +// +// The indexed export data format is an evolution of the previous +// binary export data format. Its chief contribution is introducing an +// index table, which allows efficient random access of individual +// declarations and inline function bodies. In turn, this allows +// avoiding unnecessary work for compilation units that import large +// packages. +// +// +// The top-level data format is structured as: +// +// Header struct { +// Tag byte // 'i' +// Version uvarint +// StringSize uvarint +// DataSize uvarint +// } +// +// Strings [StringSize]byte +// Data [DataSize]byte +// +// MainIndex []struct{ +// PkgPath stringOff +// PkgName stringOff +// PkgHeight uvarint +// +// Decls []struct{ +// Name stringOff +// Offset declOff +// } +// } +// +// Fingerprint [8]byte +// +// uvarint means a uint64 written out using uvarint encoding. +// +// []T means a uvarint followed by that many T objects. In other +// words: +// +// Len uvarint +// Elems [Len]T +// +// stringOff means a uvarint that indicates an offset within the +// Strings section. At that offset is another uvarint, followed by +// that many bytes, which form the string value. +// +// declOff means a uvarint that indicates an offset within the Data +// section where the associated declaration can be found. +// +// +// There are five kinds of declarations, distinguished by their first +// byte: +// +// type Var struct { +// Tag byte // 'V' +// Pos Pos +// Type typeOff +// } +// +// type Func struct { +// Tag byte // 'F' or 'G' +// Pos Pos +// TypeParams []typeOff // only present if Tag == 'G' +// Signature Signature +// } +// +// type Const struct { +// Tag byte // 'C' +// Pos Pos +// Value Value +// } +// +// type Type struct { +// Tag byte // 'T' or 'U' +// Pos Pos +// TypeParams []typeOff // only present if Tag == 'U' +// Underlying typeOff +// +// Methods []struct{ // omitted if Underlying is an interface type +// Pos Pos +// Name stringOff +// Recv Param +// Signature Signature +// } +// } +// +// type Alias struct { +// Tag byte // 'A' or 'B' +// Pos Pos +// TypeParams []typeOff // only present if Tag == 'B' +// Type typeOff +// } +// +// // "Automatic" declaration of each typeparam +// type TypeParam struct { +// Tag byte // 'P' +// Pos Pos +// Implicit bool +// Constraint typeOff +// } +// +// typeOff means a uvarint that either indicates a predeclared type, +// or an offset into the Data section. If the uvarint is less than +// predeclReserved, then it indicates the index into the predeclared +// types list (see predeclared in bexport.go for order). Otherwise, +// subtracting predeclReserved yields the offset of a type descriptor. +// +// Value means a type, kind, and type-specific value. See +// (*exportWriter).value for details. +// +// +// There are twelve kinds of type descriptors, distinguished by an itag: +// +// type DefinedType struct { +// Tag itag // definedType +// Name stringOff +// PkgPath stringOff +// } +// +// type PointerType struct { +// Tag itag // pointerType +// Elem typeOff +// } +// +// type SliceType struct { +// Tag itag // sliceType +// Elem typeOff +// } +// +// type ArrayType struct { +// Tag itag // arrayType +// Len uint64 +// Elem typeOff +// } +// +// type ChanType struct { +// Tag itag // chanType +// Dir uint64 // 1 RecvOnly; 2 SendOnly; 3 SendRecv +// Elem typeOff +// } +// +// type MapType struct { +// Tag itag // mapType +// Key typeOff +// Elem typeOff +// } +// +// type FuncType struct { +// Tag itag // signatureType +// PkgPath stringOff +// Signature Signature +// } +// +// type StructType struct { +// Tag itag // structType +// PkgPath stringOff +// Fields []struct { +// Pos Pos +// Name stringOff +// Type typeOff +// Embedded bool +// Note stringOff +// } +// } +// +// type InterfaceType struct { +// Tag itag // interfaceType +// PkgPath stringOff +// Embeddeds []struct { +// Pos Pos +// Type typeOff +// } +// Methods []struct { +// Pos Pos +// Name stringOff +// Signature Signature +// } +// } +// +// // Reference to a type param declaration +// type TypeParamType struct { +// Tag itag // typeParamType +// Name stringOff +// PkgPath stringOff +// } +// +// // Instantiation of a generic type (like List[T2] or List[int]) +// type InstanceType struct { +// Tag itag // instanceType +// Pos pos +// TypeArgs []typeOff +// BaseType typeOff +// } +// +// type UnionType struct { +// Tag itag // interfaceType +// Terms []struct { +// tilde bool +// Type typeOff +// } +// } +// +// +// +// type Signature struct { +// Params []Param +// Results []Param +// Variadic bool // omitted if Results is empty +// } +// +// type Param struct { +// Pos Pos +// Name stringOff +// Type typOff +// } +// +// +// Pos encodes a file:line:column triple, incorporating a simple delta +// encoding scheme within a data object. See exportWriter.pos for +// details. package gcimporter @@ -523,9 +741,22 @@ func (p *iexporter) doDecl(obj types.Object) { } if obj.IsAlias() { - w.tag(aliasTag) + alias, materialized := t.(*aliases.Alias) // may fail when aliases are not enabled + + var tparams *types.TypeParamList + if materialized { + tparams = aliases.TypeParams(alias) + } + if tparams.Len() == 0 { + w.tag(aliasTag) + } else { + w.tag(genericAliasTag) + } w.pos(obj.Pos()) - if alias, ok := t.(*aliases.Alias); ok { + if tparams.Len() > 0 { + w.tparamList(obj.Name(), tparams, obj.Pkg()) + } + if materialized { // Preserve materialized aliases, // even of non-exported types. t = aliases.Rhs(alias) @@ -745,7 +976,13 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) { } switch t := t.(type) { case *aliases.Alias: - // TODO(adonovan): support parameterized aliases, following *types.Named. + if targs := aliases.TypeArgs(t); targs.Len() > 0 { + w.startType(instanceType) + w.pos(t.Obj().Pos()) + w.typeList(targs, pkg) + w.typ(aliases.Origin(t), pkg) + return + } w.startType(aliasType) w.qualifiedType(t.Obj()) diff --git a/internal/gcimporter/iexport_test.go b/internal/gcimporter/iexport_test.go index da68e57d554..7e82a58189f 100644 --- a/internal/gcimporter/iexport_test.go +++ b/internal/gcimporter/iexport_test.go @@ -16,12 +16,14 @@ import ( "go/ast" "go/build" "go/constant" + "go/importer" "go/parser" "go/token" "go/types" "io" "math/big" "os" + "path/filepath" "reflect" "runtime" "sort" @@ -454,3 +456,146 @@ func TestUnexportedStructFields(t *testing.T) { type importerFunc func(path string) (*types.Package, error) func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } + +// TestIExportDataTypeParameterizedAliases tests IExportData +// on both declarations and uses of type parameterized aliases. +func TestIExportDataTypeParameterizedAliases(t *testing.T) { + testenv.NeedsGo1Point(t, 23) + + testenv.NeedsGoExperiment(t, "aliastypeparams") + t.Setenv("GODEBUG", "gotypesalias=1") + + // High level steps: + // * parse and typecheck + // * export the data for the importer (via IExportData), + // * import the data (via either x/tools or GOROOT's gcimporter), and + // * check the imported types. + + const src = `package a + +type A[T any] = *T +type B[R any, S *R] = []S +type C[U any] = B[U, A[U]] + +type Named int +type Chained = C[Named] // B[Named, A[Named]] = B[Named, *Named] = []*Named +` + + // parse and typecheck + fset1 := token.NewFileSet() + f, err := parser.ParseFile(fset1, "a", src, 0) + if err != nil { + t.Fatal(err) + } + var conf types.Config + pkg1, err := conf.Check("a", fset1, []*ast.File{f}, nil) + if err != nil { + t.Fatal(err) + } + + testcases := map[string]func(t *testing.T) *types.Package{ + // Read the result of IExportData through x/tools/internal/gcimporter.IImportData. + "tools": func(t *testing.T) *types.Package { + // export + exportdata, err := iexport(fset1, gcimporter.IExportVersion, pkg1) + if err != nil { + t.Fatal(err) + } + + // import + imports := make(map[string]*types.Package) + fset2 := token.NewFileSet() + _, pkg2, err := gcimporter.IImportData(fset2, imports, exportdata, pkg1.Path()) + if err != nil { + t.Fatalf("IImportData(%s): %v", pkg1.Path(), err) + } + return pkg2 + }, + // Read the result of IExportData through $GOROOT/src/internal/gcimporter.IImportData. + // + // This test fakes creating an old go object file in indexed format. + // This means that it can be loaded by go/importer or go/types. + // This step is not supported, but it does give test coverage for stdlib. + "goroot": func(t *testing.T) *types.Package { + // Write indexed export data file contents. + // + // TODO(taking): Slightly unclear to what extent this step should be supported by go/importer. + var buf bytes.Buffer + buf.WriteString("go object \n$$B\n") // object file header + if err := gcexportdata.Write(&buf, fset1, pkg1); err != nil { + t.Fatal(err) + } + + // Write export data to temporary file + out := t.TempDir() + name := filepath.Join(out, "a.out") + if err := os.WriteFile(name+".a", buf.Bytes(), 0644); err != nil { + t.Fatal(err) + } + pkg2, err := importer.Default().Import(name) + if err != nil { + t.Fatal(err) + } + return pkg2 + }, + } + + for name, importer := range testcases { + t.Run(name, func(t *testing.T) { + pkg := importer(t) + + obj := pkg.Scope().Lookup("A") + if obj == nil { + t.Fatalf("failed to find %q in package %s", "A", pkg) + } + + // Check that A is type A[T any] = *T. + // TODO(taking): fix how go/types prints parameterized aliases to simplify tests. + alias, ok := obj.Type().(*aliases.Alias) + if !ok { + t.Fatalf("Obj %s is not an Alias", obj) + } + + targs := aliases.TypeArgs(alias) + if targs.Len() != 0 { + t.Errorf("%s has %d type arguments. expected 0", alias, targs.Len()) + } + + tparams := aliases.TypeParams(alias) + if tparams.Len() != 1 { + t.Fatalf("%s has %d type arguments. expected 1", alias, targs.Len()) + } + tparam := tparams.At(0) + if got, want := tparam.String(), "T"; got != want { + t.Errorf("(%q).TypeParams().At(0)=%q. want %q", alias, got, want) + } + + anyt := types.Universe.Lookup("any").Type() + if c := tparam.Constraint(); !types.Identical(anyt, c) { + t.Errorf("(%q).Constraint()=%q. expected %q", tparam, c, anyt) + } + + ptparam := types.NewPointer(tparam) + if rhs := aliases.Rhs(alias); !types.Identical(ptparam, rhs) { + t.Errorf("(%q).Rhs()=%q. expected %q", alias, rhs, ptparam) + } + + // TODO(taking): add tests for B and C once it is simpler to write tests. + + chained := pkg.Scope().Lookup("Chained") + if chained == nil { + t.Fatalf("failed to find %q in package %s", "Chained", pkg) + } + + named, _ := pkg.Scope().Lookup("Named").(*types.TypeName) + if named == nil { + t.Fatalf("failed to find %q in package %s", "Named", pkg) + } + + want := types.NewSlice(types.NewPointer(named.Type())) + if got := chained.Type(); !types.Identical(got, want) { + t.Errorf("(%q).Type()=%q which should be identical to %q", chained, got, want) + } + }) + } +} diff --git a/internal/gcimporter/iimport.go b/internal/gcimporter/iimport.go index 136aa03653c..ed2d5629596 100644 --- a/internal/gcimporter/iimport.go +++ b/internal/gcimporter/iimport.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // Indexed package import. -// See cmd/compile/internal/gc/iexport.go for the export data format. +// See iexport.go for the export data format. // This file is a copy of $GOROOT/src/go/internal/gcimporter/iimport.go. @@ -562,14 +562,14 @@ func (r *importReader) obj(name string) { pos := r.pos() switch tag { - case aliasTag: + case aliasTag, genericAliasTag: + var tparams []*types.TypeParam + if tag == genericAliasTag { + tparams = r.tparamList() + } typ := r.typ() - // TODO(adonovan): support generic aliases: - // if tag == genericAliasTag { - // tparams := r.tparamList() - // alias.SetTypeParams(tparams) - // } - r.declare(aliases.NewAlias(r.p.aliases, pos, r.currPkg, name, typ)) + obj := aliases.NewAlias(r.p.aliases, pos, r.currPkg, name, typ, tparams) + r.declare(obj) case constTag: typ, val := r.value() @@ -862,7 +862,7 @@ func (r *importReader) string() string { return r.p.stringAt(r.uint64()) } func (r *importReader) doType(base *types.Named) (res types.Type) { k := r.kind() if debug { - r.p.trace("importing type %d (base: %s)", k, base) + r.p.trace("importing type %d (base: %v)", k, base) r.p.indent++ defer func() { r.p.indent-- diff --git a/internal/gcimporter/testdata/aliases/a/a.go b/internal/gcimporter/testdata/aliases/a/a.go new file mode 100644 index 00000000000..0558258e17a --- /dev/null +++ b/internal/gcimporter/testdata/aliases/a/a.go @@ -0,0 +1,14 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package a + +type A[T any] = *T + +type B = struct{ F int } + +func F() B { + type a[T any] = struct{ F T } + return a[int]{} +} diff --git a/internal/gcimporter/testdata/aliases/b/b.go b/internal/gcimporter/testdata/aliases/b/b.go new file mode 100644 index 00000000000..9a2dbe2bafb --- /dev/null +++ b/internal/gcimporter/testdata/aliases/b/b.go @@ -0,0 +1,11 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package b + +import "./a" + +type B[S any] = struct { + F a.A[[]S] +} diff --git a/internal/gcimporter/testdata/aliases/c/c.go b/internal/gcimporter/testdata/aliases/c/c.go new file mode 100644 index 00000000000..359cee61920 --- /dev/null +++ b/internal/gcimporter/testdata/aliases/c/c.go @@ -0,0 +1,26 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package c + +import ( + "./a" + "./b" +) + +type c[V any] = struct { + G b.B[[3]V] +} + +var S struct{ F int } = a.B{} +var T struct{ F int } = a.F() + +var U a.A[string] = (*string)(nil) +var V a.A[int] = (*int)(nil) + +var W b.B[string] = struct{ F *[]string }{} +var X b.B[int] = struct{ F *[]int }{} + +var Y c[string] = struct{ G struct{ F *[][3]string } }{} +var Z c[int] = struct{ G struct{ F *[][3]int } }{} diff --git a/internal/gcimporter/ureader_yes.go b/internal/gcimporter/ureader_yes.go index 2c077068877..f0742f5404b 100644 --- a/internal/gcimporter/ureader_yes.go +++ b/internal/gcimporter/ureader_yes.go @@ -52,8 +52,7 @@ func (pr *pkgReader) later(fn func()) { // See cmd/compile/internal/noder.derivedInfo. type derivedInfo struct { - idx pkgbits.Index - needed bool + idx pkgbits.Index } // See cmd/compile/internal/noder.typeInfo. @@ -110,13 +109,17 @@ func readUnifiedPackage(fset *token.FileSet, ctxt *types.Context, imports map[st r := pr.newReader(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic) pkg := r.pkg() - r.Bool() // has init + if r.Version().Has(pkgbits.HasInit) { + r.Bool() + } for i, n := 0, r.Len(); i < n; i++ { // As if r.obj(), but avoiding the Scope.Lookup call, // to avoid eager loading of imports. r.Sync(pkgbits.SyncObject) - assert(!r.Bool()) + if r.Version().Has(pkgbits.DerivedFuncInstance) { + assert(!r.Bool()) + } r.p.objIdx(r.Reloc(pkgbits.RelocObj)) assert(r.Len() == 0) } @@ -165,7 +168,7 @@ type readerDict struct { // tparams is a slice of the constructed TypeParams for the element. tparams []*types.TypeParam - // devived is a slice of types derived from tparams, which may be + // derived is a slice of types derived from tparams, which may be // instantiated while reading the current element. derived []derivedInfo derivedTypes []types.Type // lazily instantiated from derived @@ -471,7 +474,9 @@ func (r *reader) param() *types.Var { func (r *reader) obj() (types.Object, []types.Type) { r.Sync(pkgbits.SyncObject) - assert(!r.Bool()) + if r.Version().Has(pkgbits.DerivedFuncInstance) { + assert(!r.Bool()) + } pkg, name := r.p.objIdx(r.Reloc(pkgbits.RelocObj)) obj := pkgScope(pkg).Lookup(name) @@ -525,8 +530,12 @@ func (pr *pkgReader) objIdx(idx pkgbits.Index) (*types.Package, string) { case pkgbits.ObjAlias: pos := r.pos() + var tparams []*types.TypeParam + if r.Version().Has(pkgbits.AliasTypeParamNames) { + tparams = r.typeParamNames() + } typ := r.typ() - declare(aliases.NewAlias(r.p.aliases, pos, objPkg, objName, typ)) + declare(aliases.NewAlias(r.p.aliases, pos, objPkg, objName, typ, tparams)) case pkgbits.ObjConst: pos := r.pos() @@ -632,7 +641,10 @@ func (pr *pkgReader) objDictIdx(idx pkgbits.Index) *readerDict { dict.derived = make([]derivedInfo, r.Len()) dict.derivedTypes = make([]types.Type, len(dict.derived)) for i := range dict.derived { - dict.derived[i] = derivedInfo{r.Reloc(pkgbits.RelocType), r.Bool()} + dict.derived[i] = derivedInfo{idx: r.Reloc(pkgbits.RelocType)} + if r.Version().Has(pkgbits.DerivedInfoNeeded) { + assert(!r.Bool()) + } } pr.retireReader(r) diff --git a/internal/imports/mod.go b/internal/imports/mod.go index 91221fda322..8555e3f83da 100644 --- a/internal/imports/mod.go +++ b/internal/imports/mod.go @@ -245,7 +245,10 @@ func newModuleResolver(e *ProcessEnv, moduleCacheCache *DirInfoCache) (*ModuleRe // 2. Use this to separate module cache scanning from other scanning. func gomodcacheForEnv(goenv map[string]string) string { if gmc := goenv["GOMODCACHE"]; gmc != "" { - return gmc + // golang/go#67156: ensure that the module cache is clean, since it is + // assumed as a prefix to directories scanned by gopathwalk, which are + // themselves clean. + return filepath.Clean(gmc) } gopaths := filepath.SplitList(goenv["GOPATH"]) if len(gopaths) == 0 { @@ -740,8 +743,8 @@ func (r *ModuleResolver) loadExports(ctx context.Context, pkg *pkg, includeTest func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) directoryPackageInfo { subdir := "" - if dir != root.Path { - subdir = dir[len(root.Path)+len("/"):] + if prefix := root.Path + string(filepath.Separator); strings.HasPrefix(dir, prefix) { + subdir = dir[len(prefix):] } importPath := filepath.ToSlash(subdir) if strings.HasPrefix(importPath, "vendor/") { diff --git a/internal/pkgbits/decoder.go b/internal/pkgbits/decoder.go index b92e8e6eb32..f6cb37c5c3d 100644 --- a/internal/pkgbits/decoder.go +++ b/internal/pkgbits/decoder.go @@ -21,7 +21,7 @@ import ( // export data. type PkgDecoder struct { // version is the file format version. - version uint32 + version Version // sync indicates whether the file uses sync markers. sync bool @@ -68,8 +68,6 @@ func (pr *PkgDecoder) SyncMarkers() bool { return pr.sync } // NewPkgDecoder returns a PkgDecoder initialized to read the Unified // IR export data from input. pkgPath is the package path for the // compilation unit that produced the export data. -// -// TODO(mdempsky): Remove pkgPath parameter; unneeded since CL 391014. func NewPkgDecoder(pkgPath, input string) PkgDecoder { pr := PkgDecoder{ pkgPath: pkgPath, @@ -80,14 +78,15 @@ func NewPkgDecoder(pkgPath, input string) PkgDecoder { r := strings.NewReader(input) - assert(binary.Read(r, binary.LittleEndian, &pr.version) == nil) + var ver uint32 + assert(binary.Read(r, binary.LittleEndian, &ver) == nil) + pr.version = Version(ver) - switch pr.version { - default: - panic(fmt.Errorf("unsupported version: %v", pr.version)) - case 0: - // no flags - case 1: + if pr.version >= numVersions { + panic(fmt.Errorf("cannot decode %q, export data version %d is greater than maximum supported version %d", pkgPath, pr.version, numVersions-1)) + } + + if pr.version.Has(Flags) { var flags uint32 assert(binary.Read(r, binary.LittleEndian, &flags) == nil) pr.sync = flags&flagSyncMarkers != 0 @@ -102,7 +101,9 @@ func NewPkgDecoder(pkgPath, input string) PkgDecoder { assert(err == nil) pr.elemData = input[pos:] - assert(len(pr.elemData)-8 == int(pr.elemEnds[len(pr.elemEnds)-1])) + + const fingerprintSize = 8 + assert(len(pr.elemData)-fingerprintSize == int(pr.elemEnds[len(pr.elemEnds)-1])) return pr } @@ -136,7 +137,7 @@ func (pr *PkgDecoder) AbsIdx(k RelocKind, idx Index) int { absIdx += int(pr.elemEndsEnds[k-1]) } if absIdx >= int(pr.elemEndsEnds[k]) { - errorf("%v:%v is out of bounds; %v", k, idx, pr.elemEndsEnds) + panicf("%v:%v is out of bounds; %v", k, idx, pr.elemEndsEnds) } return absIdx } @@ -193,9 +194,7 @@ func (pr *PkgDecoder) NewDecoderRaw(k RelocKind, idx Index) Decoder { Idx: idx, } - // TODO(mdempsky) r.data.Reset(...) after #44505 is resolved. - r.Data = *strings.NewReader(pr.DataIdx(k, idx)) - + r.Data.Reset(pr.DataIdx(k, idx)) r.Sync(SyncRelocs) r.Relocs = make([]RelocEnt, r.Len()) for i := range r.Relocs { @@ -244,7 +243,7 @@ type Decoder struct { func (r *Decoder) checkErr(err error) { if err != nil { - errorf("unexpected decoding error: %w", err) + panicf("unexpected decoding error: %w", err) } } @@ -515,3 +514,6 @@ func (pr *PkgDecoder) PeekObj(idx Index) (string, string, CodeObj) { return path, name, tag } + +// Version reports the version of the bitstream. +func (w *Decoder) Version() Version { return w.common.version } diff --git a/internal/pkgbits/encoder.go b/internal/pkgbits/encoder.go index 6482617a4fc..c17a12399d0 100644 --- a/internal/pkgbits/encoder.go +++ b/internal/pkgbits/encoder.go @@ -12,18 +12,15 @@ import ( "io" "math/big" "runtime" + "strings" ) -// currentVersion is the current version number. -// -// - v0: initial prototype -// -// - v1: adds the flags uint32 word -const currentVersion uint32 = 1 - // A PkgEncoder provides methods for encoding a package's Unified IR // export data. type PkgEncoder struct { + // version of the bitstream. + version Version + // elems holds the bitstream for previously encoded elements. elems [numRelocs][]string @@ -47,8 +44,9 @@ func (pw *PkgEncoder) SyncMarkers() bool { return pw.syncFrames >= 0 } // export data files, but can help diagnosing desync errors in // higher-level Unified IR reader/writer code. If syncFrames is // negative, then sync markers are omitted entirely. -func NewPkgEncoder(syncFrames int) PkgEncoder { +func NewPkgEncoder(version Version, syncFrames int) PkgEncoder { return PkgEncoder{ + version: version, stringsIdx: make(map[string]Index), syncFrames: syncFrames, } @@ -64,13 +62,15 @@ func (pw *PkgEncoder) DumpTo(out0 io.Writer) (fingerprint [8]byte) { assert(binary.Write(out, binary.LittleEndian, x) == nil) } - writeUint32(currentVersion) + writeUint32(uint32(pw.version)) - var flags uint32 - if pw.SyncMarkers() { - flags |= flagSyncMarkers + if pw.version.Has(Flags) { + var flags uint32 + if pw.SyncMarkers() { + flags |= flagSyncMarkers + } + writeUint32(flags) } - writeUint32(flags) // Write elemEndsEnds. var sum uint32 @@ -159,7 +159,7 @@ type Encoder struct { // Flush finalizes the element's bitstream and returns its Index. func (w *Encoder) Flush() Index { - var sb bytes.Buffer // TODO(mdempsky): strings.Builder after #44505 is resolved + var sb strings.Builder // Backup the data so we write the relocations at the front. var tmp bytes.Buffer @@ -189,7 +189,7 @@ func (w *Encoder) Flush() Index { func (w *Encoder) checkErr(err error) { if err != nil { - errorf("unexpected encoding error: %v", err) + panicf("unexpected encoding error: %v", err) } } @@ -320,8 +320,14 @@ func (w *Encoder) Code(c Code) { // section (if not already present), and then writing a relocation // into the element bitstream. func (w *Encoder) String(s string) { + w.StringRef(w.p.StringIdx(s)) +} + +// StringRef writes a reference to the given index, which must be a +// previously encoded string value. +func (w *Encoder) StringRef(idx Index) { w.Sync(SyncString) - w.Reloc(RelocString, w.p.StringIdx(s)) + w.Reloc(RelocString, idx) } // Strings encodes and writes a variable-length slice of strings into @@ -348,7 +354,7 @@ func (w *Encoder) Value(val constant.Value) { func (w *Encoder) scalar(val constant.Value) { switch v := constant.Val(val).(type) { default: - errorf("unhandled %v (%v)", val, val.Kind()) + panicf("unhandled %v (%v)", val, val.Kind()) case bool: w.Code(ValBool) w.Bool(v) @@ -381,3 +387,6 @@ func (w *Encoder) bigFloat(v *big.Float) { b := v.Append(nil, 'p', -1) w.String(string(b)) // TODO: More efficient encoding. } + +// Version reports the version of the bitstream. +func (w *Encoder) Version() Version { return w.p.version } diff --git a/internal/pkgbits/frames_go1.go b/internal/pkgbits/frames_go1.go deleted file mode 100644 index 5294f6a63ed..00000000000 --- a/internal/pkgbits/frames_go1.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !go1.7 -// +build !go1.7 - -// TODO(mdempsky): Remove after #44505 is resolved - -package pkgbits - -import "runtime" - -func walkFrames(pcs []uintptr, visit frameVisitor) { - for _, pc := range pcs { - fn := runtime.FuncForPC(pc) - file, line := fn.FileLine(pc) - - visit(file, line, fn.Name(), pc-fn.Entry()) - } -} diff --git a/internal/pkgbits/frames_go17.go b/internal/pkgbits/frames_go17.go deleted file mode 100644 index 2324ae7adfe..00000000000 --- a/internal/pkgbits/frames_go17.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.7 -// +build go1.7 - -package pkgbits - -import "runtime" - -// walkFrames calls visit for each call frame represented by pcs. -// -// pcs should be a slice of PCs, as returned by runtime.Callers. -func walkFrames(pcs []uintptr, visit frameVisitor) { - if len(pcs) == 0 { - return - } - - frames := runtime.CallersFrames(pcs) - for { - frame, more := frames.Next() - visit(frame.File, frame.Line, frame.Function, frame.PC-frame.Entry) - if !more { - return - } - } -} diff --git a/internal/pkgbits/pkgbits_test.go b/internal/pkgbits/pkgbits_test.go new file mode 100644 index 00000000000..b8f946a0a4f --- /dev/null +++ b/internal/pkgbits/pkgbits_test.go @@ -0,0 +1,77 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgbits_test + +import ( + "strings" + "testing" + + "golang.org/x/tools/internal/pkgbits" +) + +func TestRoundTrip(t *testing.T) { + for _, version := range []pkgbits.Version{ + pkgbits.V0, + pkgbits.V1, + pkgbits.V2, + } { + pw := pkgbits.NewPkgEncoder(version, -1) + w := pw.NewEncoder(pkgbits.RelocMeta, pkgbits.SyncPublic) + w.Flush() + + var b strings.Builder + _ = pw.DumpTo(&b) + input := b.String() + + pr := pkgbits.NewPkgDecoder("package_id", input) + r := pr.NewDecoder(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic) + + if r.Version() != w.Version() { + t.Errorf("Expected reader version %q to be the writer version %q", r.Version(), w.Version()) + } + } +} + +// Type checker to enforce that know V* have the constant values they must have. +var _ [0]bool = [pkgbits.V0]bool{} +var _ [1]bool = [pkgbits.V1]bool{} + +func TestVersions(t *testing.T) { + type vfpair struct { + v pkgbits.Version + f pkgbits.Field + } + + // has field tests + for _, c := range []vfpair{ + {pkgbits.V1, pkgbits.Flags}, + {pkgbits.V2, pkgbits.Flags}, + {pkgbits.V0, pkgbits.HasInit}, + {pkgbits.V1, pkgbits.HasInit}, + {pkgbits.V0, pkgbits.DerivedFuncInstance}, + {pkgbits.V1, pkgbits.DerivedFuncInstance}, + {pkgbits.V0, pkgbits.DerivedInfoNeeded}, + {pkgbits.V1, pkgbits.DerivedInfoNeeded}, + {pkgbits.V2, pkgbits.AliasTypeParamNames}, + } { + if !c.v.Has(c.f) { + t.Errorf("Expected version %v to have field %v", c.v, c.f) + } + } + + // does not have field tests + for _, c := range []vfpair{ + {pkgbits.V0, pkgbits.Flags}, + {pkgbits.V2, pkgbits.HasInit}, + {pkgbits.V2, pkgbits.DerivedFuncInstance}, + {pkgbits.V2, pkgbits.DerivedInfoNeeded}, + {pkgbits.V0, pkgbits.AliasTypeParamNames}, + {pkgbits.V1, pkgbits.AliasTypeParamNames}, + } { + if c.v.Has(c.f) { + t.Errorf("Expected version %v to not have field %v", c.v, c.f) + } + } +} diff --git a/internal/pkgbits/support.go b/internal/pkgbits/support.go index ad26d3b28ca..50534a29553 100644 --- a/internal/pkgbits/support.go +++ b/internal/pkgbits/support.go @@ -12,6 +12,6 @@ func assert(b bool) { } } -func errorf(format string, args ...interface{}) { +func panicf(format string, args ...any) { panic(fmt.Errorf(format, args...)) } diff --git a/internal/pkgbits/sync.go b/internal/pkgbits/sync.go index 5bd51ef7170..1520b73afb9 100644 --- a/internal/pkgbits/sync.go +++ b/internal/pkgbits/sync.go @@ -6,6 +6,7 @@ package pkgbits import ( "fmt" + "runtime" "strings" ) @@ -23,6 +24,24 @@ func fmtFrames(pcs ...uintptr) []string { type frameVisitor func(file string, line int, name string, offset uintptr) +// walkFrames calls visit for each call frame represented by pcs. +// +// pcs should be a slice of PCs, as returned by runtime.Callers. +func walkFrames(pcs []uintptr, visit frameVisitor) { + if len(pcs) == 0 { + return + } + + frames := runtime.CallersFrames(pcs) + for { + frame, more := frames.Next() + visit(frame.File, frame.Line, frame.Function, frame.PC-frame.Entry) + if !more { + return + } + } +} + // SyncMarker is an enum type that represents markers that may be // written to export data to ensure the reader and writer stay // synchronized. @@ -110,4 +129,8 @@ const ( SyncStmtsEnd SyncLabel SyncOptLabel + + SyncMultiExpr + SyncRType + SyncConvRTTI ) diff --git a/internal/pkgbits/syncmarker_string.go b/internal/pkgbits/syncmarker_string.go index 4a5b0ca5f2f..582ad56d3e0 100644 --- a/internal/pkgbits/syncmarker_string.go +++ b/internal/pkgbits/syncmarker_string.go @@ -74,11 +74,14 @@ func _() { _ = x[SyncStmtsEnd-64] _ = x[SyncLabel-65] _ = x[SyncOptLabel-66] + _ = x[SyncMultiExpr-67] + _ = x[SyncRType-68] + _ = x[SyncConvRTTI-69] } -const _SyncMarker_name = "EOFBoolInt64Uint64StringValueValRelocsRelocUseRelocPublicPosPosBaseObjectObject1PkgPkgDefMethodTypeTypeIdxTypeParamNamesSignatureParamsParamCodeObjSymLocalIdentSelectorPrivateFuncExtVarExtTypeExtPragmaExprListExprsExprExprTypeAssignOpFuncLitCompLitDeclFuncBodyOpenScopeCloseScopeCloseAnotherScopeDeclNamesDeclNameStmtsBlockStmtIfStmtForStmtSwitchStmtRangeStmtCaseClauseCommClauseSelectStmtDeclsLabeledStmtUseObjLocalAddLocalLinknameStmt1StmtsEndLabelOptLabel" +const _SyncMarker_name = "EOFBoolInt64Uint64StringValueValRelocsRelocUseRelocPublicPosPosBaseObjectObject1PkgPkgDefMethodTypeTypeIdxTypeParamNamesSignatureParamsParamCodeObjSymLocalIdentSelectorPrivateFuncExtVarExtTypeExtPragmaExprListExprsExprExprTypeAssignOpFuncLitCompLitDeclFuncBodyOpenScopeCloseScopeCloseAnotherScopeDeclNamesDeclNameStmtsBlockStmtIfStmtForStmtSwitchStmtRangeStmtCaseClauseCommClauseSelectStmtDeclsLabeledStmtUseObjLocalAddLocalLinknameStmt1StmtsEndLabelOptLabelMultiExprRTypeConvRTTI" -var _SyncMarker_index = [...]uint16{0, 3, 7, 12, 18, 24, 29, 32, 38, 43, 51, 57, 60, 67, 73, 80, 83, 89, 95, 99, 106, 120, 129, 135, 140, 147, 150, 160, 168, 175, 182, 188, 195, 201, 209, 214, 218, 226, 232, 234, 241, 248, 252, 260, 269, 279, 296, 305, 313, 318, 327, 333, 340, 350, 359, 369, 379, 389, 394, 405, 416, 424, 432, 437, 445, 450, 458} +var _SyncMarker_index = [...]uint16{0, 3, 7, 12, 18, 24, 29, 32, 38, 43, 51, 57, 60, 67, 73, 80, 83, 89, 95, 99, 106, 120, 129, 135, 140, 147, 150, 160, 168, 175, 182, 188, 195, 201, 209, 214, 218, 226, 232, 234, 241, 248, 252, 260, 269, 279, 296, 305, 313, 318, 327, 333, 340, 350, 359, 369, 379, 389, 394, 405, 416, 424, 432, 437, 445, 450, 458, 467, 472, 480} func (i SyncMarker) String() string { i -= 1 diff --git a/internal/pkgbits/version.go b/internal/pkgbits/version.go new file mode 100644 index 00000000000..53af9df22b3 --- /dev/null +++ b/internal/pkgbits/version.go @@ -0,0 +1,85 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgbits + +// Version indicates a version of a unified IR bitstream. +// Each Version indicates the addition, removal, or change of +// new data in the bitstream. +// +// These are serialized to disk and the interpretation remains fixed. +type Version uint32 + +const ( + // V0: initial prototype. + // + // All data that is not assigned a Field is in version V0 + // and has not been deprecated. + V0 Version = iota + + // V1: adds the Flags uint32 word + V1 + + // V2: removes unused legacy fields and supports type parameters for aliases. + // - remove the legacy "has init" bool from the public root + // - remove obj's "derived func instance" bool + // - add a TypeParamNames field to ObjAlias + // - remove derived info "needed" bool + V2 + + numVersions = iota +) + +// Field denotes a unit of data in the serialized unified IR bitstream. +// It is conceptually a like field in a structure. +// +// We only really need Fields when the data may or may not be present +// in a stream based on the Version of the bitstream. +// +// Unlike much of pkgbits, Fields are not serialized and +// can change values as needed. +type Field int + +const ( + // Flags in a uint32 in the header of a bitstream + // that is used to indicate whether optional features are enabled. + Flags Field = iota + + // Deprecated: HasInit was a bool indicating whether a package + // has any init functions. + HasInit + + // Deprecated: DerivedFuncInstance was a bool indicating + // whether an object was a function instance. + DerivedFuncInstance + + // ObjAlias has a list of TypeParamNames. + AliasTypeParamNames + + // Deprecated: DerivedInfoNeeded was a bool indicating + // whether a type was a derived type. + DerivedInfoNeeded + + numFields = iota +) + +// introduced is the version a field was added. +var introduced = [numFields]Version{ + Flags: V1, + AliasTypeParamNames: V2, +} + +// removed is the version a field was removed in or 0 for fields +// that have not yet been deprecated. +// (So removed[f]-1 is the last version it is included in.) +var removed = [numFields]Version{ + HasInit: V2, + DerivedFuncInstance: V2, + DerivedInfoNeeded: V2, +} + +// Has reports whether field f is present in a bitstream at version v. +func (v Version) Has(f Field) bool { + return introduced[f] <= v && (v < removed[f] || removed[f] == V0) +} diff --git a/internal/stdlib/manifest.go b/internal/stdlib/manifest.go index a928acf29fa..cdaac9ab34d 100644 --- a/internal/stdlib/manifest.go +++ b/internal/stdlib/manifest.go @@ -951,7 +951,7 @@ var PackageSymbols = map[string][]Symbol{ {"ParseSessionState", Func, 21}, {"QUICClient", Func, 21}, {"QUICConfig", Type, 21}, - {"QUICConfig.EnableStoreSessionEvent", Field, 23}, + {"QUICConfig.EnableSessionEvents", Field, 23}, {"QUICConfig.TLSConfig", Field, 21}, {"QUICConn", Type, 21}, {"QUICEncryptionLevel", Type, 21}, diff --git a/internal/testfiles/testdata/versions/go.mod.test b/internal/testfiles/testdata/versions/go.mod.test index 55f69e1a51e..0dfc6f15a11 100644 --- a/internal/testfiles/testdata/versions/go.mod.test +++ b/internal/testfiles/testdata/versions/go.mod.test @@ -2,4 +2,4 @@ module golang.org/fake/versions -go 1.21 +go 1.22 diff --git a/internal/testfiles/testdata/versions/mod.go b/internal/testfiles/testdata/versions/mod.go index 13117fda09c..bd0bc18ac65 100644 --- a/internal/testfiles/testdata/versions/mod.go +++ b/internal/testfiles/testdata/versions/mod.go @@ -1,3 +1,3 @@ -// The file will be go1.21 from the go.mod. +// The file will be go1.22 from the go.mod. -package versions // want "mod.go@go1.21" +package versions // want "mod.go@go1.22" diff --git a/internal/testfiles/testdata/versions/post.go b/internal/testfiles/testdata/versions/post.go index 8c1afde22ff..c7eef6eeaa9 100644 --- a/internal/testfiles/testdata/versions/post.go +++ b/internal/testfiles/testdata/versions/post.go @@ -1,3 +1,3 @@ -//go:build go1.22 +//go:build go1.23 -package versions // want "post.go@go1.22" +package versions // want "post.go@go1.23" diff --git a/internal/testfiles/testdata/versions/pre.go b/internal/testfiles/testdata/versions/pre.go index 2b5ca6780e7..809f8b793f3 100644 --- a/internal/testfiles/testdata/versions/pre.go +++ b/internal/testfiles/testdata/versions/pre.go @@ -1,3 +1,3 @@ -//go:build go1.20 +//go:build go1.21 -package versions // want "pre.go@go1.20" +package versions // want "pre.go@go1.21" diff --git a/internal/testfiles/testdata/versions/sub.test/sub.go.test b/internal/testfiles/testdata/versions/sub.test/sub.go.test index 3b2721b9822..f573fdd782d 100644 --- a/internal/testfiles/testdata/versions/sub.test/sub.go.test +++ b/internal/testfiles/testdata/versions/sub.test/sub.go.test @@ -1 +1 @@ -package sub // want "sub.go@go1.21" +package sub // want "sub.go@go1.22" diff --git a/internal/testfiles/testfiles_test.go b/internal/testfiles/testfiles_test.go index 67951a0976c..789344601e4 100644 --- a/internal/testfiles/testfiles_test.go +++ b/internal/testfiles/testfiles_test.go @@ -19,7 +19,7 @@ import ( ) func TestTestDir(t *testing.T) { - testenv.NeedsGo1Point(t, 22) + testenv.NeedsGo1Point(t, 23) // Files are initially {go.mod.test,sub.test/sub.go.test}. fs := os.DirFS(filepath.Join(analysistest.TestData(), "versions")) diff --git a/internal/tool/tool.go b/internal/tool/tool.go index 0e59d377fa7..eadb0fb5ab9 100644 --- a/internal/tool/tool.go +++ b/internal/tool/tool.go @@ -29,7 +29,7 @@ import ( // (&Application{}).Main("myapp", "non-flag-command-line-arg-help", os.Args[1:]) // } // It recursively scans the application object for fields with a tag containing -// `flag:"flagnames" help:"short help text"`` +// `flag:"flagnames" help:"short help text"` // uses all those fields to build command line flags. It will split flagnames on // commas and add a flag per name. // It expects the Application type to have a method diff --git a/internal/typesinternal/errorcode.go b/internal/typesinternal/errorcode.go index 834e05381ce..131caab2847 100644 --- a/internal/typesinternal/errorcode.go +++ b/internal/typesinternal/errorcode.go @@ -838,7 +838,7 @@ const ( // InvalidCap occurs when an argument to the cap built-in function is not of // supported type. // - // See https://golang.org/ref/spec#Lengthand_capacity for information on + // See https://golang.org/ref/spec#Length_and_capacity for information on // which underlying types are supported as arguments to cap and len. // // Example: @@ -859,7 +859,7 @@ const ( // InvalidCopy occurs when the arguments are not of slice type or do not // have compatible type. // - // See https://golang.org/ref/spec#Appendingand_copying_slices for more + // See https://golang.org/ref/spec#Appending_and_copying_slices for more // information on the type requirements for the copy built-in. // // Example: @@ -897,7 +897,7 @@ const ( // InvalidLen occurs when an argument to the len built-in function is not of // supported type. // - // See https://golang.org/ref/spec#Lengthand_capacity for information on + // See https://golang.org/ref/spec#Length_and_capacity for information on // which underlying types are supported as arguments to cap and len. // // Example: @@ -914,7 +914,7 @@ const ( // InvalidMake occurs when make is called with an unsupported type argument. // - // See https://golang.org/ref/spec#Makingslices_maps_and_channels for + // See https://golang.org/ref/spec#Making_slices_maps_and_channels for // information on the types that may be created using make. // // Example: diff --git a/internal/versions/types_test.go b/internal/versions/types_test.go index 59f6d18b45f..df3705cc7a9 100644 --- a/internal/versions/types_test.go +++ b/internal/versions/types_test.go @@ -38,7 +38,7 @@ func Test(t *testing.T) { pversion string tests []fileTest }{ - {"", "", []fileTest{{"noversion.go", ""}, {"gobuild.go", ""}}}, + // {"", "", []fileTest{{"noversion.go", ""}, {"gobuild.go", ""}}}, // TODO(matloob): re-enable this test (with modifications) once CL 607955 has been submitted {"go1.22", "go1.22", []fileTest{{"noversion.go", "go1.22"}, {"gobuild.go", "go1.23"}}}, } { name := fmt.Sprintf("types.Config{GoVersion:%q}", item.goversion)