From 1b8198da151397e62611e086bbcb9cdceca5987e Mon Sep 17 00:00:00 2001 From: visualfc Date: Wed, 21 Sep 2022 20:39:17 +0800 Subject: [PATCH] build: support go:embed --- .std_test_pkg_exclusions | 1 - build/build.go | 59 ++++++--- build/embed.go | 175 ++++++++++++++++++++++++++ compiler/natives/src/embed/embed.go | 20 +++ go.mod | 1 + go.sum | 2 + tests/testdata/legacy_syscall/main.go | 3 +- tool.go | 7 +- 8 files changed, 246 insertions(+), 22 deletions(-) create mode 100644 build/embed.go create mode 100644 compiler/natives/src/embed/embed.go diff --git a/.std_test_pkg_exclusions b/.std_test_pkg_exclusions index b2ff33bb6..ae9691df7 100644 --- a/.std_test_pkg_exclusions +++ b/.std_test_pkg_exclusions @@ -1,4 +1,3 @@ -embed/internal/embedtest encoding/xml go/build go/internal/srcimporter diff --git a/build/build.go b/build/build.go index 24da95499..2c835ccf0 100644 --- a/build/build.go +++ b/build/build.go @@ -13,6 +13,7 @@ import ( "go/scanner" "go/token" "go/types" + "io/fs" "os" "os/exec" "path" @@ -398,11 +399,12 @@ func (p *PackageData) InternalBuildContext() *build.Context { func (p *PackageData) TestPackage() *PackageData { return &PackageData{ Package: &build.Package{ - Name: p.Name, - ImportPath: p.ImportPath, - Dir: p.Dir, - GoFiles: append(p.GoFiles, p.TestGoFiles...), - Imports: append(p.Imports, p.TestImports...), + Name: p.Name, + ImportPath: p.ImportPath, + Dir: p.Dir, + GoFiles: append(p.GoFiles, p.TestGoFiles...), + Imports: append(p.Imports, p.TestImports...), + EmbedPatternPos: joinEmbedPatternPos(p.EmbedPatternPos, p.TestEmbedPatternPos), }, IsTest: true, JSFiles: p.JSFiles, @@ -414,11 +416,12 @@ func (p *PackageData) TestPackage() *PackageData { func (p *PackageData) XTestPackage() *PackageData { return &PackageData{ Package: &build.Package{ - Name: p.Name + "_test", - ImportPath: p.ImportPath + "_test", - Dir: p.Dir, - GoFiles: p.XTestGoFiles, - Imports: p.XTestImports, + Name: p.Name + "_test", + ImportPath: p.ImportPath + "_test", + Dir: p.Dir, + GoFiles: p.XTestGoFiles, + Imports: p.XTestImports, + EmbedPatternPos: p.XTestEmbedPatternPos, }, IsTest: true, bctx: p.bctx, @@ -548,12 +551,30 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, cwd string) erro return fmt.Errorf("named files must all be in one directory; have: %v", strings.Join(dirList, ", ")) } + root := dirList[0] + ctx := build.Default + ctx.UseAllFiles = true + ctx.ReadDir = func(dir string) ([]fs.FileInfo, error) { + n := len(filenames) + infos := make([]fs.FileInfo, n) + for i := 0; i < n; i++ { + info, err := os.Stat(filenames[i]) + if err != nil { + return nil, err + } + infos[i] = info + } + return infos, nil + } + p, err := ctx.Import(".", root, 0) + if err != nil { + return err + } + p.Name = "main" + p.ImportPath = "main" + pkg := &PackageData{ - Package: &build.Package{ - Name: "main", - ImportPath: "main", - Dir: dirList[0], - }, + Package: p, // This ephemeral package doesn't have a unique import path to be used as a // build cache key, so we never cache it. SrcModTime: time.Now().Add(time.Hour), @@ -562,7 +583,6 @@ func (s *Session) BuildFiles(filenames []string, pkgObj string, cwd string) erro for _, file := range filenames { if !strings.HasSuffix(file, ".inc.js") { - pkg.GoFiles = append(pkg.GoFiles, filepath.Base(file)) continue } @@ -676,6 +696,13 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { if err != nil { return nil, err } + embed, err := embedFiles(pkg, fileSet, files) + if err != nil { + return nil, err + } + if embed != nil { + files = append(files, embed) + } importContext := &compiler.ImportContext{ Packages: s.Types, diff --git a/build/embed.go b/build/embed.go new file mode 100644 index 000000000..2b5661f6c --- /dev/null +++ b/build/embed.go @@ -0,0 +1,175 @@ +package build + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/token" + "strconv" + + "github.com/visualfc/goembed" +) + +func buildIdent(name string) string { + return fmt.Sprintf("__gopherjs_embed_%x__", name) +} + +var embed_head = `package %v + +import ( + "embed" + _ "unsafe" +) + +//go:linkname __gopherjs_embed_buildFS__ embed.buildFS +func __gopherjs_embed_buildFS__(list []struct { + name string + data string + hash [16]byte +}) (f embed.FS) +` + +// embedFiles generates an additional source file, which initializes all variables in the package with a go:embed directive. +func embedFiles(pkg *PackageData, fset *token.FileSet, files []*ast.File) (*ast.File, error) { + if len(pkg.EmbedPatternPos) == 0 { + return nil, nil + } + + ems, err := goembed.CheckEmbed(pkg.EmbedPatternPos, fset, files) + if err != nil { + return nil, err + } + + r := goembed.NewResolve() + for _, em := range ems { + fs, err := r.Load(pkg.Dir, fset, em) + if err != nil { + return nil, err + } + switch em.Kind { + case goembed.EmbedMaybeAlias: + // value = Type(data) + // valid alias string or []byte type used by types.check + em.Spec.Values = []ast.Expr{ + &ast.CallExpr{ + Fun: em.Spec.Type, + Args: []ast.Expr{ + &ast.Ident{Name: buildIdent(fs[0].Name), + NamePos: em.Spec.Names[0].NamePos}, + }, + }} + case goembed.EmbedBytes: + // value = []byte(data) + em.Spec.Values = []ast.Expr{ + &ast.CallExpr{ + Fun: em.Spec.Type, + Args: []ast.Expr{ast.NewIdent(buildIdent(fs[0].Name))}, + }} + case goembed.EmbedString: + // value = data + em.Spec.Values = []ast.Expr{ast.NewIdent(buildIdent(fs[0].Name))} + case goembed.EmbedFiles: + // value = __gopherjs_embed_buildFS__([]struct{name string; data string; hash [16]byte}{...}) + fs = goembed.BuildFS(fs) + elts := make([]ast.Expr, len(fs)) + for i, f := range fs { + if len(f.Data) == 0 { + elts[i] = &ast.CompositeLit{ + Elts: []ast.Expr{ + &ast.BasicLit{Kind: token.STRING, Value: strconv.Quote(f.Name)}, + &ast.BasicLit{Kind: token.STRING, Value: `""`}, + &ast.CompositeLit{ + Type: &ast.ArrayType{ + Len: &ast.BasicLit{Kind: token.INT, Value: "16"}, + Elt: ast.NewIdent("byte"), + }, + }, + }, + } + } else { + var hash [16]ast.Expr + for j, v := range f.Hash { + hash[j] = &ast.BasicLit{Kind: token.INT, Value: strconv.Itoa(int(v))} + } + elts[i] = &ast.CompositeLit{ + Elts: []ast.Expr{ + &ast.BasicLit{Kind: token.STRING, Value: strconv.Quote(f.Name)}, + ast.NewIdent(buildIdent(f.Name)), + &ast.CompositeLit{ + Type: &ast.ArrayType{ + Len: &ast.BasicLit{Kind: token.INT, Value: "16"}, + Elt: ast.NewIdent("byte"), + }, + Elts: hash[:], + }, + }, + } + } + } + call := &ast.CallExpr{ + Fun: ast.NewIdent("__gopherjs_embed_buildFS__"), + Args: []ast.Expr{ + &ast.CompositeLit{ + Type: &ast.ArrayType{ + Elt: &ast.StructType{ + Fields: &ast.FieldList{ + List: []*ast.Field{ + &ast.Field{ + Names: []*ast.Ident{ast.NewIdent("name")}, + Type: ast.NewIdent("string"), + }, + &ast.Field{ + Names: []*ast.Ident{ast.NewIdent("data")}, + Type: ast.NewIdent("string"), + }, + &ast.Field{ + Names: []*ast.Ident{ast.NewIdent("hash")}, + Type: &ast.ArrayType{ + Len: &ast.BasicLit{Kind: token.INT, Value: "16"}, + Elt: ast.NewIdent("byte"), + }, + }, + }, + }, + }, + }, + Elts: elts, + }, + }, + } + em.Spec.Values = []ast.Expr{call} + } + } + + var buf bytes.Buffer + fmt.Fprintf(&buf, embed_head, pkg.Name) + buf.WriteString("\nconst (\n") + for _, f := range r.Files() { + if len(f.Data) == 0 { + fmt.Fprintf(&buf, "\t%v = \"\"\n", buildIdent(f.Name)) + } else { + fmt.Fprintf(&buf, "\t%v = \"%v\"\n", buildIdent(f.Name), goembed.BytesToHex(f.Data)) + } + } + buf.WriteString(")\n\n") + f, err := parser.ParseFile(fset, "js_embed.go", buf.String(), parser.ParseComments) + if err != nil { + return nil, err + } + return f, nil +} + +func joinEmbedPatternPos(m1, m2 map[string][]token.Position) map[string][]token.Position { + if len(m1) == 0 && len(m2) == 0 { + return nil + } + m := make(map[string][]token.Position) + for k, v := range m1 { + m[k] = v + } + for k, v := range m2 { + m[k] = append(m[k], v...) + } + return m +} diff --git a/compiler/natives/src/embed/embed.go b/compiler/natives/src/embed/embed.go new file mode 100644 index 000000000..18cb7001f --- /dev/null +++ b/compiler/natives/src/embed/embed.go @@ -0,0 +1,20 @@ +//go:build js +// +build js + +package embed + +func buildFS(list []struct { + name string + data string + hash [16]byte +}) (f FS) { + n := len(list) + files := make([]file, n) + for i := 0; i < n; i++ { + files[i].name = list[i].name + files[i].data = list[i].data + files[i].hash = list[i].hash + } + f.files = &files + return +} diff --git a/go.mod b/go.mod index 32e06d4d9..12653d82d 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 + github.com/visualfc/goembed v0.3.3 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20220412211240-33da011f77ad diff --git a/go.sum b/go.sum index c4be9d5a9..01aee12b8 100644 --- a/go.sum +++ b/go.sum @@ -235,6 +235,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/visualfc/goembed v0.3.3 h1:pOL02L715tHKsLQVMcZz06tTzRDAHkJKJLRnCA22G9Q= +github.com/visualfc/goembed v0.3.3/go.mod h1:jCVCz/yTJGyslo6Hta+pYxWWBuq9ADCcIVZBTQ0/iVI= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/tests/testdata/legacy_syscall/main.go b/tests/testdata/legacy_syscall/main.go index 94c6c6413..75ba22f6b 100644 --- a/tests/testdata/legacy_syscall/main.go +++ b/tests/testdata/legacy_syscall/main.go @@ -1,4 +1,5 @@ -//go:build legacy_syscall,gopherjs +//go:build legacy_syscall && gopherjs +// +build legacy_syscall,gopherjs // This program tests GopherJS's ability to perform raw syscalls using the // deprecated node_syscall extension. See TestLegacySyscall. diff --git a/tool.go b/tool.go index bc2e7174f..9eb05b557 100644 --- a/tool.go +++ b/tool.go @@ -135,16 +135,15 @@ func main() { if err != nil { return fmt.Errorf("failed to expand patterns %v: %w", args, err) } - for _, pkgPath := range pkgs { if s.Watcher != nil { - pkg, err := xctx.Import(pkgPath, "", build.FindOnly) + pkg, err := xctx.Import(pkgPath, currentDirectory, build.FindOnly) if err != nil { return err } s.Watcher.Add(pkg.Dir) } - pkg, err := xctx.Import(pkgPath, ".", 0) + pkg, err := xctx.Import(pkgPath, currentDirectory, 0) if err != nil { return err } @@ -208,7 +207,7 @@ func main() { } } for _, pkgPath := range pkgs { - pkg, err := xctx.Import(pkgPath, ".", 0) + pkg, err := xctx.Import(pkgPath, currentDirectory, 0) if s.Watcher != nil && pkg != nil { // add watch even on error s.Watcher.Add(pkg.Dir) }