Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Orders source files before compilation to ensure reproducible output #742

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion build/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,10 @@ func parseAndAugment(pkg *build.Package, isTest bool, fileSet *token.FileSet) ([
if err != nil {
panic(err)
}
file, err := parser.ParseFile(fileSet, fullPath, r, parser.ParseComments)
// Files should be uniquely named and in the original package directory in order to be
// ordered correctly
newPath := path.Join(pkg.Dir, "__"+name)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm confused: doesn't fullPath already include name?

Copy link
Contributor Author

@dave dave Apr 6, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes it does, but fullPath is dependant on the goroot of the machine you're building this on... So some machines these files will come before the other files, and some after.

So on Andy's machine the file order would be:

/andys_go_root/src/time/format.go
/andys_go_root/src/time/sleep.go
/andys_go_root/src/time/sys_unix.go
/andys_go_root/src/time/tick.go
/andys_go_root/src/time/time.go
/andys_go_root/src/time/zoneinfo.go
/andys_go_root/src/time/zoneinfo_read.go
/src/time/time.go

... but on Zoe's machine it would be:

/src/time/time.go
/zoes_go_root/src/time/format.go
/zoes_go_root/src/time/sleep.go
/zoes_go_root/src/time/sys_unix.go
/zoes_go_root/src/time/tick.go
/zoes_go_root/src/time/time.go
/zoes_go_root/src/time/zoneinfo.go
/zoes_go_root/src/time/zoneinfo_read.go

See the issue for an explanation.

With this change the order will be identical on any machine because they'll be like this:

/src/time/__format.go
/src/time/__sleep.go
/src/time/__sys_unix.go
/src/time/__tick.go
/src/time/__time.go
/src/time/time.go
/src/time/__zoneinfo.go
/src/time/__zoneinfo_read.go

file, err := parser.ParseFile(fileSet, newPath, r, parser.ParseComments)
if err != nil {
panic(err)
}
Expand Down
153 changes: 153 additions & 0 deletions compiler/compiler_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
package compiler

import (
"bytes"
"fmt"
"go/ast"
"go/build"
"go/parser"
"go/token"
"go/types"
"testing"

"github.com/sergi/go-diff/diffmatchpatch"
"golang.org/x/tools/go/loader"
)

func TestOrder(t *testing.T) {
fileA := `
package foo

var Avar = "a"

type Atype struct{}

func Afunc() int {
var varA = 1
var varB = 2
return varA+varB
}
`

fileB := `
package foo

var Bvar = "b"

type Btype struct{}

func Bfunc() int {
var varA = 1
var varB = 2
return varA+varB
}
`
files := []source{{"fileA.go", []byte(fileA)}, {"fileB.go", []byte(fileB)}}

compare(t, "foo", files, false)
compare(t, "foo", files, true)

}

func compare(t *testing.T, path string, sourceFiles []source, minify bool) {
outputNormal, err := compile(path, sourceFiles, minify)
if err != nil {
t.Fatal(err)
}

// reverse the array
for i, j := 0, len(sourceFiles)-1; i < j; i, j = i+1, j-1 {
sourceFiles[i], sourceFiles[j] = sourceFiles[j], sourceFiles[i]
}

outputReversed, err := compile(path, sourceFiles, minify)
if err != nil {
t.Fatal(err)
}

if string(outputNormal) != string(outputReversed) {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(string(outputNormal), string(outputReversed), true)
fmt.Println(dmp.DiffPrettyText(diffs))
t.Fatal("files in different order produces differens JS")
}
}

type source struct {
name string
contents []byte
}

func compile(path string, sourceFiles []source, minify bool) ([]byte, error) {
conf := loader.Config{}
conf.Fset = token.NewFileSet()
conf.ParserMode = parser.ParseComments

context := build.Default // make a copy of build.Default
conf.Build = &context
conf.Build.BuildTags = []string{"js"}

var astFiles []*ast.File
for _, sourceFile := range sourceFiles {
astFile, err := parser.ParseFile(conf.Fset, sourceFile.name, sourceFile.contents, parser.ParseComments)
if err != nil {
return nil, err
}
astFiles = append(astFiles, astFile)
}
conf.CreateFromFiles(path, astFiles...)
prog, err := conf.Load()
if err != nil {
return nil, err
}

archiveCache := map[string]*Archive{}
var importContext *ImportContext
importContext = &ImportContext{
Packages: make(map[string]*types.Package),
Import: func(path string) (*Archive, error) {

// find in local cache
if a, ok := archiveCache[path]; ok {
return a, nil
}

pi := prog.Package(path)
importContext.Packages[path] = pi.Pkg

// compile package
a, err := Compile(path, pi.Files, prog.Fset, importContext, minify)
if err != nil {
return nil, err
}
archiveCache[path] = a
return a, nil
},
}

a, err := importContext.Import(path)
if err != nil {
return nil, err
}
b, err := renderPackage(a)
if err != nil {
return nil, err
}
return b, nil
}

func renderPackage(archive *Archive) ([]byte, error) {

selection := make(map[*Decl]struct{})
for _, d := range archive.Declarations {
selection[d] = struct{}{}
}

buf := &bytes.Buffer{}

if err := WritePkgCode(archive, selection, false, &SourceMapFilter{Writer: buf}); err != nil {
return nil, err
}

return buf.Bytes(), nil
}
6 changes: 6 additions & 0 deletions compiler/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,12 @@ func (pi packageImporter) Import(path string) (*types.Package, error) {
}

func Compile(importPath string, files []*ast.File, fileSet *token.FileSet, importContext *ImportContext, minify bool) (*Archive, error) {

// Files must be in the same order to get reproducible JS
sort.Slice(files, func(i, j int) bool {
return fileSet.File(files[i].Pos()).Name() > fileSet.File(files[j].Pos()).Name()
})

typesInfo := &types.Info{
Types: make(map[ast.Expr]types.TypeAndValue),
Defs: make(map[*ast.Ident]types.Object),
Expand Down