Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 144b1ed

Browse files
aslilacaslilac
committed
fix: reduce size of terraform modules archive
1 parent 5c77cdf commit 144b1ed

File tree

6 files changed

+340
-286
lines changed

6 files changed

+340
-286
lines changed

‎provisioner/terraform/modules.go

+67-36
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,8 @@ type modulesFile struct {
2424
Modules []*module `json:"Modules"`
2525
}
2626

27-
func getModulesDirectory(workdir string) string {
28-
return filepath.Join(workdir, ".terraform", "modules")
29-
}
30-
3127
func getModulesFilePath(workdir string) string {
32-
return filepath.Join(getModulesDirectory(workdir), "modules.json")
28+
return filepath.Join(workdir, ".terraform", "modules", "modules.json")
3329
}
3430

3531
func parseModulesFile(filePath string) ([]*proto.Module, error) {
@@ -72,52 +68,83 @@ func getModules(workdir string) ([]*proto.Module, error) {
7268
}
7369

7470
func getModulesArchive(workdir string) ([]byte, error) {
75-
modulesDir := getModulesDirectory(workdir)
76-
if _, err := os.ReadDir(modulesDir); err != nil {
71+
modulesFile, err := os.ReadFile(getModulesFilePath(workdir))
72+
if err != nil {
7773
if os.IsNotExist(err) {
7874
return []byte{}, nil
7975
}
80-
81-
return nil, err
76+
return nil, xerrors.Errorf("failed to read modules.json: %w", err)
8277
}
78+
var m struct{ Modules []*proto.Module }
79+
err = json.Unmarshal(modulesFile, &m)
80+
if err != nil {
81+
return nil, xerrors.Errorf("failed to parse modules.json: %w", err)
82+
}
83+
8384
empty := true
8485
var b bytes.Buffer
8586
w := tar.NewWriter(&b)
86-
err := filepath.WalkDir(modulesDir, func(filePath string, info fs.DirEntry, err error) error {
87-
if err != nil {
88-
return xerrors.Errorf("failed to create modules archive: %w", err)
89-
}
90-
if info.IsDir() {
91-
return nil
92-
}
93-
archivePath, found := strings.CutPrefix(filePath, workdir+string(os.PathSeparator))
94-
if !found {
95-
return xerrors.Errorf("walked invalid file path: %q", filePath)
96-
}
9787

98-
content, err := os.ReadFile(filePath)
99-
if err != nil {
100-
return xerrors.Errorf("failed to read module file while archiving: %w", err)
88+
for _, module := range m.Modules {
89+
// Check to make sure that the module is a remote module fetched by
90+
// Terraform. Any module that doesn't start with this path is already local,
91+
// and should be part of the template files already.
92+
if !strings.HasPrefix(module.Dir, ".terraform/modules/") {
93+
continue
10194
}
102-
empty = false
103-
err = w.WriteHeader(&tar.Header{
104-
Name: archivePath,
105-
Size: int64(len(content)),
106-
Mode: 0o644,
107-
Uid: 1000,
108-
Gid: 1000,
95+
96+
err := filepath.WalkDir(filepath.Join(workdir, module.Dir), func(filePath string, info fs.DirEntry, err error) error {
97+
if err != nil {
98+
return xerrors.Errorf("failed to create modules archive: %w", err)
99+
}
100+
if info.IsDir() {
101+
return nil
102+
}
103+
archivePath, found := strings.CutPrefix(filePath, workdir+string(os.PathSeparator))
104+
if !found {
105+
return xerrors.Errorf("walked invalid file path: %q", filePath)
106+
}
107+
108+
content, err := os.ReadFile(filePath)
109+
if err != nil {
110+
return xerrors.Errorf("failed to read module file while archiving: %w", err)
111+
}
112+
empty = false
113+
err = w.WriteHeader(&tar.Header{
114+
Name: archivePath,
115+
Size: int64(len(content)),
116+
Mode: 0o644,
117+
Uid: 1000,
118+
Gid: 1000,
119+
})
120+
if err != nil {
121+
return xerrors.Errorf("failed to add module file to archive: %w", err)
122+
}
123+
if _, err = w.Write(content); err != nil {
124+
return xerrors.Errorf("failed to write module file to archive: %w", err)
125+
}
126+
return nil
109127
})
110128
if err != nil {
111-
return xerrors.Errorf("failed to add module file to archive: %w", err)
112-
}
113-
if _, err = w.Write(content); err != nil {
114-
return xerrors.Errorf("failed to write module file to archive: %w", err)
129+
return nil, err
115130
}
116-
return nil
131+
}
132+
133+
err = w.WriteHeader(&tar.Header{
134+
Name: ".terraform/modules/modules.json",
135+
Size: int64(len(modulesFile)),
136+
Mode: 0o644,
137+
Uid: 1000,
138+
Gid: 1000,
117139
})
118140
if err != nil {
119-
return nil, err
141+
return nil, xerrors.Errorf("failed to write modules.json to archive: %w", err)
142+
}
143+
_, err = w.Write(modulesFile)
144+
if err != nil {
145+
return nil, xerrors.Errorf("failed to write modules.json to archive: %w", err)
120146
}
147+
121148
err = w.Close()
122149
if err != nil {
123150
return nil, xerrors.Errorf("failed to close module files archive: %w", err)
@@ -126,5 +153,9 @@ func getModulesArchive(workdir string) ([]byte, error) {
126153
if empty {
127154
return []byte{}, nil
128155
}
156+
157+
if b.Len() > (10 << 20) {
158+
return nil, xerrors.New("modules archive exceeds 10MB, modules will not be persisted")
159+
}
129160
return b.Bytes(), nil
130161
}

‎provisioner/terraform/modules_internal_test.go

+11-3
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,14 @@ func TestGetModulesArchive(t *testing.T) {
2525
require.NoError(t, err)
2626

2727
// Check that all of the files it should contain are correct
28-
r := bytes.NewBuffer(archive)
29-
tarfs := archivefs.FromTarReader(r)
30-
content, err := fs.ReadFile(tarfs, ".terraform/modules/example_module/main.tf")
28+
b := bytes.NewBuffer(archive)
29+
tarfs := archivefs.FromTarReader(b)
30+
31+
content, err := fs.ReadFile(tarfs, ".terraform/modules/modules.json")
32+
require.NoError(t, err)
33+
require.True(t, strings.HasPrefix(string(content), `{"Modules":[{"Key":"","Source":"","Dir":"."},`))
34+
35+
content, err = fs.ReadFile(tarfs, ".terraform/modules/example_module/main.tf")
3136
require.NoError(t, err)
3237
require.True(t, strings.HasPrefix(string(content), "terraform {"))
3338
if runtime.GOOS != "windows" {
@@ -36,6 +41,9 @@ func TestGetModulesArchive(t *testing.T) {
3641
require.Len(t, content, 3812)
3742
}
3843

44+
_, err = fs.ReadFile(tarfs, ".terraform/modules/stuff_that_should_not_be_included/nothing.txt")
45+
require.Error(t, err)
46+
3947
// It should always be byte-identical to optimize storage
4048
hashBytes := sha256.Sum256(archive)
4149
hash := hex.EncodeToString(hashBytes[:])

‎provisioner/terraform/testdata/modules-source-caching/.terraform/modules/stuff_that_should_not_be_included/nothing.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ここには何もありません

0 commit comments

Comments
 (0)