Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit f11fea6

Browse files
hugodutkahugodutka
committed
set up a terraform providers mirror for tests
1 parent f670bc3 commit f11fea6

File tree

4 files changed

+231
-34
lines changed

4 files changed

+231
-34
lines changed

‎provisioner/terraform/executor.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,9 @@ type executor struct {
3535
mut *sync.Mutex
3636
binaryPath string
3737
// cachePath and workdir must not be used by multiple processes at once.
38-
cachePath string
39-
workdir string
38+
cachePath string
39+
cliConfigPath string
40+
workdir string
4041
// used to capture execution times at various stages
4142
timings *timingAggregator
4243
}
@@ -50,6 +51,9 @@ func (e *executor) basicEnv() []string {
5051
if e.cachePath != "" && runtime.GOOS == "linux" {
5152
env = append(env, "TF_PLUGIN_CACHE_DIR="+e.cachePath)
5253
}
54+
if e.cliConfigPath != "" {
55+
env = append(env, "TF_CLI_CONFIG_FILE="+e.cliConfigPath)
56+
}
5357
return env
5458
}
5559

‎provisioner/terraform/provision_test.go

+175-12
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,17 @@
33
package terraform_test
44

55
import (
6+
"bytes"
67
"context"
8+
"crypto/sha256"
9+
"encoding/hex"
710
"encoding/json"
811
"errors"
912
"fmt"
1013
"net"
1114
"net/http"
1215
"os"
16+
"os/exec"
1317
"path/filepath"
1418
"sort"
1519
"strings"
@@ -29,10 +33,11 @@ import (
2933
)
3034

3135
type provisionerServeOptions struct {
32-
binaryPath string
33-
exitTimeout time.Duration
34-
workDir string
35-
logger *slog.Logger
36+
binaryPath string
37+
cliConfigPath string
38+
exitTimeout time.Duration
39+
workDir string
40+
logger *slog.Logger
3641
}
3742

3843
func setupProvisioner(t *testing.T, opts *provisionerServeOptions) (context.Context, proto.DRPCProvisionerClient) {
@@ -66,9 +71,10 @@ func setupProvisioner(t *testing.T, opts *provisionerServeOptions) (context.Cont
6671
Logger: *opts.logger,
6772
WorkDirectory: opts.workDir,
6873
},
69-
BinaryPath: opts.binaryPath,
70-
CachePath: cachePath,
71-
ExitTimeout: opts.exitTimeout,
74+
BinaryPath: opts.binaryPath,
75+
CachePath: cachePath,
76+
ExitTimeout: opts.exitTimeout,
77+
CliConfigPath: opts.cliConfigPath,
7278
})
7379
}()
7480
api := proto.NewDRPCProvisionerClient(client)
@@ -85,6 +91,149 @@ func configure(ctx context.Context, t *testing.T, client proto.DRPCProvisionerCl
8591
return sess
8692
}
8793

94+
func hashTemplateFilesAndTestName(t *testing.T, testName string, templateFiles map[string]string) string {
95+
t.Helper()
96+
97+
sortedFileNames := make([]string, 0, len(templateFiles))
98+
for fileName := range templateFiles {
99+
sortedFileNames = append(sortedFileNames, fileName)
100+
}
101+
sort.Strings(sortedFileNames)
102+
103+
hasher := sha256.New()
104+
for _, fileName := range sortedFileNames {
105+
file := templateFiles[fileName]
106+
_, err := hasher.Write([]byte(fileName))
107+
require.NoError(t, err)
108+
_, err = hasher.Write([]byte(file))
109+
require.NoError(t, err)
110+
}
111+
_, err := hasher.Write([]byte(testName))
112+
require.NoError(t, err)
113+
return hex.EncodeToString(hasher.Sum(nil))
114+
}
115+
116+
const (
117+
terraformConfigFileName = "terraform.rc"
118+
cacheProvidersDirName = "providers"
119+
cacheTemplateFilesDirName = "files"
120+
)
121+
122+
// Writes a Terraform CLI config file (`terraform.rc`) in `dir` to enforce using the local provider mirror.
123+
// This blocks network access for providers, forcing Terraform to use only what's cached in `dir`.
124+
// Returns the path to the generated config file.
125+
func writeCliConfig(t *testing.T, dir string) string {
126+
t.Helper()
127+
128+
cliConfigPath := filepath.Join(dir, terraformConfigFileName)
129+
require.NoError(t, os.MkdirAll(filepath.Dir(cliConfigPath), 0o700))
130+
131+
content := fmt.Sprintf(`
132+
provider_installation {
133+
filesystem_mirror {
134+
path = "%s"
135+
include = ["*/*"]
136+
}
137+
direct {
138+
exclude = ["*/*"]
139+
}
140+
}
141+
`, filepath.Join(dir, cacheProvidersDirName))
142+
require.NoError(t, os.WriteFile(cliConfigPath, []byte(content), 0o600))
143+
return cliConfigPath
144+
}
145+
146+
func runCmd(t *testing.T, dir string, args ...string) {
147+
t.Helper()
148+
149+
stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil)
150+
cmd := exec.Command(args[0], args[1:]...) //#nosec
151+
cmd.Dir = dir
152+
cmd.Stdout = stdout
153+
cmd.Stderr = stderr
154+
if err := cmd.Run(); err != nil {
155+
t.Fatalf("failed to run %s: %s\nstdout: %s\nstderr: %s", strings.Join(args, " "), err, stdout.String(), stderr.String())
156+
}
157+
}
158+
159+
// Ensures Terraform providers are downloaded and cached locally in a unique directory for this test.
160+
// Uses `terraform init` then `mirror` to populate the cache if needed.
161+
// Returns the cache directory path.
162+
func downloadProviders(t *testing.T, rootDir string, templateFiles map[string]string) string {
163+
t.Helper()
164+
165+
// Each test gets a unique cache dir based on its name and template files.
166+
// This ensures that tests can download providers in parallel and that they
167+
// will redownload providers if the template files change.
168+
hash := hashTemplateFilesAndTestName(t, t.Name(), templateFiles)
169+
dir := filepath.Join(rootDir, hash[:12])
170+
if _, err := os.Stat(dir); err == nil {
171+
t.Logf("%s: using cached terraform providers", t.Name())
172+
return dir
173+
}
174+
filesDir := filepath.Join(dir, cacheTemplateFilesDirName)
175+
defer func() {
176+
// The files dir will contain a copy of terraform providers generated
177+
// by the terraform init command. We don't want to persist them since
178+
// we already have a registry mirror in the providers dir.
179+
if err := os.RemoveAll(filesDir); err != nil {
180+
t.Logf("failed to remove files dir %s: %s", filesDir, err)
181+
}
182+
if !t.Failed() {
183+
return
184+
}
185+
if err := os.RemoveAll(dir); err != nil {
186+
t.Logf("failed to remove dir %s: %s", dir, err)
187+
}
188+
}()
189+
190+
require.NoError(t, os.MkdirAll(filesDir, 0o700))
191+
192+
for fileName, file := range templateFiles {
193+
filePath := filepath.Join(filesDir, fileName)
194+
if _, err := os.Stat(filePath); os.IsNotExist(err) {
195+
require.NoError(t, os.MkdirAll(filepath.Dir(filePath), 0o700))
196+
require.NoError(t, os.WriteFile(filePath, []byte(file), 0o600))
197+
}
198+
}
199+
200+
providersDir := filepath.Join(dir, cacheProvidersDirName)
201+
require.NoError(t, os.MkdirAll(providersDir, 0o700))
202+
203+
// We need to run init because if a test uses modules in its template,
204+
// the mirror command will fail without it.
205+
runCmd(t, filesDir, "terraform", "init")
206+
// Now, mirror the providers into `providersDir`. We use this explicit mirror
207+
// instead of relying only on the standard Terraform plugin cache.
208+
//
209+
// Why? Because this mirror, when used with the CLI config from `writeCliConfig`,
210+
// prevents Terraform from hitting the network registry during `plan`. This cuts
211+
// down on network calls, making CI tests less flaky.
212+
//
213+
// In contrast, the standard cache *still* contacts the registry for metadata
214+
// during `init`, even if the plugins are already cached locally - see link below.
215+
//
216+
// Ref: https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache
217+
// > When a plugin cache directory is enabled, the terraform init command will
218+
// > still use the configured or implied installation methods to obtain metadata
219+
// > about which plugins are available
220+
runCmd(t, filesDir, "terraform", "providers", "mirror", providersDir)
221+
222+
return dir
223+
}
224+
225+
// Caches providers locally and generates a Terraform CLI config to use *only* that cache.
226+
// This setup prevents network access for providers during `terraform init`, improving reliability
227+
// in subsequent test runs.
228+
// Returns the path to the generated CLI config file.
229+
func cacheProviders(t *testing.T, templateFiles map[string]string, rootDir string) string {
230+
t.Helper()
231+
232+
providersParentDir := downloadProviders(t, rootDir, templateFiles)
233+
cliConfigPath := writeCliConfig(t, providersParentDir)
234+
return cliConfigPath
235+
}
236+
88237
func readProvisionLog(t *testing.T, response proto.DRPCProvisioner_SessionClient) string {
89238
var logBuf strings.Builder
90239
for {
@@ -352,6 +501,8 @@ func TestProvision(t *testing.T) {
352501
Apply bool
353502
// Some tests may need to be skipped until the relevant provider version is released.
354503
SkipReason string
504+
// If SkipCacheProviders is true, then skip caching the terraform providers for this test.
505+
SkipCacheProviders bool
355506
}{
356507
{
357508
Name: "missing-variable",
@@ -422,16 +573,18 @@ func TestProvision(t *testing.T) {
422573
Files: map[string]string{
423574
"main.tf": `a`,
424575
},
425-
ErrorContains: "initialize terraform",
426-
ExpectLogContains: "Argument or block definition required",
576+
ErrorContains: "initialize terraform",
577+
ExpectLogContains: "Argument or block definition required",
578+
SkipCacheProviders: true,
427579
},
428580
{
429581
Name: "bad-syntax-2",
430582
Files: map[string]string{
431583
"main.tf": `;asdf;`,
432584
},
433-
ErrorContains: "initialize terraform",
434-
ExpectLogContains: `The ";" character is not valid.`,
585+
ErrorContains: "initialize terraform",
586+
ExpectLogContains: `The ";" character is not valid.`,
587+
SkipCacheProviders: true,
435588
},
436589
{
437590
Name: "destroy-no-state",
@@ -847,7 +1000,17 @@ func TestProvision(t *testing.T) {
8471000
t.Skip(testCase.SkipReason)
8481001
}
8491002

850-
ctx, api := setupProvisioner(t, nil)
1003+
cliConfigPath := ""
1004+
if !testCase.SkipCacheProviders {
1005+
cliConfigPath = cacheProviders(
1006+
t,
1007+
testCase.Files,
1008+
filepath.Join(testutil.PersistentCacheDir(t), "terraform_provision_test"),
1009+
)
1010+
}
1011+
ctx, api := setupProvisioner(t, &provisionerServeOptions{
1012+
cliConfigPath: cliConfigPath,
1013+
})
8511014
sess := configure(ctx, t, api, &proto.Config{
8521015
TemplateSourceArchive: testutil.CreateTar(t, testCase.Files),
8531016
})

‎provisioner/terraform/serve.go

+25-20
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ type ServeOptions struct {
2828
BinaryPath string
2929
// CachePath must not be used by multiple processes at once.
3030
CachePath string
31-
Tracer trace.Tracer
31+
// CliConfigPath is the path to the Terraform CLI config file.
32+
CliConfigPath string
33+
Tracer trace.Tracer
3234

3335
// ExitTimeout defines how long we will wait for a running Terraform
3436
// command to exit (cleanly) if the provision was stopped. This
@@ -132,22 +134,24 @@ func Serve(ctx context.Context, options *ServeOptions) error {
132134
options.ExitTimeout = unhanger.HungJobExitTimeout
133135
}
134136
return provisionersdk.Serve(ctx, &server{
135-
execMut: &sync.Mutex{},
136-
binaryPath: options.BinaryPath,
137-
cachePath: options.CachePath,
138-
logger: options.Logger,
139-
tracer: options.Tracer,
140-
exitTimeout: options.ExitTimeout,
137+
execMut: &sync.Mutex{},
138+
binaryPath: options.BinaryPath,
139+
cachePath: options.CachePath,
140+
cliConfigPath: options.CliConfigPath,
141+
logger: options.Logger,
142+
tracer: options.Tracer,
143+
exitTimeout: options.ExitTimeout,
141144
}, options.ServeOptions)
142145
}
143146

144147
type server struct {
145-
execMut *sync.Mutex
146-
binaryPath string
147-
cachePath string
148-
logger slog.Logger
149-
tracer trace.Tracer
150-
exitTimeout time.Duration
148+
execMut *sync.Mutex
149+
binaryPath string
150+
cachePath string
151+
cliConfigPath string
152+
logger slog.Logger
153+
tracer trace.Tracer
154+
exitTimeout time.Duration
151155
}
152156

153157
func (s *server) startTrace(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
@@ -158,12 +162,13 @@ func (s *server) startTrace(ctx context.Context, name string, opts ...trace.Span
158162

159163
func (s *server) executor(workdir string, stage database.ProvisionerJobTimingStage) *executor {
160164
return &executor{
161-
server: s,
162-
mut: s.execMut,
163-
binaryPath: s.binaryPath,
164-
cachePath: s.cachePath,
165-
workdir: workdir,
166-
logger: s.logger.Named("executor"),
167-
timings: newTimingAggregator(stage),
165+
server: s,
166+
mut: s.execMut,
167+
binaryPath: s.binaryPath,
168+
cachePath: s.cachePath,
169+
cliConfigPath: s.cliConfigPath,
170+
workdir: workdir,
171+
logger: s.logger.Named("executor"),
172+
timings: newTimingAggregator(stage),
168173
}
169174
}

‎testutil/cache.go

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package testutil
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"testing"
7+
8+
"github.com/stretchr/testify/require"
9+
)
10+
11+
// PersistentCacheDir returns a path to a directory
12+
// that will be cached between test runs in Github Actions.
13+
func PersistentCacheDir(t *testing.T) string {
14+
t.Helper()
15+
16+
// We don't use os.UserCacheDir() because the path it
17+
// returns is different on different operating systems.
18+
// This would make it harder to specify which cache dir to use
19+
// in Github Actions.
20+
home, err := os.UserHomeDir()
21+
require.NoError(t, err)
22+
dir := filepath.Join(home, ".cache", "coderv2-test")
23+
24+
return dir
25+
}

0 commit comments

Comments
 (0)