Thanks to visit codestin.com
Credit goes to github.com

Skip to content

feat: allow for configurable ignore paths #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 18, 2023
Merged

feat: allow for configurable ignore paths #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b1cdfcc
fix: skip /var/run on unpack
kylecarbs Jul 18, 2023
8cd52ce
Fix startup build because of cache
kylecarbs Jul 18, 2023
a7ed19e
Allow for configurable ignore paths
kylecarbs Jul 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/envbuilder/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func main() {
SilenceUsage: true,
SilenceErrors: true,
RunE: func(cmd *cobra.Command, args []string) error {
options := envbuilder.OptionsFromEnv(os.Getenv)
options := envbuilder.OptionsFromEnv(os.LookupEnv)

var sendLogs func(ctx context.Context, log ...agentsdk.StartupLog) error
agentURL := os.Getenv("CODER_AGENT_URL")
Expand Down
46 changes: 40 additions & 6 deletions envbuilder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,17 @@ type Options struct {
// and pulling from container registries.
Insecure bool `env:"INSECURE"`

// IgnorePaths is a comma separated list of paths
// to ignore when building the workspace.
IgnorePaths []string `env:"IGNORE_PATHS"`

// SkipRebuild skips building if the MagicFile exists.
// This is used to skip building when a container is
// restarting. e.g. docker stop -> docker start
// This value can always be set to true - even if the
// container is being started for the first time.
SkipRebuild bool `env:"SKIP_REBUILD"`

// GitURL is the URL of the Git repository to clone.
// This is optional!
GitURL string `env:"GIT_URL"`
Expand Down Expand Up @@ -184,6 +195,12 @@ func Run(ctx context.Context, options Options) error {
if options.InitCommand == "" {
options.InitCommand = "/bin/sh"
}
if options.IgnorePaths == nil {
// Kubernetes frequently stores secrets in /var/run/secrets, and
// other applications might as well. This seems to be a sensible
// default, but if that changes, it's simple to adjust.
options.IgnorePaths = []string{"/var/run"}
}
// Default to the shell!
initArgs := []string{"-c", options.InitScript}
if options.InitArgs != "" {
Expand Down Expand Up @@ -464,17 +481,22 @@ func Run(ctx context.Context, options Options) error {
// IgnorePaths in the Kaniko options doesn't properly ignore paths.
// So we add them to the default ignore list. See:
// https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136
ignorePaths := []string{MagicDir, options.LayerCacheDir, options.WorkspaceFolder}
ignorePaths := append([]string{
MagicDir,
options.LayerCacheDir,
options.WorkspaceFolder,
}, options.IgnorePaths...)

for _, ignorePath := range ignorePaths {
util.AddToDefaultIgnoreList(util.IgnoreListEntry{
Path: ignorePath,
PrefixMatchOnly: true,
PrefixMatchOnly: false,
})
}

build := func() (v1.Image, error) {
_, err := options.Filesystem.Stat(MagicFile)
if err == nil {
if err == nil && options.SkipRebuild {
endStage := startStage("🏗️ Skipping build because of cache...")
imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent)
if err != nil {
Expand Down Expand Up @@ -557,6 +579,10 @@ func Run(ctx context.Context, options Options) error {
case strings.Contains(err.Error(), "authentication required"):
fallback = true
fallbackErr = err
// This occurs from Docker Hub when the image cannot be found!
case strings.Contains(err.Error(), "manifest unknown"):
fallback = true
fallbackErr = err
case strings.Contains(err.Error(), "unexpected status code 401 Unauthorized"):
logf(codersdk.LogLevelError, "Unable to pull the provided image. Ensure your registry credentials are correct!")
}
Expand Down Expand Up @@ -851,7 +877,7 @@ func findUser(nameOrID string) (*user.User, error) {
}

// OptionsFromEnv returns a set of options from environment variables.
func OptionsFromEnv(getEnv func(string) string) Options {
func OptionsFromEnv(getEnv func(string) (string, bool)) Options {
options := Options{}

val := reflect.ValueOf(&options).Elem()
Expand All @@ -866,10 +892,18 @@ func OptionsFromEnv(getEnv func(string) string) Options {
}
switch fieldTyp.Type.Kind() {
case reflect.String:
field.SetString(getEnv(env))
v, _ := getEnv(env)
field.SetString(v)
case reflect.Bool:
v, _ := strconv.ParseBool(getEnv(env))
e, _ := getEnv(env)
v, _ := strconv.ParseBool(e)
field.SetBool(v)
case reflect.Slice:
v, ok := getEnv(env)
if !ok {
continue
}
field.Set(reflect.ValueOf(strings.Split(v, ",")))
}
}

Expand Down
4 changes: 2 additions & 2 deletions envbuilder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ func TestSystemOptions(t *testing.T) {
"GIT_URL": "https://github.com/coder/coder",
"WORKSPACE_FOLDER": "/workspaces/coder",
}
env := envbuilder.OptionsFromEnv(func(s string) string {
return opts[s]
env := envbuilder.OptionsFromEnv(func(s string) (string, bool) {
return opts[s], true
})
require.Equal(t, "echo hello", env.InitScript)
require.Equal(t, "kylecarbs/testing", env.CacheRepo)
Expand Down
96 changes: 64 additions & 32 deletions integration/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ import (
"net/http/httptest"
"net/http/httputil"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -52,9 +54,9 @@ func TestFailsGitAuth(t *testing.T) {
username: "kyle",
password: "testing",
})
_, err := runEnvbuilder(t, []string{
_, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
})
}})
require.ErrorContains(t, err, "authentication required")
}

Expand All @@ -67,12 +69,12 @@ func TestSucceedsGitAuth(t *testing.T) {
username: "kyle",
password: "testing",
})
_, err := runEnvbuilder(t, []string{
_, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
"DOCKERFILE_PATH=Dockerfile",
"GIT_USERNAME=kyle",
"GIT_PASSWORD=testing",
})
}})
require.NoError(t, err)
}

Expand Down Expand Up @@ -111,9 +113,9 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) {
".devcontainer/Dockerfile": "FROM ubuntu",
},
})
ctr, err := runEnvbuilder(t, []string{
ctr, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
})
}})
require.NoError(t, err)

output := execContainer(t, ctr, "cat /test")
Expand All @@ -127,9 +129,32 @@ func TestBuildFromDockerfile(t *testing.T) {
"Dockerfile": "FROM alpine:latest",
},
})
ctr, err := runEnvbuilder(t, []string{
ctr, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
"DOCKERFILE_PATH=Dockerfile",
}})
require.NoError(t, err)

output := execContainer(t, ctr, "echo hello")
require.Equal(t, "hello", strings.TrimSpace(output))
}

func TestBuildIgnoreVarRunSecrets(t *testing.T) {
// Ensures that a Git repository with a Dockerfile is cloned and built.
url := createGitServer(t, gitServerOptions{
files: map[string]string{
"Dockerfile": "FROM alpine:latest",
},
})
dir := t.TempDir()
err := os.WriteFile(filepath.Join(dir, "secret"), []byte("test"), 0644)
require.NoError(t, err)
ctr, err := runEnvbuilder(t, options{
env: []string{
"GIT_URL=" + url,
"DOCKERFILE_PATH=Dockerfile",
},
binds: []string{fmt.Sprintf("%s:/var/run/secrets", dir)},
})
require.NoError(t, err)

Expand All @@ -144,11 +169,11 @@ func TestBuildWithSetupScript(t *testing.T) {
"Dockerfile": "FROM alpine:latest",
},
})
ctr, err := runEnvbuilder(t, []string{
ctr, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
"DOCKERFILE_PATH=Dockerfile",
"SETUP_SCRIPT=echo \"INIT_ARGS=-c 'echo hi > /wow && sleep infinity'\" >> $ENVBUILDER_ENV",
})
}})
require.NoError(t, err)

output := execContainer(t, ctr, "cat /wow")
Expand All @@ -161,14 +186,14 @@ func TestBuildCustomCertificates(t *testing.T) {
"Dockerfile": "FROM alpine:latest",
},
}))
ctr, err := runEnvbuilder(t, []string{
ctr, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + srv.URL,
"DOCKERFILE_PATH=Dockerfile",
"SSL_CERT_BASE64=" + base64.StdEncoding.EncodeToString(pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: srv.TLS.Certificates[0].Certificate[0],
})),
})
}})
require.NoError(t, err)

output := execContainer(t, ctr, "echo hello")
Expand All @@ -182,10 +207,11 @@ func TestBuildStopStartCached(t *testing.T) {
"Dockerfile": "FROM alpine:latest",
},
})
ctr, err := runEnvbuilder(t, []string{
ctr, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
"DOCKERFILE_PATH=Dockerfile",
})
"SKIP_REBUILD=true",
}})
require.NoError(t, err)

cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
Expand All @@ -212,9 +238,9 @@ func TestCloneFailsFallback(t *testing.T) {
t.Parallel()
t.Run("BadRepo", func(t *testing.T) {
t.Parallel()
_, err := runEnvbuilder(t, []string{
_, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=bad-value",
})
}})
require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error())
})
}
Expand All @@ -229,10 +255,10 @@ func TestBuildFailsFallback(t *testing.T) {
"Dockerfile": "bad syntax",
},
})
_, err := runEnvbuilder(t, []string{
_, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
"DOCKERFILE_PATH=Dockerfile",
})
}})
require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error())
require.ErrorContains(t, err, "dockerfile parse error")
})
Expand All @@ -245,10 +271,10 @@ func TestBuildFailsFallback(t *testing.T) {
RUN exit 1`,
},
})
_, err := runEnvbuilder(t, []string{
_, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
"DOCKERFILE_PATH=Dockerfile",
})
}})
require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error())
})
t.Run("BadDevcontainer", func(t *testing.T) {
Expand All @@ -259,9 +285,9 @@ RUN exit 1`,
".devcontainer/devcontainer.json": "not json",
},
})
_, err := runEnvbuilder(t, []string{
_, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
})
}})
require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error())
})
t.Run("NoImageOrDockerfile", func(t *testing.T) {
Expand All @@ -271,10 +297,10 @@ RUN exit 1`,
".devcontainer/devcontainer.json": "{}",
},
})
ctr, err := runEnvbuilder(t, []string{
ctr, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
"FALLBACK_IMAGE=alpine:latest",
})
}})
require.NoError(t, err)

output := execContainer(t, ctr, "echo hello")
Expand All @@ -297,10 +323,10 @@ func TestPrivateRegistry(t *testing.T) {
"Dockerfile": "FROM " + image,
},
})
_, err := runEnvbuilder(t, []string{
_, err := runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
"DOCKERFILE_PATH=Dockerfile",
})
}})
require.ErrorContains(t, err, "Unauthorized")
})
t.Run("Auth", func(t *testing.T) {
Expand All @@ -326,11 +352,11 @@ func TestPrivateRegistry(t *testing.T) {
})
require.NoError(t, err)

_, err = runEnvbuilder(t, []string{
_, err = runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
"DOCKERFILE_PATH=Dockerfile",
"DOCKER_CONFIG_BASE64=" + base64.StdEncoding.EncodeToString(config),
})
}})
require.NoError(t, err)
})
t.Run("InvalidAuth", func(t *testing.T) {
Expand All @@ -356,11 +382,11 @@ func TestPrivateRegistry(t *testing.T) {
})
require.NoError(t, err)

_, err = runEnvbuilder(t, []string{
_, err = runEnvbuilder(t, options{env: []string{
"GIT_URL=" + url,
"DOCKERFILE_PATH=Dockerfile",
"DOCKER_CONFIG_BASE64=" + base64.StdEncoding.EncodeToString(config),
})
}})
require.ErrorContains(t, err, "Unauthorized")
})
}
Expand Down Expand Up @@ -409,7 +435,7 @@ func setupPassthroughRegistry(t *testing.T, image string, auth *registryAuth) st
}

func TestNoMethodFails(t *testing.T) {
_, err := runEnvbuilder(t, []string{})
_, err := runEnvbuilder(t, options{env: []string{}})
require.ErrorContains(t, err, envbuilder.ErrNoFallbackImage.Error())
}

Expand Down Expand Up @@ -509,9 +535,14 @@ func cleanOldEnvbuilders() {
}
}

type options struct {
binds []string
env []string
}

// runEnvbuilder starts the envbuilder container with the given environment
// variables and returns the container ID.
func runEnvbuilder(t *testing.T, env []string) (string, error) {
func runEnvbuilder(t *testing.T, options options) (string, error) {
t.Helper()
ctx := context.Background()
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
Expand All @@ -521,12 +552,13 @@ func runEnvbuilder(t *testing.T, env []string) (string, error) {
})
ctr, err := cli.ContainerCreate(ctx, &container.Config{
Image: "envbuilder:latest",
Env: env,
Env: options.env,
Labels: map[string]string{
testContainerLabel: "true",
},
}, &container.HostConfig{
NetworkMode: container.NetworkMode("host"),
Binds: options.binds,
}, nil, nil, "")
require.NoError(t, err)
t.Cleanup(func() {
Expand Down