Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion cmd/crio/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,13 +114,24 @@ default_mounts = [
# pids_limit is the number of processes allowed in a container
pids_limit = {{ .PidsLimit }}

# enable using a shared PID namespace for containers in a pod
# enable using a shared PID namespace for containers in a pod.
# Deprecated: use pid_namespace = "pod" instead.
enable_shared_pid_namespace = {{ .EnableSharedPIDNamespace }}

# log_size_max is the max limit for the container log size in bytes.
# Negative values indicate that no limit is imposed.
log_size_max = {{ .LogSizeMax }}

# Select the PID namespace scope. Choose from 'container' for all
# containers (including pod infra containers) to have sibling PID
# namespaces (the default), 'pod' for all containers to share a
# single, per-pod namespace, or 'pod-container' to have the pod's
# infra container in one PID namespace with the non-infra containers
# in per-container PID namespaces that are children of the pod's infra
# PID namespace . A 'hostPID' Kubernetes pod specification overrides
# this setting.
pid_namespace = "{{ .PIDNamespace }}"

# The "crio.image" table contains settings pertaining to the
# management of OCI images.
[crio.image]
Expand Down
16 changes: 12 additions & 4 deletions cmd/crio/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,16 @@ func mergeConfig(config *server.Config, ctx *cli.Context) error {
if ctx.GlobalIsSet("default-mounts") {
config.DefaultMounts = ctx.GlobalStringSlice("default-mounts")
}
if ctx.GlobalIsSet("pid-namespace") {
config.PIDNamespace = lib.PIDNamespaceType(ctx.GlobalString("pid-namespace"))
} else if ctx.GlobalIsSet("enable-shared-pid-namespace") {
if ctx.GlobalBool("enable-shared-pid-namespace") {
config.PIDNamespace = lib.PIDNamespacePod
}
}
if ctx.GlobalIsSet("pids-limit") {
config.PidsLimit = ctx.GlobalInt64("pids-limit")
}
if ctx.GlobalIsSet("enable-shared-pid-namespace") {
config.EnableSharedPIDNamespace = ctx.GlobalBool("enable-shared-pid-namespace")
}
if ctx.GlobalIsSet("log-size-max") {
config.LogSizeMax = ctx.GlobalInt64("log-size-max")
}
Expand Down Expand Up @@ -295,14 +299,18 @@ func main() {
Name: "cgroup-manager",
Usage: "cgroup manager (cgroupfs or systemd)",
},
cli.StringFlag{
Name: "pid-namespace",
Usage: "select the PID namespace scope (\"container\" default, \"pod\", or \"pod-container\")",
},
cli.Int64Flag{
Name: "pids-limit",
Value: lib.DefaultPidsLimit,
Usage: "maximum number of processes allowed in a container",
},
cli.BoolFlag{
Name: "enable-shared-pid-namespace",
Usage: "enable using a shared PID namespace for containers in a pod",
Usage: "enable using a shared PID namespace for containers in a pod. Deprecated: use --pid-namespace=pod instead.",
},
cli.Int64Flag{
Name: "log-size-max",
Expand Down
20 changes: 20 additions & 0 deletions conmon/conmon.c
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <sys/uio.h>
#include <sys/ioctl.h>
#include <termios.h>
#include <sched.h>
#include <syslog.h>
#include <unistd.h>
#include <inttypes.h>
Expand Down Expand Up @@ -104,6 +105,7 @@ static char *opt_cuuid = NULL;
static char *opt_runtime_path = NULL;
static char *opt_bundle_path = NULL;
static char *opt_pid_file = NULL;
static char *opt_pid_namespace = NULL;
static bool opt_systemd_cgroup = false;
static bool opt_no_pivot = false;
static char *opt_exec_process_spec = NULL;
Expand All @@ -123,6 +125,7 @@ static GOptionEntry opt_entries[] =
{ "no-pivot", 0, 0, G_OPTION_ARG_NONE, &opt_no_pivot, "do not use pivot_root", NULL },
{ "bundle", 'b', 0, G_OPTION_ARG_STRING, &opt_bundle_path, "Bundle path", NULL },
{ "pidfile", 'p', 0, G_OPTION_ARG_STRING, &opt_pid_file, "PID file", NULL },
{ "pid-namespace", 0, 0, G_OPTION_ARG_STRING, &opt_pid_namespace, "PID namespace", NULL },
{ "systemd-cgroup", 's', 0, G_OPTION_ARG_NONE, &opt_systemd_cgroup, "Enable systemd cgroup manager", NULL },
{ "exec", 'e', 0, G_OPTION_ARG_NONE, &opt_exec, "Exec a command in a running container", NULL },
{ "exec-process-spec", 0, 0, G_OPTION_ARG_STRING, &opt_exec_process_spec, "Path to the process spec for exec", NULL },
Expand Down Expand Up @@ -1095,6 +1098,7 @@ int main(int argc, char *argv[])
int num_read;
int sync_pipe_fd = -1;
int start_pipe_fd = -1;
int pid_namespace_fd = -1;
GError *error = NULL;
GOptionContext *context;
GPtrArray *runtime_argv = NULL;
Expand Down Expand Up @@ -1202,6 +1206,22 @@ int main(int argc, char *argv[])
pexit("Failed to set as subreaper");
}

if (opt_pid_namespace) {
pid_namespace_fd = open(opt_pid_namespace, O_RDONLY);
if (pid_namespace_fd == -1) {
pexit("Failed to open PID namespace at %s", opt_pid_namespace);
}
ret = setns(pid_namespace_fd, CLONE_NEWPID);
if (ret != 0) {
close(pid_namespace_fd);
pexit("Failed to join the PID namespace at %s: %s", opt_pid_namespace, strerror(errno));
}
ret = close(pid_namespace_fd);
if (ret != 0) {
pexit("Failed to close the PID namespace at %s", opt_pid_namespace);
}
}

if (opt_terminal) {
csname = setup_console_socket();
} else {
Expand Down
5 changes: 4 additions & 1 deletion docs/crio.8.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ crio
[--log-level value]
[--pause-command=[value]]
[--pause-image=[value]]
[--pid-namespace=[value]]
[--registry=[value]]
[--root=[value]]
[--runroot=[value]]
Expand Down Expand Up @@ -92,9 +93,11 @@ crio [GLOBAL OPTIONS] config [OPTIONS]

**--pause-image**="": Image which contains the pause executable (default: "kubernetes/pause")

**--pid-namespace**="": Select the PID namespace scope. Choose from `container` for all containers (including pod infra containers) to have sibling PID namespaces (the default), `pod` for all containers to share a single, per-pod namespace, or `pod-container` to have the pod's infra container in one PID namespace with the non-infra containers in per-container PID namespaces that are children of the pod's infra PID namespace. A `hostPID` Kubernetes pod specification overrides this setting.

**--pids-limit**="": Maximum number of processes allowed in a container (default: 1024)

**--enable-shared-pid-namespace**="": Enable using a shared PID namespace for containers in a pod (default: false)
**--enable-shared-pid-namespace**="": Enable using a shared PID namespace for containers in a pod. Deprecated: use `--pid-namespace=pod` instead.

**--root**="": The crio root dir (default: "/var/lib/containers/storage")

Expand Down
5 changes: 4 additions & 1 deletion docs/crio.conf.5.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,14 @@ Example:
If it is positive, it must be >= 8192 (to match/exceed conmon read buffer).
The file is truncated and re-opened so the limit is never exceeded.

**pid_namespace**=""
Select the PID namespace scope. Choose from `container` for all containers (including pod infra containers) to have sibling PID namespaces (the default), `pod` for all containers to share a single, per-pod namespace, or `pod-container` to have the pod's infra container in one PID namespace with the non-infra containers in per-container PID namespaces that are children of the pod's infra PID namespace. A `hostPID` Kubernetes pod specification overrides this setting.

**pids_limit**=""
Maximum number of processes allowed in a container (default: 1024)

**enable_shared_pid_namespace**=""
Enable using a shared PID namespace for containers in a pod (default: false)
Enable using a shared PID namespace for containers in a pod. Deprecated: use `pid_namespace = "pod"` instead.

**runtime**=""
OCI runtime path (default: "/usr/bin/runc")
Expand Down
27 changes: 26 additions & 1 deletion lib/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,24 @@ const (
ImageVolumesBind ImageVolumesType = "bind"
)

// PIDNamespaceType describes pod PID namespace strategies.
type PIDNamespaceType string

const (
// PIDNamespaceContainer is for all containers (including pod infra
// containers) to have sibling PID namespaces.
PIDNamespaceContainer PIDNamespaceType = "container"

// PIDNamespacePod is for all containers to share a single, per-pod
// namespace.
PIDNamespacePod PIDNamespaceType = "pod"

// PIDNamespacePodContainer is for the pod's infra container in one
// PID namespace with the non-infra container in per-container PID
// namespaces that are children of the pod's infra PID namespace.
PIDNamespacePodContainer PIDNamespaceType = "pod-container"
)

const (
// DefaultPidsLimit is the default value for maximum number of processes
// allowed inside a container
Expand Down Expand Up @@ -121,7 +139,10 @@ type RuntimeConfig struct {
// NoPivot instructs the runtime to not use `pivot_root`, but instead use `MS_MOVE`
NoPivot bool `toml:"no_pivot"`

// EnableSharePidNamespace instructs the runtime to enable share pid namespace
// EnableSharePidNamespace instructs the runtime to enable share pid
// namespace.
//
// Deprecated: use PIDNamespace = PIDNamespacePod instead.
EnableSharedPIDNamespace bool `toml:"enable_shared_pid_namespace"`

// Conmon is the path to conmon binary, used for managing the runtime.
Expand Down Expand Up @@ -155,6 +176,10 @@ type RuntimeConfig struct {
// Hooks List of hooks to run with container
Hooks map[string]HookParams

// PIDNamespace selects the PID namespace scope. A 'hostPID'
// Kubernetes pod specification overrides this setting.
PIDNamespace PIDNamespaceType `toml:"pid_namespace"`

// PidsLimit is the number of processes each container is restricted to
// by the cgroup process number controller.
PidsLimit int64 `toml:"pids_limit"`
Expand Down
4 changes: 2 additions & 2 deletions lib/container_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ func (c *ContainerServer) LoadSandbox(id string) error {
return err
}

scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), "", labels, m.Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
if err != nil {
return err
}
Expand Down Expand Up @@ -513,7 +513,7 @@ func (c *ContainerServer) LoadContainer(id string) error {
return err
}

ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, img, imgName, imgRef, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), "", labels, m.Annotations, kubeAnnotations, img, imgName, imgRef, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
if err != nil {
return err
}
Expand Down
4 changes: 3 additions & 1 deletion oci/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type Container struct {
image string
sandbox string
netns ns.NetNS
pidNamespace string
terminal bool
stdin bool
stdinOnce bool
Expand Down Expand Up @@ -71,7 +72,7 @@ type ContainerState struct {
}

// NewContainer creates a container object.
func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, crioAnnotations map[string]string, annotations map[string]string, image string, imageName string, imageRef string, metadata *pb.ContainerMetadata, sandbox string, terminal bool, stdin bool, stdinOnce bool, privileged bool, trusted bool, dir string, created time.Time, stopSignal string) (*Container, error) {
func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, pidNamespace string, labels map[string]string, crioAnnotations map[string]string, annotations map[string]string, image string, imageName string, imageRef string, metadata *pb.ContainerMetadata, sandbox string, terminal bool, stdin bool, stdinOnce bool, privileged bool, trusted bool, dir string, created time.Time, stopSignal string) (*Container, error) {
state := &ContainerState{}
state.Created = created
c := &Container{
Expand All @@ -82,6 +83,7 @@ func NewContainer(id string, name string, bundlePath string, logPath string, net
labels: labels,
sandbox: sandbox,
netns: netns,
pidNamespace: pidNamespace,
terminal: terminal,
stdin: stdin,
stdinOnce: stdinOnce,
Expand Down
3 changes: 3 additions & 0 deletions oci/oci.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,9 @@ func (r *Runtime) CreateContainer(c *Container, cgroupParent string) (err error)
if r.noPivot {
args = append(args, "--no-pivot")
}
if c.pidNamespace != "" {
args = append(args, "--pid-namespace", c.pidNamespace)
}
if c.terminal {
args = append(args, "-t")
} else if c.stdin {
Expand Down
23 changes: 18 additions & 5 deletions server/container_create.go
Original file line number Diff line number Diff line change
Expand Up @@ -981,13 +981,26 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
return nil, err
}

var pidNamespace string
if containerConfig.GetLinux().GetSecurityContext().GetNamespaceOptions().GetHostPid() {
// kubernetes PodSpec specify to use Host PID namespace
specgen.RemoveLinuxNamespace(string(rspec.PIDNamespace))
} else if s.config.EnableSharedPIDNamespace {
// share Pod PID namespace
pidNsPath := fmt.Sprintf("/proc/%d/ns/pid", podInfraState.Pid)
if err := specgen.AddOrReplaceLinuxNamespace(string(rspec.PIDNamespace), pidNsPath); err != nil {
} else {
if s.config.EnableSharedPIDNamespace && len(s.config.PIDNamespace) == 0 {
s.config.PIDNamespace = lib.PIDNamespacePod
}
podPIDNsPath := fmt.Sprintf("/proc/%d/ns/pid", podInfraState.Pid)
var pidNsPath string
switch s.config.PIDNamespace {
case lib.PIDNamespaceContainer:
case lib.PIDNamespacePod:
pidNsPath = podPIDNsPath
case lib.PIDNamespacePodContainer:
pidNamespace = podPIDNsPath
default:
return nil, fmt.Errorf("unrecognized PID namespace configuration: %s", s.config.PIDNamespace)
}
if err = specgen.AddOrReplaceLinuxNamespace(string(rspec.PIDNamespace), pidNsPath); err != nil {
return nil, err
}
}
Expand Down Expand Up @@ -1268,7 +1281,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,

crioAnnotations := specgen.Spec().Annotations

container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, crioAnnotations, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), pidNamespace, labels, crioAnnotations, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
if err != nil {
return nil, err
}
Expand Down
6 changes: 3 additions & 3 deletions server/inspect_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func TestGetContainerInfo(t *testing.T) {
"io.kubernetes.test1": "value1",
}
getContainerFunc := func(id string) *oci.Container {
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "image", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, "", labels, annotations, annotations, "image", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -184,7 +184,7 @@ func TestGetContainerInfoCtrStateNil(t *testing.T) {
labels := map[string]string{}
annotations := map[string]string{}
getContainerFunc := func(id string) *oci.Container {
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, "", labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -215,7 +215,7 @@ func TestGetContainerInfoSandboxNotFound(t *testing.T) {
labels := map[string]string{}
annotations := map[string]string{}
getContainerFunc := func(id string) *oci.Container {
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
container, err := oci.NewContainer("testid", "testname", "", "/container/logs", mockNetNS{}, "", labels, annotations, annotations, "imageName", "imageName", "imageRef", &runtime.ContainerMetadata{}, "testsandboxid", false, false, false, false, false, "/root/for/container", created, "SIGKILL")
if err != nil {
t.Fatal(err)
}
Expand Down
2 changes: 1 addition & 1 deletion server/sandbox_run.go
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
g.AddAnnotation(annotations.HostnamePath, hostnamePath)
sb.AddHostnamePath(hostnamePath)

container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, g.Spec().Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), "", labels, g.Spec().Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
if err != nil {
return nil, err
}
Expand Down
4 changes: 1 addition & 3 deletions test/helpers.bash
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,6 @@ IMAGE_VOLUMES=${IMAGE_VOLUMES:-mkdir}
PIDS_LIMIT=${PIDS_LIMIT:-1024}
# Log size max limit
LOG_SIZE_MAX_LIMIT=${LOG_SIZE_MAX_LIMIT:--1}
# enable share container pid namespace
ENABLE_SHARED_PID_NAMESPACE=${ENABLE_SHARED_PID_NAMESPACE:-false}

TESTDIR=$(mktemp -d)

Expand Down Expand Up @@ -217,7 +215,7 @@ function start_crio() {
"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/mrunalp/image-volume-test:latest --import-from=dir:"$ARTIFACTS_PATH"/image-volume-test-image --signature-policy="$INTEGRATION_ROOT"/policy.json
"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/library/busybox:latest --import-from=dir:"$ARTIFACTS_PATH"/busybox-image --signature-policy="$INTEGRATION_ROOT"/policy.json
"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/runcom/stderr-test:latest --import-from=dir:"$ARTIFACTS_PATH"/stderr-test --signature-policy="$INTEGRATION_ROOT"/policy.json
"$CRIO_BINARY" ${DEFAULT_MOUNTS_OPTS} ${HOOKS_OPTS} --conmon "$CONMON_BINARY" --listen "$CRIO_SOCKET" --cgroup-manager "$CGROUP_MANAGER" --registry "docker.io" --runtime "$RUNTIME_BINARY" --root "$TESTDIR/crio" --runroot "$TESTDIR/crio-run" $STORAGE_OPTIONS --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$CRIO_CNI_CONFIG" --cni-plugin-dir "$CRIO_CNI_PLUGIN" --signature-policy "$INTEGRATION_ROOT"/policy.json --image-volumes "$IMAGE_VOLUMES" --pids-limit "$PIDS_LIMIT" --enable-shared-pid-namespace=${ENABLE_SHARED_PID_NAMESPACE} --log-size-max "$LOG_SIZE_MAX_LIMIT" --config /dev/null config >$CRIO_CONFIG
"$CRIO_BINARY" ${DEFAULT_MOUNTS_OPTS} ${HOOKS_OPTS} --conmon "$CONMON_BINARY" --listen "$CRIO_SOCKET" --cgroup-manager "$CGROUP_MANAGER" --registry "docker.io" --runtime "$RUNTIME_BINARY" --root "$TESTDIR/crio" --runroot "$TESTDIR/crio-run" $STORAGE_OPTIONS --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$CRIO_CNI_CONFIG" --cni-plugin-dir "$CRIO_CNI_PLUGIN" --signature-policy "$INTEGRATION_ROOT"/policy.json --image-volumes "$IMAGE_VOLUMES" --pids-limit "$PIDS_LIMIT" --log-size-max "$LOG_SIZE_MAX_LIMIT" $ADDITIONAL_CRIO_OPTIONS --config /dev/null config >$CRIO_CONFIG

# Prepare the CNI configuration files, we're running with non host networking by default
if [[ -n "$4" ]]; then
Expand Down
Loading