Not able to use custom docker image, healthcheck failing #17821

santurini · 2025-05-12T11:08:22Z

santurini
May 12, 2025

Is there an existing issue for this?

I have searched the existing issues

Current Behavior

I am using a custom dockerfile to integrare docker in docker on kubernetes, the problem is that to do so I have to install the coder server and launch it as a post-hook, and this seems to break the healthcheck from the UI and therefore I am not able to connect to the pod terminal from the UI.

This is the Dockerfile:

FROM nestybox/ubuntu-focal-systemd-docker:latest

RUN /bin/bash -c 'apt-get update && \
    apt-get install --yes tmux wget && \
    rm -rf /var/lib/apt/lists/*'

WORKDIR /home/coder

This is the template:

terraform {
  required_providers {
    coder = {
      source = "coder/coder"
    }
    kubernetes = {
      source = "hashicorp/kubernetes"
    }
  }
}

provider "coder" {
}

data "coder_parameter" "cpu" {
  name         = "cpu"
  display_name = "CPU"
  description  = "The number of CPU cores"
  default      = "1"
  type         = "number"
  icon         = "/icon/memory.svg"
  mutable      = true
  validation {
    min = 1
    max = 8
  }
}

data "coder_parameter" "memory" {
  name         = "memory"
  display_name = "Memory"
  description  = "The amount of memory in GB"
  default      = "2"
  type         = "number"
  icon         = "/icon/memory.svg"
  mutable      = true
  validation {
    min = 1
    max = 256
  }
}

data "coder_parameter" "home_disk_size" {
  name         = "home_disk_size"
  display_name = "Home disk size"
  description  = "The size of the home disk in GB"
  default      = "10"
  type         = "number"
  icon         = "/emojis/1f4be.png"
  mutable      = false
  validation {
    min = 1
    max = 1000
  }
}

provider "kubernetes" {
  config_path = null
}

data "coder_workspace" "me" {}
data "coder_workspace_owner" "me" {}

resource "coder_agent" "main" {
  os             = "linux"
  arch           = "amd64"
  startup_script = <<-EOT
    set -e

    # Start the pre-installed code-server in the background
    curl -fsSL https://code-server.dev/install.sh | sh -s -- --method=standalone --prefix=/usr/local/code-server
    /usr/local/code-server/bin/code-server --auth none --port 13337 >code-server.log 2>&1 &

  EOT

  metadata {
    display_name = "CPU Usage"
    key          = "0_cpu_usage"
    script       = "coder stat cpu"
    interval     = 10
    timeout      = 1
  }

  metadata {
    display_name = "RAM Usage"
    key          = "1_ram_usage"
    script       = "coder stat mem"
    interval     = 10
    timeout      = 1
  }

  metadata {
    display_name = "Home Disk"
    key          = "3_home_disk"
    script       = "coder stat disk --path $${HOME}"
    interval     = 60
    timeout      = 1
  }

  metadata {
    display_name = "CPU Usage (Host)"
    key          = "4_cpu_usage_host"
    script       = "coder stat cpu --host"
    interval     = 10
    timeout      = 1
  }

  metadata {
    display_name = "Memory Usage (Host)"
    key          = "5_mem_usage_host"
    script       = "coder stat mem --host"
    interval     = 10
    timeout      = 1
  }

  metadata {
    display_name = "Load Average (Host)"
    key          = "6_load_host"
    # get load avg scaled by number of cores
    script   = <<EOT
      echo "`cat /proc/loadavg | awk '{ print $1 }'` `nproc`" | awk '{ printf "%0.2f", $1/$2 }'
    EOT
    interval = 60
    timeout  = 1
  }
}

# code-server
resource "coder_app" "code-server" {
  agent_id     = coder_agent.main.id
  slug         = "code-server"
  display_name = "code-server"
  icon         = "/icon/code.svg"
  url          = "http://localhost:13337?folder=/home/coder"
  subdomain    = false
  share        = "owner"

  healthcheck {
    url       = "http://localhost:13337/healthz"
    interval  = 3
    threshold = 10
  }
}

resource "kubernetes_persistent_volume_claim" "home" {
  metadata {
    name      = "coder-${data.coder_workspace_owner.me.name}-${data.coder_workspace.me.name}-home"
    namespace = "coder"
    labels = {
      "app.kubernetes.io/name"     = "coder-pvc"
      "app.kubernetes.io/instance" = "coder-pvc-${data.coder_workspace.me.id}"
      "app.kubernetes.io/part-of"  = "coder"
      "com.coder.resource"       = "true"
      "com.coder.workspace.id"   = data.coder_workspace.me.id
      "com.coder.workspace.name" = data.coder_workspace.me.name
      "com.coder.user.id"        = data.coder_workspace_owner.me.id
      "com.coder.user.username"  = data.coder_workspace_owner.me.name
    }
    annotations = {
      "com.coder.user.email" = data.coder_workspace_owner.me.email
    }
  }
  wait_until_bound = false
  spec {
    access_modes = ["ReadWriteOnce"]
    storage_class_name = "longhorn"
    resources {
      requests = {
        storage = "${data.coder_parameter.home_disk_size.value}Gi"
      }
    }
  }
}

resource "kubernetes_deployment" "main" {
  count = data.coder_workspace.me.start_count
  depends_on = [
    kubernetes_persistent_volume_claim.home
  ]
  wait_for_rollout = false
  metadata {
    name      = "${data.coder_workspace_owner.me.name}-${data.coder_workspace.me.name}"
    namespace = "coder"
    labels = {
      "app.kubernetes.io/name"     = "coder-workspace"
      "app.kubernetes.io/instance" = "coder-workspace-${data.coder_workspace.me.id}"
      "app.kubernetes.io/part-of"  = "coder"
      "com.coder.resource"         = "true"
      "com.coder.workspace.id"     = data.coder_workspace.me.id
      "com.coder.workspace.name"   = data.coder_workspace.me.name
      "com.coder.user.id"          = data.coder_workspace_owner.me.id
      "com.coder.user.username"    = data.coder_workspace_owner.me.name
    }
    annotations = {
      "com.coder.user.email" = data.coder_workspace_owner.me.email
      "io.kubernetes.cri-o.userns-mode" = "auto:size=65536"
    }
  }

  spec {
    replicas = 1
    selector {
      match_labels = {
        "app.kubernetes.io/name"     = "coder-workspace"
        "app.kubernetes.io/instance" = "coder-workspace-${data.coder_workspace.me.id}"
        "app.kubernetes.io/part-of"  = "coder"
        "com.coder.resource"         = "true"
        "com.coder.workspace.id"     = data.coder_workspace.me.id
        "com.coder.workspace.name"   = data.coder_workspace.me.name
        "com.coder.user.id"          = data.coder_workspace_owner.me.id
        "com.coder.user.username"    = data.coder_workspace_owner.me.name
      }
    }
    strategy {
      type = "Recreate"
    }

    template {
      metadata {
        labels = {
          "app.kubernetes.io/name"     = "coder-workspace"
          "app.kubernetes.io/instance" = "coder-workspace-${data.coder_workspace.me.id}"
          "app.kubernetes.io/part-of"  = "coder"
          "com.coder.resource"         = "true"
          "com.coder.workspace.id"     = data.coder_workspace.me.id
          "com.coder.workspace.name"   = data.coder_workspace.me.name
          "com.coder.user.id"          = data.coder_workspace_owner.me.id
          "com.coder.user.username"    = data.coder_workspace_owner.me.name
        }
        annotations = {
          "io.kubernetes.cri-o.userns-mode" = "auto:size=65536"
        }
      }
      spec {
        image_pull_secrets {
          name = "regcred"
        }
        
        runtime_class_name = "sysbox-runc"
        
        container {
          name              = "dev"
          image             = "translatednet/cpu-machine:sysbox"
          image_pull_policy = "Always"
          command           = ["/sbin/init"]

          env {
            name  = "CODER_AGENT_TOKEN"
            value = coder_agent.main.token
          }
          
          lifecycle {
            post_start {
              exec {
                command = ["/bin/bash", "-c", "${coder_agent.main.startup_script}"]
              }
            }
          }
          
          resources {
            requests = {
              "cpu"    = "250m"
              "memory" = "512Mi"
            }
            limits = {
              "cpu"    = "${data.coder_parameter.cpu.value}"
              "memory" = "${data.coder_parameter.memory.value}Gi"
            }
          }
          volume_mount {
            mount_path = "/home/coder"
            name       = "home"
            read_only  = false
          }
        }

        volume {
          name = "home"
          persistent_volume_claim {
            claim_name = kubernetes_persistent_volume_claim.home.metadata.0.name
            read_only  = false
          }
        }

        affinity {
          pod_anti_affinity {
            preferred_during_scheduling_ignored_during_execution {
              weight = 1
              pod_affinity_term {
                topology_key = "kubernetes.io/hostname"
                label_selector {
                  match_expressions {
                    key      = "app.kubernetes.io/name"
                    operator = "In"
                    values   = ["coder-workspace"]
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}

And this is the healthcheck from inside the container (sometimes expired sometimes alive):

root@santurini-test-5b794f88c9-9t6zk:/home/coder# curl http://127.0.0.1:13337/healthz
{"status":"expired","lastHeartbeat":1747047539127}

Relevant Log Output

Expected Behavior

I would like to be able to integrate docker in docker without compromising coder.

Steps to Reproduce

Install Coder
Install sysbox
Create docker image
Create workspace with custom template

Environment

Host OS: Ubuntu 20.04
Coder version: v2.21.3+bd1ef88

Additional Context

No response

santurini · 2025-05-13T09:39:47Z

santurini
May 13, 2025
Author

I tried using this startup-script that gets ran as a post start command:

    #!/usr/bin/env bash
    set -eu

    # Error handling - to preserve logs in case of failure
    waitonexit() {
      echo "=== Agent script exited with non-zero code ($?). Sleeping 1h to preserve logs..."
      sleep 3600
    }
    trap waitonexit EXIT

    # Create a directory with the right permissions
    CODE_SERVER_DIR="/home/coder/.local/code-server"
    mkdir -p "$CODE_SERVER_DIR"
    
    # Install the latest code-server
    echo "Installing code-server..."
    curl -fsSL https://code-server.dev/install.sh | sh -s -- --method=standalone --prefix="$CODE_SERVER_DIR"
    
    # Ensure correct permissions
    chmod -R 755 "$CODE_SERVER_DIR"
    
    # Start code-server in the background
    echo "Starting code-server..."
    "$CODE_SERVER_DIR/bin/code-server" --auth none --port 13337 >/tmp/code-server.log 2>&1 &
    
    # Verify code-server is running
    sleep 2
    if ! ps aux | grep -v grep | grep code-server > /dev/null; then
      echo "WARNING: code-server may have failed to start, check logs"
      cat /tmp/code-server.log
    else
      echo "code-server is running"
    fi

    # If the script completes successfully, remove the trap
    trap - EXIT
    echo "Startup script completed successfully"
  EOT

Same as before, the agent is started but from the UI im not able to connect to the workspace even if the pod is running.
How can I start the agent without changing the container command:

     spec {
       image_pull_secrets {
         name = "regcred"
       }

       runtime_class_name = "sysbox-runc"

       container {
         name = "dev"
         image             = "translatednet/cpu-machine:sysbox"
         image_pull_policy = "Always"
         command           = ["/sbin/init"]

         env {
           name  = "CODER_AGENT_TOKEN"
           value = coder_agent.main.token
         }
         
         lifecycle {
           post_start {
             exec {
               command = ["/bin/bash", "-c", coder_agent.main.startup_script]
             }
           }
         }

0 replies

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Not able to use custom docker image, healthcheck failing #17821

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{editor}}'s edit

{{editor}}'s edit

Uh oh!

Replies: 1 comment

Uh oh!

{{title}}

Uh oh!

Select a reply

Uh oh!

Not able to use custom docker image, healthcheck failing #17821

Uh oh!

Uh oh!

santurini May 12, 2025

Is there an existing issue for this?

Current Behavior

Relevant Log Output

Expected Behavior

Steps to Reproduce

Environment

Additional Context

Replies: 1 comment

Uh oh!

santurini May 13, 2025 Author

santurini
May 12, 2025

santurini
May 13, 2025
Author