# Save cluster config to <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kwok.yaml
mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>
mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki
# Generate PKI to <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki
mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs
touch <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/audit.log
cp <ROOT_DIR>/test/kwokctl/audit-policy.yaml <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/audit.yaml
mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/etcd
nerdctl pull registry.k8s.io/etcd:3.5.15-0
nerdctl pull registry.k8s.io/kube-apiserver:v1.32.2
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/apiserver-tracing-config.yaml
apiVersion: apiserver.config.k8s.io/v1alpha1
kind: TracingConfiguration
endpoint: kwok-<CLUSTER_NAME>-jaeger:4317
samplingRatePerMillion: 1000000
EOF
nerdctl pull registry.k8s.io/kubectl:v1.32.2
nerdctl pull registry.k8s.io/kube-controller-manager:v1.32.2
cp <ROOT_DIR>/test/kwokctl/scheduler-config.yaml <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/scheduler.yaml
cat <<EOF >><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/scheduler.yaml
clientConnection:
  kubeconfig: "~/.kube/config"
EOF
nerdctl pull registry.k8s.io/kube-scheduler:v1.32.2
nerdctl pull registry.k8s.io/kwok/kwok:v0.7.0
nerdctl pull registry.k8s.io/metrics-server/metrics-server:v0.7.1
nerdctl pull docker.io/prom/prometheus:v2.53.0
nerdctl pull docker.io/jaegertracing/all-in-one:1.58.1
nerdctl pull docker.io/kubernetesui/dashboard:v2.7.0
nerdctl pull docker.io/kubernetesui/metrics-scraper:v1.0.9
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/prometheus.yaml
global:
  evaluation_interval: 15s
  scrape_interval: 15s
  scrape_timeout: 10s
scrape_configs:
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: etcd
  metrics_path: /metrics
  scheme: http
  static_configs:
  - targets:
    - kwok-<CLUSTER_NAME>-etcd:2379
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: kube-apiserver
  metrics_path: /metrics
  scheme: https
  static_configs:
  - targets:
    - kwok-<CLUSTER_NAME>-kube-apiserver:6443
  tls_config:
    cert_file: /etc/kubernetes/pki/admin.crt
    insecure_skip_verify: true
    key_file: /etc/kubernetes/pki/admin.key
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: kube-controller-manager
  metrics_path: /metrics
  scheme: https
  static_configs:
  - targets:
    - kwok-<CLUSTER_NAME>-kube-controller-manager:10257
  tls_config:
    cert_file: /etc/kubernetes/pki/admin.crt
    insecure_skip_verify: true
    key_file: /etc/kubernetes/pki/admin.key
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: kube-scheduler
  metrics_path: /metrics
  scheme: https
  static_configs:
  - targets:
    - kwok-<CLUSTER_NAME>-kube-scheduler:10259
  tls_config:
    cert_file: /etc/kubernetes/pki/admin.crt
    insecure_skip_verify: true
    key_file: /etc/kubernetes/pki/admin.key
- http_sd_configs:
  - url: http://kwok-<CLUSTER_NAME>-kwok-controller:10247/discovery/prometheus
  job_name: kwok-controller-metrics-discovery
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: kwok-controller
  metrics_path: /metrics
  scheme: http
  static_configs:
  - targets:
    - kwok-<CLUSTER_NAME>-kwok-controller:10247
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: metrics-server
  metrics_path: /metrics
  scheme: https
  static_configs:
  - targets:
    - kwok-<CLUSTER_NAME>-kwok-controller:4443
  tls_config:
    cert_file: /etc/kubernetes/pki/admin.crt
    insecure_skip_verify: true
    key_file: /etc/kubernetes/pki/admin.key
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: prometheus
  metrics_path: /metrics
  scheme: http
  static_configs:
  - targets:
    - 127.0.0.1:9090
EOF
chmod 0644 <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/prometheus.yaml
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml
apiVersion: v1
clusters:
- cluster:
    certificate-authority: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt
    server: https://127.0.0.1:32766
  name: kwok-<CLUSTER_NAME>
contexts:
- context:
    cluster: kwok-<CLUSTER_NAME>
    user: kwok-<CLUSTER_NAME>
  name: kwok-<CLUSTER_NAME>
current-context: kwok-<CLUSTER_NAME>
kind: Config
preferences: {}
users:
- name: kwok-<CLUSTER_NAME>
  user:
    client-certificate: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt
    client-key: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key
EOF
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig
apiVersion: v1
clusters:
- cluster:
    certificate-authority: /etc/kubernetes/pki/ca.crt
    server: https://kwok-<CLUSTER_NAME>-kube-apiserver:6443
  name: kwok-<CLUSTER_NAME>
contexts:
- context:
    cluster: kwok-<CLUSTER_NAME>
    user: kwok-<CLUSTER_NAME>
  name: kwok-<CLUSTER_NAME>
current-context: kwok-<CLUSTER_NAME>
kind: Config
preferences: {}
users:
- name: kwok-<CLUSTER_NAME>
  user:
    client-certificate: /etc/kubernetes/pki/admin.crt
    client-key: /etc/kubernetes/pki/admin.key
EOF
# Save cluster config to <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kwok.yaml
nerdctl network create kwok-<CLUSTER_NAME> --label=com.docker.compose.project=kwok-<CLUSTER_NAME>
nerdctl create --name=kwok-<CLUSTER_NAME>-etcd --pull=never --entrypoint=etcd --network=kwok-<CLUSTER_NAME> --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> registry.k8s.io/etcd:3.5.15-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --experimental-enable-distributed-tracing=true --experimental-distributed-tracing-address=kwok-<CLUSTER_NAME>-jaeger:4317 --experimental-distributed-tracing-sampling-rate=1000000
nerdctl create --name=kwok-<CLUSTER_NAME>-jaeger --pull=never --network=kwok-<CLUSTER_NAME> --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.58.1 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --collector.otlp.grpc.host-port=0.0.0.0:4317
nerdctl create --name=kwok-<CLUSTER_NAME>-kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok-<CLUSTER_NAME> --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> --publish=32766:6443/tcp --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.32.2 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok-<CLUSTER_NAME>-etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml
nerdctl create --name=kwok-<CLUSTER_NAME>-kube-apiserver-insecure-proxy --pull=never --entrypoint=kubectl --network=kwok-<CLUSTER_NAME> --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> --publish=6080:8001/tcp --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig:~/.kube/config:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kubectl:v1.32.2 proxy --accept-hosts=^*$ --address=0.0.0.0 --kubeconfig=~/.kube/config --port=8001
nerdctl create --name=kwok-<CLUSTER_NAME>-kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok-<CLUSTER_NAME> --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig:~/.kube/config:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.32.2 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000
nerdctl create --name=kwok-<CLUSTER_NAME>-kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok-<CLUSTER_NAME> --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig:~/.kube/config:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.32.2 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000
nerdctl create --name=kwok-<CLUSTER_NAME>-kwok-controller --pull=never --entrypoint=kwok --network=kwok-<CLUSTER_NAME> --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig:~/.kube/config:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kwok.yaml:~/.kwok/kwok.yaml:ro registry.k8s.io/kwok/kwok:v0.7.0 --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok-<CLUSTER_NAME>-kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=200 --tracing-endpoint=kwok-<CLUSTER_NAME>-jaeger:4317 --tracing-sampling-rate-per-million=1000000
nerdctl create --name=kwok-<CLUSTER_NAME>-dashboard --pull=never --network=kwok-<CLUSTER_NAME> --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> --publish=8000:8080/tcp --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig:~/.kube/config:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --sidecar-host=kwok-<CLUSTER_NAME>-dashboard-metrics-scraper:8000 --system-banner=Welcome to kwok-<CLUSTER_NAME> --kubeconfig=~/.kube/config --insecure-port=8080
nerdctl create --name=kwok-<CLUSTER_NAME>-metrics-server --pull=never --entrypoint=/metrics-server --network=kwok-<CLUSTER_NAME> --user=root --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig:~/.kube/config:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/metrics-server/metrics-server:v0.7.1 --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --kubelet-use-node-status-port --kubelet-insecure-tls --metric-resolution=15s --bind-address=0.0.0.0 --secure-port=4443 --kubeconfig=~/.kube/config --authentication-kubeconfig=~/.kube/config --authorization-kubeconfig=~/.kube/config --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key
nerdctl create --name=kwok-<CLUSTER_NAME>-prometheus --pull=never --entrypoint=prometheus --network=kwok-<CLUSTER_NAME> --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> --publish=9090:9090/tcp --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.53.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090
nerdctl create --name=kwok-<CLUSTER_NAME>-dashboard-metrics-scraper --pull=never --network=kwok-<CLUSTER_NAME> --user=root --restart=unless-stopped --label=com.docker.compose.project=kwok-<CLUSTER_NAME> --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig:~/.kube/config:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/metrics-scraper:v1.0.9 --db-file=/metrics.db --kubeconfig=~/.kube/config
# Add context kwok-<CLUSTER_NAME> to ~/.kube/config
nerdctl start kwok-<CLUSTER_NAME>-etcd
nerdctl start kwok-<CLUSTER_NAME>-jaeger
nerdctl start kwok-<CLUSTER_NAME>-kube-apiserver
nerdctl start kwok-<CLUSTER_NAME>-kube-apiserver-insecure-proxy
nerdctl start kwok-<CLUSTER_NAME>-kube-controller-manager
nerdctl start kwok-<CLUSTER_NAME>-kube-scheduler
nerdctl start kwok-<CLUSTER_NAME>-kwok-controller
nerdctl start kwok-<CLUSTER_NAME>-dashboard
nerdctl start kwok-<CLUSTER_NAME>-metrics-server
nerdctl start kwok-<CLUSTER_NAME>-prometheus
nerdctl start kwok-<CLUSTER_NAME>-dashboard-metrics-scraper
# Set up apiservice for metrics server
