# Save cluster config to <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kwok.yaml
mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>
mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki
# Generate PKI to <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki
mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/etcd
docker pull docker.io/kindest/node:v1.32.2
mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki
mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/manifests
mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs
touch <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/audit.log
cp <ROOT_DIR>/test/kwokctl/audit-policy.yaml <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/audit.yaml
cp <ROOT_DIR>/test/kwokctl/scheduler-config.yaml <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/scheduler.yaml
cat <<EOF >><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/scheduler.yaml
clientConnection:
  kubeconfig: "/etc/kubernetes/scheduler.conf"
EOF
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/apiserver-tracing-config.yaml
apiVersion: apiserver.config.k8s.io/v1alpha1
kind: TracingConfiguration
endpoint: 0.0.0.0:4317
samplingRatePerMillion: 1000000
EOF
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kind.yaml
apiVersion: kind.x-k8s.io/v1alpha4
kind: Cluster
kubeadmConfigPatches:
- |
  apiServer:
    extraArgs:
      audit-log-path: /var/log/kubernetes/audit.log
      audit-policy-file: /etc/kubernetes/audit/audit.yaml
      enable-priority-and-fairness: "false"
      max-mutating-requests-inflight: "0"
      max-requests-inflight: "0"
      tracing-config-file: /etc/kubernetes/apiserver-tracing-config.yaml
    extraVolumes:
    - hostPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml
      mountPath: /etc/kubernetes/audit/audit.yaml
      name: audit-policy-file
      pathType: File
      readOnly: true
    - hostPath: /var/components/apiserver/var/log/kubernetes/audit.log
      mountPath: /var/log/kubernetes/audit.log
      name: audit-log-path
      pathType: File
    - hostPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml
      mountPath: /etc/kubernetes/apiserver-tracing-config.yaml
      name: apiserver-tracing-config
      pathType: File
      readOnly: true
  apiVersion: kubeadm.k8s.io/v1beta3
  controllerManager:
    extraArgs:
      kube-api-burst: "10000"
      kube-api-qps: "5000"
  dns: {}
  etcd:
    local:
      dataDir: /var/lib/etcd
      extraArgs:
        experimental-distributed-tracing-address: 127.0.0.1:4317
        experimental-distributed-tracing-sampling-rate: "1000000"
        experimental-enable-distributed-tracing: "true"
        quota-backend-bytes: "8589934592"
  kind: ClusterConfiguration
  networking: {}
  scheduler:
    extraArgs:
      config: /etc/kubernetes/scheduler/scheduler.yaml
      kube-api-burst: "10000"
      kube-api-qps: "5000"
    extraVolumes:
    - hostPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml
      mountPath: /etc/kubernetes/scheduler/scheduler.yaml
      name: config
      pathType: File
      readOnly: true
networking:
  apiServerPort: 32766
nodes:
- extraMounts:
  - containerPath: /etc/kwok/
    hostPath: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>
  - containerPath: /etc/kubernetes/manifests
    hostPath: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/manifests
  - containerPath: /etc/kubernetes/pki
    hostPath: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki
  - containerPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml
    hostPath: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/audit.yaml
    readOnly: true
  - containerPath: /var/components/apiserver/var/log/kubernetes/audit.log
    hostPath: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/audit.log
  - containerPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml
    hostPath: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/apiserver-tracing-config.yaml
    readOnly: true
  - containerPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml
    hostPath: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/scheduler.yaml
    readOnly: true
  - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml
    hostPath: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/prometheus.yaml
  extraPortMappings:
  - containerPort: 8001
    hostPort: 6080
    protocol: TCP
  - containerPort: 8080
    hostPort: 8000
    protocol: TCP
  - containerPort: 9090
    hostPort: 9090
    protocol: TCP
  - containerPort: 16686
    hostPort: 16686
    protocol: TCP
  role: control-plane
EOF
docker pull registry.k8s.io/kubectl:v1.32.2
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/manifests/kube-apiserver-insecure-proxy.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  name: kube-apiserver-insecure-proxy
  namespace: kube-system
spec:
  containers:
  - args:
    - proxy
    - --accept-hosts=^*$
    - --address=0.0.0.0
    - --kubeconfig=~/.kube/config
    - --port=8001
    command:
    - kubectl
    image: registry.k8s.io/kubectl:v1.32.2
    imagePullPolicy: Never
    name: kube-apiserver-insecure-proxy
    ports:
    - containerPort: 8001
      hostPort: 8001
      name: http
      protocol: TCP
    resources: {}
    volumeMounts:
    - mountPath: ~/.kube/config
      name: volume-0
      readOnly: true
    - mountPath: /etc/kubernetes/pki/ca.crt
      name: volume-1
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.crt
      name: volume-2
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.key
      name: volume-3
      readOnly: true
  hostNetwork: true
  restartPolicy: Always
  securityContext:
    runAsGroup: 0
    runAsUser: 0
  volumes:
  - hostPath:
      path: /etc/kubernetes/admin.conf
    name: volume-0
  - hostPath:
      path: /etc/kubernetes/pki/ca.crt
    name: volume-1
  - hostPath:
      path: /etc/kubernetes/pki/admin.crt
    name: volume-2
  - hostPath:
      path: /etc/kubernetes/pki/admin.key
    name: volume-3
status: {}
EOF
docker pull registry.k8s.io/kwok/kwok:v0.7.0
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/manifests/kwok-controller.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  name: kwok-controller
  namespace: kube-system
spec:
  containers:
  - args:
    - --manage-all-nodes=false
    - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake
    - --kubeconfig=~/.kube/config
    - --config=~/.kwok/kwok.yaml
    - --tls-cert-file=/etc/kubernetes/pki/admin.crt
    - --tls-private-key-file=/etc/kubernetes/pki/admin.key
    - --node-ip=$(POD_IP)
    - --node-name=kwok-controller.kube-system.svc
    - --node-port=10247
    - --server-address=0.0.0.0:10247
    - --node-lease-duration-seconds=40
    - --tracing-endpoint=127.0.0.1:4317
    - --tracing-sampling-rate-per-million=1000000
    command:
    - kwok
    env:
    - name: POD_IP
      valueFrom:
        fieldRef:
          fieldPath: status.podIP
    image: registry.k8s.io/kwok/kwok:v0.7.0
    imagePullPolicy: Never
    name: kwok-controller
    ports:
    - containerPort: 10247
      hostPort: 10247
      name: http
      protocol: TCP
    resources: {}
    volumeMounts:
    - mountPath: ~/.kube/config
      name: volume-0
      readOnly: true
    - mountPath: /etc/kubernetes/pki/ca.crt
      name: volume-1
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.crt
      name: volume-2
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.key
      name: volume-3
      readOnly: true
    - mountPath: ~/.kwok/kwok.yaml
      name: volume-4
      readOnly: true
  hostNetwork: true
  restartPolicy: Always
  securityContext:
    runAsGroup: 0
    runAsUser: 0
  volumes:
  - hostPath:
      path: /etc/kubernetes/admin.conf
    name: volume-0
  - hostPath:
      path: /etc/kubernetes/pki/ca.crt
    name: volume-1
  - hostPath:
      path: /etc/kubernetes/pki/admin.crt
    name: volume-2
  - hostPath:
      path: /etc/kubernetes/pki/admin.key
    name: volume-3
  - hostPath:
      path: /etc/kwok/kwok.yaml
    name: volume-4
status: {}
EOF
docker pull docker.io/kubernetesui/dashboard:v2.7.0
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/manifests/dashboard.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  name: dashboard
  namespace: kube-system
spec:
  containers:
  - args:
    - --insecure-bind-address=0.0.0.0
    - --bind-address=127.0.0.1
    - --port=0
    - --enable-insecure-login
    - --enable-skip-login
    - --disable-settings-authorizer
    - --sidecar-host=127.0.0.1:8000
    - --system-banner=Welcome to kwok-<CLUSTER_NAME>
    - --kubeconfig=~/.kube/config
    - --insecure-port=8080
    image: docker.io/kubernetesui/dashboard:v2.7.0
    imagePullPolicy: Never
    name: dashboard
    ports:
    - containerPort: 8080
      hostPort: 8080
      name: http
      protocol: TCP
    resources: {}
    volumeMounts:
    - mountPath: ~/.kube/config
      name: volume-0
      readOnly: true
    - mountPath: /etc/kubernetes/pki/ca.crt
      name: volume-1
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.crt
      name: volume-2
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.key
      name: volume-3
      readOnly: true
  hostNetwork: true
  restartPolicy: Always
  securityContext:
    runAsGroup: 0
    runAsUser: 0
  volumes:
  - hostPath:
      path: /etc/kubernetes/admin.conf
    name: volume-0
  - hostPath:
      path: /etc/kubernetes/pki/ca.crt
    name: volume-1
  - hostPath:
      path: /etc/kubernetes/pki/admin.crt
    name: volume-2
  - hostPath:
      path: /etc/kubernetes/pki/admin.key
    name: volume-3
status: {}
EOF
docker pull docker.io/kubernetesui/metrics-scraper:v1.0.9
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/manifests/dashboard-metrics-scraper.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  name: dashboard-metrics-scraper
  namespace: kube-system
spec:
  containers:
  - args:
    - --db-file=/metrics.db
    - --kubeconfig=~/.kube/config
    image: docker.io/kubernetesui/metrics-scraper:v1.0.9
    imagePullPolicy: Never
    name: dashboard-metrics-scraper
    resources: {}
    volumeMounts:
    - mountPath: ~/.kube/config
      name: volume-0
      readOnly: true
    - mountPath: /etc/kubernetes/pki/ca.crt
      name: volume-1
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.crt
      name: volume-2
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.key
      name: volume-3
      readOnly: true
  hostNetwork: true
  restartPolicy: Always
  securityContext:
    runAsGroup: 0
    runAsUser: 0
  volumes:
  - hostPath:
      path: /etc/kubernetes/admin.conf
    name: volume-0
  - hostPath:
      path: /etc/kubernetes/pki/ca.crt
    name: volume-1
  - hostPath:
      path: /etc/kubernetes/pki/admin.crt
    name: volume-2
  - hostPath:
      path: /etc/kubernetes/pki/admin.key
    name: volume-3
status: {}
EOF
docker pull registry.k8s.io/metrics-server/metrics-server:v0.7.1
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/manifests/metrics-server.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  name: metrics-server
  namespace: kube-system
spec:
  containers:
  - args:
    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
    - --kubelet-use-node-status-port
    - --kubelet-insecure-tls
    - --metric-resolution=15s
    - --bind-address=0.0.0.0
    - --secure-port=4443
    - --kubeconfig=~/.kube/config
    - --authentication-kubeconfig=~/.kube/config
    - --authorization-kubeconfig=~/.kube/config
    - --tls-cert-file=/etc/kubernetes/pki/admin.crt
    - --tls-private-key-file=/etc/kubernetes/pki/admin.key
    command:
    - /metrics-server
    image: registry.k8s.io/metrics-server/metrics-server:v0.7.1
    imagePullPolicy: Never
    name: metrics-server
    ports:
    - containerPort: 4443
      hostPort: 4443
      name: https
      protocol: TCP
    resources: {}
    volumeMounts:
    - mountPath: ~/.kube/config
      name: volume-0
      readOnly: true
    - mountPath: /etc/kubernetes/pki/ca.crt
      name: volume-1
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.crt
      name: volume-2
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.key
      name: volume-3
      readOnly: true
  hostNetwork: true
  restartPolicy: Always
  securityContext:
    runAsGroup: 0
    runAsUser: 0
  volumes:
  - hostPath:
      path: /etc/kubernetes/admin.conf
    name: volume-0
  - hostPath:
      path: /etc/kubernetes/pki/ca.crt
    name: volume-1
  - hostPath:
      path: /etc/kubernetes/pki/admin.crt
    name: volume-2
  - hostPath:
      path: /etc/kubernetes/pki/admin.key
    name: volume-3
status: {}
EOF
docker pull docker.io/prom/prometheus:v2.53.0
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/manifests/prometheus.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  name: prometheus
  namespace: kube-system
spec:
  containers:
  - args:
    - --config.file=/etc/prometheus/prometheus.yaml
    - --web.listen-address=0.0.0.0:9090
    command:
    - prometheus
    image: docker.io/prom/prometheus:v2.53.0
    imagePullPolicy: Never
    name: prometheus
    ports:
    - containerPort: 9090
      hostPort: 9090
      name: http
      protocol: TCP
    resources: {}
    volumeMounts:
    - mountPath: /etc/prometheus/prometheus.yaml
      name: volume-0
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.crt
      name: volume-1
      readOnly: true
    - mountPath: /etc/kubernetes/pki/admin.key
      name: volume-2
      readOnly: true
    - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt
      name: volume-3
      readOnly: true
    - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key
      name: volume-4
      readOnly: true
  hostNetwork: true
  restartPolicy: Always
  securityContext:
    runAsGroup: 0
    runAsUser: 0
  volumes:
  - hostPath:
      path: /var/components/prometheus/etc/prometheus/prometheus.yaml
    name: volume-0
  - hostPath:
      path: /etc/kubernetes/pki/admin.crt
    name: volume-1
  - hostPath:
      path: /etc/kubernetes/pki/admin.key
    name: volume-2
  - hostPath:
      path: /etc/kubernetes/pki/apiserver-etcd-client.crt
    name: volume-3
  - hostPath:
      path: /etc/kubernetes/pki/apiserver-etcd-client.key
    name: volume-4
status: {}
EOF
docker pull docker.io/jaegertracing/all-in-one:1.58.1
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/manifests/jaeger.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  name: jaeger
  namespace: kube-system
spec:
  containers:
  - args:
    - --collector.otlp.enabled=true
    - --query.http-server.host-port=0.0.0.0:16686
    - --collector.otlp.grpc.host-port=0.0.0.0:4317
    image: docker.io/jaegertracing/all-in-one:1.58.1
    imagePullPolicy: Never
    name: jaeger
    ports:
    - containerPort: 16686
      hostPort: 16686
      name: http
      protocol: TCP
    - containerPort: 4317
      hostPort: 4317
      name: otlp-grpc
      protocol: TCP
    resources: {}
  hostNetwork: true
  restartPolicy: Always
  securityContext:
    runAsGroup: 0
    runAsUser: 0
status: {}
EOF
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/prometheus.yaml
global:
  evaluation_interval: 15s
  scrape_interval: 15s
  scrape_timeout: 10s
scrape_configs:
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: etcd
  metrics_path: /metrics
  scheme: https
  static_configs:
  - targets:
    - 127.0.0.1:2379
  tls_config:
    cert_file: /etc/kubernetes/pki/apiserver-etcd-client.crt
    insecure_skip_verify: true
    key_file: /etc/kubernetes/pki/apiserver-etcd-client.key
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: kube-apiserver
  metrics_path: /metrics
  scheme: https
  static_configs:
  - targets:
    - 127.0.0.1:6443
  tls_config:
    cert_file: /etc/kubernetes/pki/admin.crt
    insecure_skip_verify: true
    key_file: /etc/kubernetes/pki/admin.key
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: kube-controller-manager
  metrics_path: /metrics
  scheme: https
  static_configs:
  - targets:
    - 127.0.0.1:10257
  tls_config:
    cert_file: /etc/kubernetes/pki/admin.crt
    insecure_skip_verify: true
    key_file: /etc/kubernetes/pki/admin.key
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: kube-scheduler
  metrics_path: /metrics
  scheme: https
  static_configs:
  - targets:
    - 127.0.0.1:10259
  tls_config:
    cert_file: /etc/kubernetes/pki/admin.crt
    insecure_skip_verify: true
    key_file: /etc/kubernetes/pki/admin.key
- http_sd_configs:
  - url: http://127.0.0.1:10247/discovery/prometheus
  job_name: kwok-controller-metrics-discovery
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: kwok-controller
  metrics_path: /metrics
  scheme: http
  static_configs:
  - targets:
    - 127.0.0.1:10247
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: metrics-server
  metrics_path: /metrics
  scheme: https
  static_configs:
  - targets:
    - 127.0.0.1:4443
  tls_config:
    cert_file: /etc/kubernetes/pki/admin.crt
    insecure_skip_verify: true
    key_file: /etc/kubernetes/pki/admin.key
- enable_http2: true
  follow_redirects: true
  honor_timestamps: true
  job_name: prometheus
  metrics_path: /metrics
  scheme: http
  static_configs:
  - targets:
    - 127.0.0.1:9090
EOF
# Save cluster config to <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kwok.yaml
KIND_EXPERIMENTAL_PROVIDER=docker kind create cluster --config <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kind.yaml --name kwok-<CLUSTER_NAME> --image docker.io/kindest/node:v1.32.2 --wait 29m
KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image registry.k8s.io/kubectl:v1.32.2 --name kwok-<CLUSTER_NAME>
KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image registry.k8s.io/kwok/kwok:v0.7.0 --name kwok-<CLUSTER_NAME>
KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/kubernetesui/dashboard:v2.7.0 --name kwok-<CLUSTER_NAME>
KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/kubernetesui/metrics-scraper:v1.0.9 --name kwok-<CLUSTER_NAME>
KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image registry.k8s.io/metrics-server/metrics-server:v0.7.1 --name kwok-<CLUSTER_NAME>
KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/prom/prometheus:v2.53.0 --name kwok-<CLUSTER_NAME>
KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/jaegertracing/all-in-one:1.58.1 --name kwok-<CLUSTER_NAME>
kubectl config view --minify=true --raw=true
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml
EOF
kubectl cordon kwok-<CLUSTER_NAME>-control-plane
docker exec kwok-<CLUSTER_NAME>-control-plane chmod -R +r /etc/kubernetes/pki
# Set up apiservice for metrics server
# Add context kwok-<CLUSTER_NAME> to ~/.kube/config
