Merge pull request #35308 from vmware/automated-cherry-pick-of-#35232-upstream-release-1.4

Kubernetes Submit Queue · web-flow · commit 68ba01dd0d77 · 2016-10-24T17:46:36.000-07:00
Automatic merge from submit-queue Wait for cbr0 configuration to complete before setting up routes. - f… Automated cherry pick of #35232 , this is needed for kube-up to work on vSphere, need this in the next 1.4 release. fixes #34248, #31310 - Remove optimizations for salt status check till flakyness of install is addressed - fix indentation
diff --git a/cluster/vsphere/config-default.sh b/cluster/vsphere/config-default.sh
@@ -28,7 +28,7 @@ MASTER_MEMORY_MB=1024
 MASTER_CPU=1
 
 NODE_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_NODES}}))
-NODE_IP_RANGES="10.244.0.0/16"
+NODE_IP_RANGES="10.244.0.0/16" # Min Prefix supported is 16
 MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
 NODE_MEMORY_MB=2048
 NODE_CPU=1
diff --git a/cluster/vsphere/util.sh b/cluster/vsphere/util.sh
@@ -325,23 +325,52 @@ function setup-pod-routes {
   # identify the subnet assigned to the node by the kubernertes controller manager.
   KUBE_NODE_BRIDGE_NETWORK=()
   for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
-     printf " finding network of cbr0 bridge on node  ${NODE_NAMES[$i]}\n"
-     network=$(kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} 'sudo ip route show | grep -E "dev cbr0" | cut -d     " " -f1')
-     KUBE_NODE_BRIDGE_NETWORK+=("${network}")
-  done
+    printf " finding network of cbr0 bridge on node  ${NODE_NAMES[$i]}\n"
+
+    network=""
+    top2_octets_final=$(echo $NODE_IP_RANGES | awk -F "." '{ print $1 "." $2 }') # Assume that a 24 bit mask per node
+
+    attempt=0
+    max_attempt=60
+    while true ; do
+      attempt=$(($attempt+1))
+
+      network=$(kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} 'sudo ip route show | grep -E "dev cbr0" | cut -d     " " -f1')
+      top2_octets_read=$(echo $network | awk -F "." '{ print $1 "." $2 }')
+
+      if [[ "$top2_octets_read" == "$top2_octets_final" ]]; then
+        break
+      fi
+
+      if (( $attempt == $max_attempt )); then
+        echo
+        echo "(Failed) Waiting for cbr0 bridge to come up @ ${NODE_NAMES[$i]}"
+        echo
+        exit 1
+      fi
 
+      printf "."
+      sleep 5
+    done
+
+    printf "\n"
+    KUBE_NODE_BRIDGE_NETWORK+=("${network}")
+  done
 
   # Make the pods visible to each other and to the master.
   # The master needs have routes to the pods for the UI to work.
   local j
   for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
-     printf "setting up routes for ${NODE_NAMES[$i]}"
-     kube-ssh "${KUBE_MASTER_IP}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${i}]} gw ${KUBE_NODE_IP_ADDRESSES[${i}]}"
-     for (( j=0; j<${#NODE_NAMES[@]}; j++)); do
-        if [[ $i != $j ]]; then
-           kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[$j]} gw ${KUBE_NODE_IP_ADDRESSES[$j]}"
-        fi
-      done
+    printf "setting up routes for ${NODE_NAMES[$i]}\n"
+    printf " adding route to ${MASTER_NAME} for network ${KUBE_NODE_BRIDGE_NETWORK[${i}]} via ${KUBE_NODE_IP_ADDRESSES[${i}]}\n"
+    kube-ssh "${KUBE_MASTER_IP}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${i}]} gw ${KUBE_NODE_IP_ADDRESSES[${i}]}"
+    for (( j=0; j<${#NODE_NAMES[@]}; j++)); do
+      if [[ $i != $j ]]; then
+        printf " adding route to ${NODE_NAMES[$j]} for network ${KUBE_NODE_BRIDGE_NETWORK[${i}]} via ${KUBE_NODE_IP_ADDRESSES[${i}]}\n" 
+        kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[$j]} gw ${KUBE_NODE_IP_ADDRESSES[$j]}"
+      fi
+    done
+    printf "\n"
   done
 }
 
