retry some operation in the gke kube-up.sh

cluster/gke/util.sh

@@ -25,6 +25,25 @@ source "${KUBE_ROOT}/cluster/gke/${KUBE_CONFIG_FILE:-config-default.sh}"
 source "${KUBE_ROOT}/cluster/common.sh"
 source "${KUBE_ROOT}/cluster/lib/util.sh"
 
+function with-retry() {
+  local retry_limit=$1
+  local cmd=("${@:2}")
+
+  local retry_count=0
+  local rc=0
+
+  until [[ ${retry_count} -ge ${retry_limit} ]]; do
+    ((retry_count+=1))
+    "${cmd[@]}" && rc=0 || rc=$?
+    if [[ ${rc} == 0 ]]; then
+      return 0
+    fi
+    sleep 3
+  done
+
+  return ${rc}
+}
+
 # Perform preparations required to run e2e tests
 #
 # Assumed vars:
@@ -140,7 +159,7 @@ function kube-up() {
   # Make the specified network if we need to.
   if ! "${GCLOUD}" compute networks --project "${PROJECT}" describe "${NETWORK}" &>/dev/null; then
     echo "Creating new network: ${NETWORK}" >&2
-    "${GCLOUD}" compute networks create "${NETWORK}" --project="${PROJECT}" --range "${NETWORK_RANGE}"
+    with-retry 3 "${GCLOUD}" compute networks create "${NETWORK}" --project="${PROJECT}" --range "${NETWORK_RANGE}"
   else
     echo "... Using network: ${NETWORK}" >&2
   fi
@@ -149,7 +168,7 @@ function kube-up() {
   # such a rule exists, only whether we've created this exact rule.
   if ! "${GCLOUD}" compute firewall-rules --project "${PROJECT}" describe "${FIREWALL_SSH}" &>/dev/null; then
     echo "Creating new firewall for SSH: ${FIREWALL_SSH}" >&2
-    "${GCLOUD}" compute firewall-rules create "${FIREWALL_SSH}" \
+    with-retry 3 "${GCLOUD}" compute firewall-rules create "${FIREWALL_SSH}" \
       --allow="tcp:22" \
       --network="${NETWORK}" \
       --project="${PROJECT}" \
@@ -226,14 +245,14 @@ function test-setup() {
   OLD_NODE_TAG="k8s-${CLUSTER_NAME}-node"
 
   # Open up port 80 & 8080 so common containers on minions can be reached.
-  "${GCLOUD}" compute firewall-rules create \
+  with-retry 3 "${GCLOUD}" compute firewall-rules create \
     "${CLUSTER_NAME}-http-alt" \
     --allow tcp:80,tcp:8080 \
     --project "${PROJECT}" \
     --target-tags "${NODE_TAG},${OLD_NODE_TAG}" \
     --network="${NETWORK}" &
 
-  "${GCLOUD}" compute firewall-rules create \
+  with-retry 3 "${GCLOUD}" compute firewall-rules create \
     "${CLUSTER_NAME}-nodeports" \
     --allow tcp:30000-32767,udp:30000-32767 \
     --project "${PROJECT}" \
@@ -372,7 +391,7 @@ function test-teardown() {
   # NOTE: Keep in sync with names above in test-setup.
   for fw in "${CLUSTER_NAME}-http-alt" "${CLUSTER_NAME}-nodeports" "${FIREWALL_SSH}"; do
     if [[ -n $("${GCLOUD}" compute firewall-rules --project "${PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then
-      "${GCLOUD}" compute firewall-rules delete "${fw}" --project="${PROJECT}" --quiet &
+      with-retry 3 "${GCLOUD}" compute firewall-rules delete "${fw}" --project="${PROJECT}" --quiet &
     fi
   done
 
@@ -384,7 +403,7 @@ function test-teardown() {
   # symmetry.
   if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then
     if [[ -n $("${GCLOUD}" compute networks --project "${PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then
-      if ! "${GCLOUD}" compute networks delete --project "${PROJECT}" --quiet "${NETWORK}"; then
+      if ! with-retry 3 "${GCLOUD}" compute networks delete --project "${PROJECT}" --quiet "${NETWORK}"; then
         echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:"
         "${GCLOUD}" compute firewall-rules --project "${PROJECT}" list --filter="network=${NETWORK}"
       fi
@@ -402,7 +421,7 @@ function kube-down() {
   echo "... in gke:kube-down()" >&2
   detect-project >&2
   if "${GCLOUD}" ${CMD_GROUP:-} container clusters describe --project="${PROJECT}" --zone="${ZONE}" "${CLUSTER_NAME}" --quiet &>/dev/null; then
-    "${GCLOUD}" ${CMD_GROUP:-} container clusters delete --project="${PROJECT}" \
+    with-retry 3 "${GCLOUD}" ${CMD_GROUP:-} container clusters delete --project="${PROJECT}" \
       --zone="${ZONE}" "${CLUSTER_NAME}" --quiet
   fi
 }