Merge pull request #9063 from piosz/cluster_upgrade

cluster/common.sh

@@ -168,3 +168,55 @@ function get-kubeconfig-bearertoken() {
     KUBE_BEARER_TOKEN=''
   fi
 }
+
+# Sets binary_version variable to the version passed in as an argument, or if argument is
+# latest_stable, latest_release, or latest_ci fetches and sets the correponding version number
+#
+# Args:
+#   $1 version string from command line
+# Vars set:
+#   KUBE_VERSION
+function set_binary_version() {
+  if [[ "${1}" == "latest_stable" ]]; then
+    KUBE_VERSION=$(gsutil cat gs://kubernetes-release/release/stable.txt)
+    echo "Using latest stable version: ${binary_version}"
+  elif [[ "${1}" == "latest_release" ]]; then
+    KUBE_VERSION=$(gsutil cat gs://kubernetes-release/release/latest.txt)
+    echo "Using latest release version: ${binary_version}"
+  elif [[ "${1}" == "latest_ci" ]]; then
+    KUBE_VERSION=$(gsutil cat gs://kubernetes-release/ci/latest.txt)
+    echo "Using latest ci version: ${binary_version}"
+  else
+    KUBE_VERSION=${1}
+  fi
+}
+
+# Figure out which binary use on the server and assure it is available.
+# If KUBE_VERSION is specified use binaries specified by it, otherwise
+# use local dev binaries.
+#
+# Assumed vars:
+#   PROJECT
+# Vars set:
+#   SERVER_BINARY_TAR_URL
+#   SALT_TAR_URL
+function tars_from_version() {
+  if [[ -z "${KUBE_VERSION-}" ]]; then
+    find-release-tars
+    upload-server-tars
+  elif [[ ${KUBE_VERSION} =~ ${KUBE_VERSION_REGEX} ]]; then
+    SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
+    SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/kubernetes-salt.tar.gz"
+  elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then
+    SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz"
+    SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${KUBE_VERSION}/kubernetes-salt.tar.gz"
+  else
+    echo "Version doesn't match regexp" >&2
+    exit 1
+  fi
+
+  if ! curl -Ss --range 0-1 ${SERVER_BINARY_TAR_URL} >&/dev/null; then
+    echo "Can't find release at ${SERVER_BINARY_TAR_URL}" >&2
+    exit 1
+  fi
+}

cluster/gce/upgrade.sh

@@ -22,12 +22,6 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-# VERSION_REGEX matches things like "v0.13.1"
-readonly VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$"
-
-# CI_VERSION_REGEX matches things like "v0.14.1-341-ge0c9d9e"
-readonly CI_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-(.*)$"
-
 if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then
   echo "!!! ${1} only works on GCE" >&2
   exit 1
@@ -95,41 +89,14 @@ function wait-for-master() {
   echo "== Done =="
 }
 
-# Sets binary_version variable to the version passed in as an argument, or if argument is
-# latest_stable, latest_release, or latest_ci fetches and sets the correponding version number
-#
-# Args:
-# $1 version string from command line
-function set_binary_version() {
-  if [[ "${1}" == "latest_stable" ]]; then
-    binary_version=$(gsutil cat gs://kubernetes-release/release/stable.txt)
-    echo "Using latest stable version: ${binary_version}"
-  elif [[ "${1}" == "latest_release" ]]; then
-    binary_version=$(gsutil cat gs://kubernetes-release/release/latest.txt)
-    echo "Using latest release version: ${binary_version}"
-  elif [[ "${1}" == "latest_ci" ]]; then
-    binary_version=$(gsutil cat gs://kubernetes-release/ci/latest.txt)
-    echo "Using latest ci version: ${binary_version}"
-  else
-    binary_version=${1}
-  fi
-}
-
 # Perform common upgrade setup tasks
 #
 # Assumed vars
-#   local_binaries
-#   binary_version
+#   KUBE_VERSION
 function prepare-upgrade() {
   ensure-temp-dir
   detect-project
-
-  if [[ "${local_binaries}" == "true" ]]; then
-    find-release-tars
-    upload-server-tars
-  else
-    tars_from_version ${binary_version}
-  fi
+  tars_from_version
 }
 
 # Reads kube-env metadata from master and extracts value from provided key.
@@ -148,11 +115,15 @@ function get-env-val() {
     | grep ${1} | cut -d : -f 2 | cut -d \' -f 2
 }
 
-# $1 veresion
+# Assumed vars:
+#   KUBE_VERSION
+#   MINION_SCOPES
+#   NODE_INSTANCE_PREFIX
+#   PROJECT
+#   ZONE
 function upgrade-nodes() {
-  local version=${1}
-  local sanitized_version=$(echo ${version} | sed s/"\."/-/g)
-  echo "== Upgrading nodes to ${version}. =="
+  local sanitized_version=$(echo ${KUBE_VERSION} | sed s/"\."/-/g)
+  echo "== Upgrading nodes to ${KUBE_VERSION}. =="
 
   detect-minion-names
 
@@ -187,28 +158,6 @@ function upgrade-nodes() {
   echo "== Done =="
 }
 
-function tars_from_version() {
-  version=${1-}
-
-  if [[ ${version} =~ ${VERSION_REGEX} ]]; then
-    SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${version}/kubernetes-server-linux-amd64.tar.gz"
-    SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${version}/kubernetes-salt.tar.gz"
-  elif [[ ${version} =~ ${CI_VERSION_REGEX} ]]; then
-    SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${version}/kubernetes-server-linux-amd64.tar.gz"
-    SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${version}/kubernetes-salt.tar.gz"
-  else
-    echo "!!! Version not provided or version doesn't match regexp" >&2
-    exit 1
-  fi
-
-  if ! curl -Ss --range 0-1 ${SERVER_BINARY_TAR_URL} >&/dev/null; then
-    echo "!!! Can't find release at ${SERVER_BINARY_TAR_URL}" >&2
-    exit 1
-  fi
-
-  echo "== Release ${version} validated =="
-}
-
 master_upgrade=true
 node_upgrade=true
 local_binaries=false
@@ -261,7 +210,7 @@ if [[ "${node_upgrade}" == "true" ]]; then
   if [[ "${local_binaries}" == "true" ]]; then
     echo "Upgrading nodes to local binaries is not yet supported." >&2
   else
-    upgrade-nodes ${binary_version}
+    upgrade-nodes
   fi
 fi
 

cluster/gce/util.sh

@@ -36,6 +36,13 @@ ALLOCATE_NODE_CIDRS=true
 KUBE_PROMPT_FOR_UPDATE=y
 KUBE_SKIP_UPDATE=${KUBE_SKIP_UPDATE-"n"}
 
+# VERSION_REGEX matches things like "v0.13.1"
+readonly KUBE_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$"
+
+# CI_VERSION_REGEX matches things like "v0.14.1-341-ge0c9d9e"
+readonly KUBE_CI_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-(.*)$"
+
+
 function join_csv {
   local IFS=','; echo "$*";
 }
@@ -837,12 +844,13 @@ function kube-down {
   set -e
 }
 
-# Update a kubernetes cluster with latest source
-function kube-push {
+# Prepare to push new binaries to kubernetes cluster
+#  $1 - whether prepare push to node
+function prepare-push() {
   #TODO(dawnchen): figure out how to upgrade coreos node
   if [[ "${OS_DISTRIBUTION}" != "debian" ]]; then
     echo "Updating a kubernetes cluster with ${OS_DISTRIBUTION} is not supported yet." >&2
-    return
+    exit 1
   fi
 
   OUTPUT=${KUBE_ROOT}/_output/logs
@@ -856,17 +864,79 @@ function kube-push {
   get-bearer-token
 
   # Make sure we have the tar files staged on Google Storage
-  find-release-tars
-  upload-server-tars
+  tars_from_version
+
+  # Prepare node env vars and update MIG template
+  if [[ "${1-}" == "true" ]]; then
+    write-node-env
+
+    # TODO(mbforbes): Refactor setting scope flags.
+    local -a scope_flags=()
+    if (( "${#MINION_SCOPES[@]}" > 0 )); then
+      scope_flags=("--scopes" "${MINION_SCOPES[@]}")
+    else
+      scope_flags=("--no-scopes")
+    fi
+
+    # Ugly hack: Since it is not possible to delete instance-template that is currently
+    # being used, create a temp one, then delete the old one and recreate it once again.
+    create-node-instance-template "tmp"
+
+    gcloud preview managed-instance-groups --zone "${ZONE}" \
+      set-template "${NODE_INSTANCE_PREFIX}-group" \
+      --project "${PROJECT}" \
+      --template "${NODE_INSTANCE_PREFIX}-template-tmp" || true;
+
+    gcloud compute instance-templates delete \
+      --project "${PROJECT}" \
+      --quiet \
+      "${NODE_INSTANCE_PREFIX}-template" || true
+
+    create-node-instance-template
+
+    gcloud preview managed-instance-groups --zone "${ZONE}" \
+      set-template "${NODE_INSTANCE_PREFIX}-group" \
+      --project "${PROJECT}" \
+      --template "${NODE_INSTANCE_PREFIX}-template" || true;
+
+    gcloud compute instance-templates delete \
+      --project "${PROJECT}" \
+      --quiet \
+      "${NODE_INSTANCE_PREFIX}-template-tmp" || true
+  fi
+}
 
+# Push binaries to kubernetes master
+function push-master {
   echo "Updating master metadata ..."
   write-master-env
   add-instance-metadata-from-file "${KUBE_MASTER}" "kube-env=${KUBE_TEMP}/master-kube-env.yaml" "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh"
 
-  echo "Pushing to master (log at ${OUTPUT}/kube-push-${KUBE_MASTER}.log) ..."
-  cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${KUBE_MASTER}" --command "sudo bash -s -- --push" &> ${OUTPUT}/kube-push-"${KUBE_MASTER}".log
+  echo "Pushing to master (log at ${OUTPUT}/push-${KUBE_MASTER}.log) ..."
+  cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${KUBE_MASTER}" --command "sudo bash -s -- --push" &> ${OUTPUT}/push-"${KUBE_MASTER}".log
+}
+
+# Push binaries to kubernetes node
+function push-node() {
+  node=${1}
+
+  echo "Updating node ${node} metadata... "
+  add-instance-metadata-from-file "${node}" "kube-env=${KUBE_TEMP}/node-kube-env.yaml" "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh"
+
+  echo "Start upgrading node ${node} (log at ${OUTPUT}/push-${node}.log) ..."
+  cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${node}" --command "sudo bash -s -- --push" &> ${OUTPUT}/push-"${node}".log
+}
+
+# Push binaries to kubernetes cluster
+function kube-push {
+  prepare-push true
 
-  kube-update-nodes push
+  push-master
+
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+    push-node "${MINION_NAMES[$i]}" &
+  done
+  wait-for-jobs
 
   # TODO(zmerlynn): Re-create instance-template with the new
   # node-kube-env. This isn't important until the node-ip-range issue
@@ -885,43 +955,6 @@ function kube-push {
   echo
 }
 
-# Push or upgrade nodes.
-#
-# TODO: This really needs to trampoline somehow to the configure-vm.sh
-# from the .tar.gz that we're actually pushing onto the node, because
-# that configuration shifts over versions. Right now, we're blasting
-# the configure-vm from our version instead.
-#
-# Assumed vars:
-#  KUBE_ROOT
-#  MINION_NAMES
-#  KUBE_TEMP
-#  PROJECT
-#  ZONE
-function kube-update-nodes() {
-  action=${1}
-
-  OUTPUT=${KUBE_ROOT}/_output/logs
-  mkdir -p ${OUTPUT}
-
-  echo "Updating node metadata... "
-  write-node-env
-  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
-    add-instance-metadata-from-file "${MINION_NAMES[$i]}" "kube-env=${KUBE_TEMP}/node-kube-env.yaml" "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" &
-  done
-  wait-for-jobs
-  echo "Done"
-
-  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
-    echo "Starting ${action} on node (log at ${OUTPUT}/kube-${action}-${MINION_NAMES[$i]}.log) ..."
-    cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${MINION_NAMES[$i]}" --command "sudo bash -s -- --push" &> ${OUTPUT}/kube-${action}-"${MINION_NAMES[$i]}".log &
-  done
-
-  echo -n "Waiting..."
-  wait-for-jobs
-  echo "Done"
-}
-
 # -----------------------------------------------------------------------------
 # Cluster specific test helpers used from hack/e2e-test.sh
 

cluster/kube-push.sh

@@ -27,10 +27,61 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/cluster/kube-env.sh"
 source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
 
-echo "Updating cluster using provider: $KUBERNETES_PROVIDER"
+function usage() {
+  echo "${0} [-m|-n <node id>] <version>"
+  echo "  Updates Kurnetes binaries. Can be done for all components (by default), master(-m) or specified node(-n)."
+  echo "  If the version is not specified will try to use local binaries."
+  echo "  Warning: upgrading single node is experimental"
+}
+
+push_to_master=false
+push_to_node=false
+
+while getopts "mn:h" opt; do
+  case ${opt} in
+    m)
+      push_to_master=true;;
+    n)
+      push_to_node=true
+      node_id="$OPTARG";;
+    h)
+      usage
+      exit 0;;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      usage
+      exit 1;;
+  esac
+done
+shift $((OPTIND-1))
+
+if [[ "${push_to_master}" == "true" ]] && [[ "${push_to_node}" == "true" ]]; then
+  echo "Only one of options -m -n should be specified"
+  usage
+  exit 1
+fi
 
 verify-prereqs
-kube-push
+KUBE_VERSION=${1-}
+
+if [[ "${push_to_master}" == "false" ]] && [[ "${push_to_node}" == "false" ]]; then
+  echo "Updating cluster using provider: $KUBERNETES_PROVIDER"
+  kube-push
+fi
+
+if [[ "${push_to_master}" == "true" ]]; then
+  echo "Udating master to version ${KUBE_VERSION:-"dev"}"
+  prepare-push false
+  push-master
+fi
+
+if [[ "${push_to_node}" == "true" ]]; then
+  echo "Updating node $node_id to version ${KUBE_VERSION:-"dev"}"
+  prepare-push true
+  push-node $node_id
+fi
+
+echo "Validating cluster post-push..."
 
 "${KUBE_ROOT}/cluster/validate-cluster.sh"
 

cluster/kube-util.sh

@@ -47,11 +47,26 @@ function kube-down {
 	echo "TODO"
 }
 
-# Update a kubernetes cluster with latest source
+# Update a kubernetes cluster
 function kube-push {
 	echo "TODO"
 }
 
+# Prepare update a kubernetes component
+function prepare-push {
+	echo "TODO"
+}
+
+# Update a kubernetes master
+function push-master {
+	echo "TODO"
+}
+
+# Update a kubernetes node
+function push-node {
+	echo "TODO"
+}
+
 # Execute prior to running tests to build a release if required for env
 function test-build-release {
 	echo "TODO"