Support upgrade/downgrade in etcd image.

cluster/images/etcd/migrate-if-needed.sh

-#!/bin/sh
+#!/bin/bash
 
 # Copyright 2016 The Kubernetes Authors.
 #
@@ -14,49 +14,93 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This script performs data migration between etcd2 and etcd3 versions
-# if needed.
-# Expected usage of it is:
-#   ./migrate_if_needed <target-storage> <data-dir>
-# It will look into the contents of file <data-dir>/version.txt to
-# determine the current storage version (no file means etcd2).
+# NOTES
+# This script performs etcd upgrade based on the following environmental
+# variables:
+# TARGET_STORAGE - API of etcd to be used (supported: 'etcd2', 'etcd3')
+# TARGET_VERSION - etcd release to be used (supported: '2.2.1', '2.3.7', '3.0.10')
+# DATA_DIRECTORY - directory with etcd data
+#
+# The current etcd version and storage format is detected based on the
+# contents of "${DATA_DIRECTORY}/version.txt" file (if the file doesn't
+# exist, we default it to "2.2.1/etcd2".
+#
+# The update workflow support the following upgrade steps:
+# - 2.2.1/etcd2 -> 2.3.7/etcd2
+# - 2.3.7/etcd2 -> 3.0.10/etcd2
+# - 3.0.10/etcd2 -> 3.0.10/etcd3
+#
+# NOTE: The releases supported in this script has to match release binaries
+# present in the etcd image (to make this script work correctly).
+#
+# Based on the current etcd version and storage format we detect what
+# upgrade step from this should be done to get reach target configuration
 
 set -o errexit
 set -o nounset
 
 if [ -z "${TARGET_STORAGE:-}" ]; then
-  echo "TARGET_USAGE variable unset - skipping migration"
+  echo "TARGET_STORAGE variable unset - skipping migration"
+  exit 0
+fi
+if [ -z "${TARGET_VERSION:-}" ]; then
+  echo "TARGET_VERSION variable unset - skipping migration"
   exit 0
 fi
-
 if [ -z "${DATA_DIRECTORY:-}" ]; then
   echo "DATA_DIRECTORY variable unset - skipping migration"
   exit 0
 fi
 
-ETCD="${ETCD:-/usr/local/bin/etcd}"
-ETCDCTL="${ETCDCTL:-/usr/local/bin/etcdctl}"
-ATTACHLEASE="${ATTACHLEASE:-/usr/local/bin/attachlease}"
+if [ "${TARGET_STORAGE}" != "etcd2" -a "${TARGET_STORAGE}" != "etcd3" ]; then
+  echo "Not supported version of storage: ${TARGET_STORAGE}"
+  exit 1
+fi
+
+# NOTE: SUPPORTED_VERSION has to match release binaries present in the
+# etcd image (to make this script work correctly).
+SUPPORTED_VERSIONS=("2.2.1" "2.3.7" "3.0.10")
+
 VERSION_FILE="version.txt"
-CURRENT_STORAGE='etcd2'
+CURRENT_STORAGE="etcd2"
+CURRENT_VERSION="2.2.1"
 if [ -e "${DATA_DIRECTORY}/${VERSION_FILE}" ]; then
-  CURRENT_STORAGE="$(cat ${DATA_DIRECTORY}/${VERSION_FILE})"
+  VERSION_CONTENTS="$(cat ${DATA_DIRECTORY}/${VERSION_FILE})"
+  # Example usage: if contents of VERSION_FILE is 2.3.7/etcd2, then
+  # - CURRENT_VERSION would be '2.3.7'
+  # - CURRENT_STORAGE would be 'etcd2'
+  CURRENT_VERSION="$(echo $VERSION_CONTENTS | cut -d '/' -f 1)"
+  CURRENT_STORAGE="$(echo $VERSION_CONTENTS | cut -d '/' -f 2)"
 fi
 
+# Starts 'etcd' version ${START_VERSION} and writes to it:
+# 'etcd_version' -> "${START_VERSION}"
+# Successful write confirms that etcd is up and running.
+# Sets ETCD_PID at the end.
+# Returns 0 if etcd was successfully started, non-0 otherwise.
 start_etcd() {
+  # Use random ports, so that apiserver cannot connect to etcd.
   ETCD_PORT=18629
   ETCD_PEER_PORT=18630
-  ${ETCD} --data-dir=${DATA_DIRECTORY} \
+  local ETCD_CMD="${ETCD:-/usr/local/bin/etcd-${START_VERSION}}"
+  local ETCDCTL_CMD="${ETCDCTL:-/usr/local/bin/etcdctl-${START_VERSION}}"
+  if [ "${START_VERSION:0:2}" == "2." ]; then
+    ETCDCTL_CMD="${ETCDCTL_CMD} --endpoint=http://127.0.0.1:${ETCD_PORT} set"
+  else
+    ETCDCTL_CMD="${ETCDCTL_CMD} --endpoints=http://127.0.0.1:${ETCD_PORT} put"
+  fi
+  ${ETCD_CMD} --data-dir=${DATA_DIRECTORY} \
     --listen-client-urls http://127.0.0.1:${ETCD_PORT} \
     --advertise-client-urls http://127.0.0.1:${ETCD_PORT} \
     --listen-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \
     --initial-advertise-peer-urls http://127.0.0.1:${ETCD_PEER_PORT} \
     1>>/dev/null 2>&1 &
   ETCD_PID=$!
-  # Wait until etcd is up.
-  for i in $(seq 30); do
-    local out
-    if out=$(wget -q --timeout=1 http://127.0.0.1:${ETCD_PORT}/v2/machines 2> /dev/null); then
+  local API_VERSION="${START_STORAGE:4:4}"
+  # Wait until we can write to etcd.
+  for i in $(seq 240); do
+    ETCDCTL_API="${API_VERSION}" ${ETCDCTL_CMD} 'etcd_version' ${START_VERSION}
+    if [ "$?" -eq "0" ]; then
       echo "Etcd on port ${ETCD_PORT} is up."
       return 0
     fi
@@ -66,43 +110,85 @@ start_etcd() {
   return 1
 }
 
+# Stops etcd with ${ETCD_PID} pid.
 stop_etcd() {
   kill "${ETCD_PID-}" >/dev/null 2>&1 || :
   wait "${ETCD_PID-}" >/dev/null 2>&1 || :
 }
 
-if [ "${CURRENT_STORAGE}" = "etcd2" -a "${TARGET_STORAGE}" = "etcd3" ]; then
-  # If directory doesn't exist or is empty, this means that there aren't any
-  # data for migration, which means we can skip this step.
+ATTACHLEASE="${ATTACHLEASE:-/usr/local/bin/attachlease}"
+ROLLBACK="${ROLLBACK:-/usr/local/bin/rollback}"
+
+# Do the roll-forward migration if needed.
+for step in "${SUPPORTED_VERSIONS[@]}"; do
+  if [ "${step}" == "${CURRENT_VERSION}"  -a "${CURRENT_VERSION}" != "${TARGET_VERSION}" ]; then
+    # Do the migration step, by just starting etcd in this version.
+    START_VERSION="${step}"
+    START_STORAGE="${CURRENT_STORAGE}"
+    if ! start_etcd; then
+      # Migration failed.
+      echo "Starting etcd ${step} failed"
+      exit 1
+    fi
+    # Kill etcd and wait until this is down.
+    stop_etcd
+  fi
+  CURRENT_VERSION=${step}
+  echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"
+  if [ "${CURRENT_VERSION:0:2}" == "3." -a "${CURRENT_STORAGE}" == "etcd2" -a "${TARGET_STORAGE}" == "etcd3" ]; then
+    # If it is the first 3.x release in the list and we are migrating
+    # also from 'etcd2' to 'etcd3', do the migration now.
+    if [ -d "${DATA_DIRECTORY}" ]; then
+      if [ "$(ls -A ${DATA_DIRECTORY})" ]; then
+        echo "Performing etcd2 -> etcd3 migration"
+        START_VERSION="${step}"
+        START_STORAGE="etcd3"
+        ETCDCTL_CMD="${ETCDCTL:-/usr/local/bin/etcdctl-${START_VERSION}}"
+        ETCDCTL_API=3 ${ETCDCTL_CMD} migrate --data-dir=${DATA_DIRECTORY}
+        echo "Attaching leases to TTL entries"
+        # Now attach lease to all keys.
+        # To do it, we temporarily start etcd on a random port (so that
+        # apiserver actually cannot access it).
+        if ! start_etcd; then
+          echo "Starting etcd ${step} in v3 mode failed"
+          exit 1
+        fi
+        # Create a lease and attach all keys to it.
+        ${ATTACHLEASE} \
+          --etcd-address http://127.0.0.1:${ETCD_PORT} \
+          --ttl-keys-prefix "${TTL_KEYS_DIRECTORY:-/registry/events}" \
+          --lease-duration 1h
+        # Kill etcd and wait until this is down.
+        stop_etcd
+      fi
+    fi
+    CURRENT_STORAGE="etcd3"
+    echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"
+  fi
+  if [ "${CURRENT_VERSION}" == "${TARGET_VERSION}" -a "${CURRENT_STORAGE}" == "${TARGET_STORAGE}" ]; then
+    break
+  fi
+done
+
+# Do the rollback of needed.
+# NOTE: Rollback is only supported from "3.0.x" version in 'etcd3' mode to
+# "2.3.7" version in 'etcd2' mode.
+if [ "${CURRENT_STORAGE}" == "etcd3" -a "${TARGET_STORAGE}" == "etcd2" ]; then
+  if [ "${CURRENT_VERSION:0:4}" != "3.0." -o "${TARGET_VERSION}" != "2.3.7" ]; then
+    echo "etcd3 -> etcd2 downgrade is supported only between 3.0.x and 2.3.7"
+    return 0
+  fi
   if [ -d "${DATA_DIRECTORY}" ]; then
     if [ "$(ls -A ${DATA_DIRECTORY})" ]; then
-      echo "Performing etcd2 -> etcd3 migration"
-      ETCDCTL_API=3 ${ETCDCTL} migrate --data-dir=${DATA_DIRECTORY}
-      echo "Attaching leases to TTL entries"
-      # Now attach lease to all keys.
-      # To do it, we temporarily start etcd on a random port (so that
-      # apiserver actually cannot access it).
-      start_etcd
-      # Create a lease and attach all keys to it.
-      ${ATTACHLEASE} \
-        --etcd-address http://127.0.0.1:${ETCD_PORT} \
-        --ttl-keys-prefix "${TTL_KEYS_DIRECTORY:-/registry/events}" \
-        --lease-duration 1h
-      # Kill etcd and wait until this is down.
-      stop_etcd
+      echo "Performing etcd3 -> etcd2 rollback"
+      ${ROLLBACK} --data-dir "${DATA_DIRECTORY}"
+      if [ "$?" -ne "0" ]; then
+        echo "Rollback to etcd2 failed"
+        exit 1
+      fi
     fi
   fi
+  CURRENT_STORAGE="etcd2"
+  CURRENT_VERSION="2.3.7"
+  echo "${CURRENT_VERSION}/${CURRENT_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"
 fi
-
-if [ "${CURRENT_STORAGE}" = "etcd3" -a "${TARGET_STORAGE}" = "etcd2" ]; then
-  echo "Performing etcd3 -> etcd2 migration"
-  # TODO: Implement rollback once this will be supported.
-  echo "etcd3 -> etcd2 downgrade is NOT supported."
-  # FIXME: On rollback, we will not support TTLs - we will simply clear
-  # all events.
-fi
-
-# Write current storage version to avoid future migrations.
-# If directory doesn't exist, we need to create it first.
-mkdir -p "${DATA_DIRECTORY}"
-echo "${TARGET_STORAGE}" > "${DATA_DIRECTORY}/${VERSION_FILE}"

hack/test-update-storage-objects.sh

@@ -103,6 +103,7 @@ make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver
 make -C "${KUBE_ROOT}" WHAT=cluster/images/etcd/attachlease
 
 kube::etcd::start
+echo "${ETCD_VERSION}/${STORAGE_BACKEND_ETCD2}" > "${ETCD_DIR}/version.txt"
 
 ### BEGIN TEST DEFINITION CUSTOMIZATION ###
 
@@ -159,6 +160,7 @@ killApiServer
 
 kube::etcd::stop
 TARGET_STORAGE="etcd3" \
+  TARGET_VERSION="3.0.10" \
   DATA_DIRECTORY="${ETCD_DIR}" \
   ETCD=$(which etcd) \
   ETCDCTL=$(which etcdctl) \