Merge pull request #1011 from pietern/vsphere-squash

README.md

@@ -20,6 +20,7 @@ While the concepts and architecture in Kubernetes represent years of experience
   * [Locally](docs/getting-started-guides/locally.md)
   * [CoreOS](docs/getting-started-guides/coreos.md)
   * [Fedora](docs/getting-started-guides/fedora.md)
+  * [vSphere](docs/getting-started-guides/vsphere.md)
 * [kubecfg command line tool](https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/cli.md)
 * [Kubernetes API Documentation](http://cdn.rawgit.com/GoogleCloudPlatform/kubernetes/ce4fcc4ad89ed7b481a46a0ad4c99dee4c6f24ba/api/kubernetes.html)
 * [Discussion and Community Support](#community-discussion-and-support)

cluster/kube-env.sh

@@ -18,6 +18,6 @@
 # You can override the default provider by exporting the KUBERNETES_PROVIDER
 # variable in your bashrc
 #
-# The valid values: 'gce', 'azure', 'vagrant', 'local'
+# The valid values: 'gce', 'azure', 'vagrant', 'local', 'vsphere'
 
 KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER:-gce}

cluster/saltbase/salt/apiserver/default

@@ -26,6 +26,15 @@
 {% elif grains.cloud is defined and grains.cloud == 'azure' %}
   MACHINES="{{ salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').values()|join(',', attribute='hostnamef') }}" 
   {% set machines = "-machines $MACHINES" %}
+{% else %}
+  # No cloud defined, collect IPs of minions as machines list.
+  # Use a bash array to build the value we need. It doesn't appear to be
+  # possible call functions map or zip, or use lambda's from Jinja.
+  MACHINE_IPS=()
+  {% for addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', expr_form='grain').values() %}
+  MACHINE_IPS+=( {{ addrs[0] }} )
+  {% endfor %}
+  {% set machines = "-machines=$(echo ${MACHINE_IPS[@]} | xargs -n1 echo | paste -sd,)" %}
 {% endif %}
 
 DAEMON_ARGS="{{daemon_args}} {{address}} {{machines}} {{etcd_servers}} {{ minion_regexp }} {{ cloud_provider }}"

cluster/saltbase/salt/static-routes/if-down

+#!/bin/bash
+
+[ "$IFACE" == "eth0" ] || exit 0
+
+{% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %}
+{% if ip_addrs[0] != salt['network.ip_addrs']('eth0')[0] %}
+{% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %}
+route del -net {{ cidr }}
+{% endif %}
+{% endfor %}

cluster/saltbase/salt/static-routes/if-up

+#!/bin/bash
+
+[ "$IFACE" == "eth0" ] || exit 0
+
+{% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %}
+{% if ip_addrs[0] != salt['network.ip_addrs']('eth0')[0] %}
+{% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %}
+route add -net {{ cidr }} gw {{ ip_addrs[0] }}
+{% endif %}
+{% endfor %}

cluster/saltbase/salt/static-routes/init.sls

+# Add static routes to every minion to enable pods in the 10.244.x.x range to
+# reach each other. This is suboptimal, but necessary to let every pod have
+# its IP and have pods between minions be able to talk with each other.
+# This will be obsolete when we figure out the right way to make this work.
+
+/etc/network/if-up.d/static-routes:
+  file.managed:
+    - source: salt://static-routes/if-up
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 755
+
+/etc/network/if-down.d/static-routes:
+  file.managed:
+    - source: salt://static-routes/if-down
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 755
+
+refresh-routes:
+  cmd.wait_script:
+    - source: salt://static-routes/refresh
+    - cwd: /etc/network/
+    - user: root
+    - group: root
+    - watch:
+      - file: /etc/network/if-up.d/static-routes
+      - file: /etc/network/if-down.d/static-routes

cluster/saltbase/salt/static-routes/refresh

+#!/bin/bash
+
+# Fake an ifup/ifdown event
+export IFACE=eth0
+
+if-down.d/static-routes || true
+if-up.d/static-routes || true

cluster/saltbase/salt/top.sls

@@ -19,3 +19,7 @@ base:
     - controller-manager
     - scheduler
     - nginx
+
+  'roles:kubernetes-pool-vsphere':
+    - match: grain
+    - static-routes

cluster/validate-cluster.sh

@@ -33,6 +33,13 @@ detect-minions > /dev/null
 MINIONS_FILE=/tmp/minions
 $(dirname $0)/kubecfg.sh -template '{{range.Items}}{{.ID}}:{{end}}' list minions > ${MINIONS_FILE}
 
+# On vSphere, use minion IPs as their names
+if [ "$KUBERNETES_PROVIDER" == "vsphere" ]; then
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+    MINION_NAMES[i]=${KUBE_MINION_IP_ADDRESSES[i]}
+  done
+fi
+
 for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
     # Grep returns an exit status of 1 when line is not found, so we need the : to always return a 0 exit status
     count=$(grep -c ${MINION_NAMES[i]} ${MINIONS_FILE}) || :

cluster/vsphere/config-common.sh

+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function public-key {
+  local dir=${HOME}/.ssh
+
+  for f in $HOME/.ssh/{id_{rsa,dsa},*}.pub; do
+    if [ -r $f ]; then
+      echo $f
+      return
+    fi
+  done
+
+  echo "Can't find public key file..."
+  exit 1
+}
+
+DISK=kube.vmdk
+GUEST_ID=debian7_64Guest
+PUBLIC_KEY_FILE=${PUBLIC_KEY_FILE-$(public-key)}
+SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null"
+
+# These need to be set
+#export GOVC_URL=
+#export GOVC_DATACENTER=
+#export GOVC_DATASTORE=
+#export GOVC_RESOURCE_POOL=
+#export GOVC_NETWORK=
+#export GOVC_GUEST_LOGIN='kube:kube'

cluster/vsphere/config-default.sh

+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source $(dirname ${BASH_SOURCE})/config-common.sh
+
+NUM_MINIONS=4
+INSTANCE_PREFIX=kubernetes
+
+MASTER_NAME="${INSTANCE_PREFIX}-master"
+MASTER_MEMORY_MB=1024
+MASTER_CPU=1
+
+MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
+MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
+MINION_MEMORY_MB=2048
+MINION_CPU=1

cluster/vsphere/config-test.sh

+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source $(dirname ${BASH_SOURCE})/config-common.sh
+
+NUM_MINIONS=2
+INSTANCE_PREFIX="e2e-test-${USER}"
+
+MASTER_NAME="${INSTANCE_PREFIX}-master"
+MASTER_MEMORY_MB=1024
+MASTER_CPU=1
+
+MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}}))
+MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
+MINION_MEMORY_MB=1024
+MINION_CPU=1

cluster/vsphere/templates/hostname.sh

+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Remove kube.vm from /etc/hosts
+sed -i -e 's/\b\w\+.vm\b//' /etc/hosts
+
+# Update hostname in /etc/hosts and /etc/hostname
+sed -i -e "s/\\bkube\\b/${MY_NAME}/g" /etc/host{s,name}
+hostname ${MY_NAME}

cluster/vsphere/templates/install-release.sh

+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Install release
+
+echo "Unpacking release"
+rm -rf master-release || false
+tar xzf master-release.tgz
+
+echo "Running release install script"
+sudo master-release/src/scripts/master-release-install.sh

cluster/vsphere/templates/salt-master.sh

+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use other Debian mirror
+sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
+
+# Prepopulate the name of the Master
+mkdir -p /etc/salt/minion.d
+echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
+
+cat <<EOF >/etc/salt/minion.d/grains.conf
+grains:
+  roles:
+    - kubernetes-master
+EOF
+
+# Auto accept all keys from minions that try to join
+mkdir -p /etc/salt/master.d
+cat <<EOF >/etc/salt/master.d/auto-accept.conf
+auto_accept: True
+EOF
+
+cat <<EOF >/etc/salt/master.d/reactor.conf
+# React to new minions starting by running highstate on them.
+reactor:
+  - 'salt/minion/*/start':
+    - /srv/reactor/start.sls
+EOF
+
+mkdir -p /srv/salt/nginx
+echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd
+
+# Install Salt
+#
+# We specify -X to avoid a race condition that can cause minion failure to
+# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
+#
+# -M installs the master
+if [ ! -x /etc/init.d/salt-master ]; then
+  wget -q -O - https://bootstrap.saltstack.com | sh -s -- -M -X
+else
+  /etc/init.d/salt-master restart
+  /etc/init.d/salt-minion restart
+fi
+
+echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd

cluster/vsphere/templates/salt-minion.sh

+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use other Debian mirror
+sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
+
+# Resolve hostname of master
+if ! grep -q $MASTER_NAME /etc/hosts; then
+  echo "Adding host entry for $MASTER_NAME"
+  echo "$MASTER_IP $MASTER_NAME" >> /etc/hosts
+fi
+
+# Prepopulate the name of the Master
+mkdir -p /etc/salt/minion.d
+echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
+
+# Turn on debugging for salt-minion
+# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion
+
+# Our minions will have a pool role to distinguish them from the master.
+#
+# Setting the "minion_ip" here causes the kubelet to use its IP for
+# identification instead of its hostname.
+#
+cat <<EOF >/etc/salt/minion.d/grains.conf
+grains:
+  minion_ip: $(ip route get 1.1.1.1 | awk '{print $7}')
+  roles:
+    - kubernetes-pool
+    - kubernetes-pool-vsphere
+  cbr-cidr: $MINION_IP_RANGE
+EOF
+
+# Install Salt
+#
+# We specify -X to avoid a race condition that can cause minion failure to
+# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
+if [ ! -x /etc/init.d/salt-minion ]; then
+  wget -q -O - https://bootstrap.saltstack.com | sh -s -- -X
+else
+  /etc/init.d/salt-minion restart
+fi

cluster/vsphere/util.sh

+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A library of helper functions and constants for the local config.
+
+# Use the config file specified in $KUBE_CONFIG_FILE, or default to
+# config-default.sh.
+source $(dirname ${BASH_SOURCE})/${KUBE_CONFIG_FILE-"config-default.sh"}
+
+function detect-master {
+  KUBE_MASTER=${MASTER_NAME}
+  if [ -z "$KUBE_MASTER_IP" ]; then
+    KUBE_MASTER_IP=$(govc vm.ip ${MASTER_NAME})
+  fi
+  if [ -z "$KUBE_MASTER_IP" ]; then
+    echo "Could not detect Kubernetes master node.  Make sure you've launched a cluster with 'kube-up.sh'"
+    exit 1
+  fi
+  echo "Found ${KUBE_MASTER} at ${KUBE_MASTER_IP}"
+}
+
+function detect-minions {
+  KUBE_MINION_IP_ADDRESSES=()
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+    local minion_ip=$(govc vm.ip ${MINION_NAMES[$i]})
+    echo "Found ${MINION_NAMES[$i]} at ${minion_ip}"
+    KUBE_MINION_IP_ADDRESSES+=("${minion_ip}")
+  done
+  if [ -z "$KUBE_MINION_IP_ADDRESSES" ]; then
+    echo "Could not detect Kubernetes minion nodes.  Make sure you've launched a cluster with 'kube-up.sh'"
+    exit 1
+  fi
+}
+
+# Verify prereqs on host machine
+function verify-prereqs {
+  if [ "$(which govc)" == "" ]; then
+    echo "Can't find govc in PATH, please install and retry."
+    echo ""
+    echo "    go install github.com/vmware/govmomi/govc"
+    echo ""
+    exit 1
+  fi
+}
+
+# Run command over ssh
+function kube-ssh {
+  local host=$1
+  shift
+  ssh ${SSH_OPTS} kube@${host} "$*" 2> /dev/null
+}
+
+# Instantiate a generic kubernetes virtual machine (master or minion)
+function kube-up-vm {
+  local vm_name=$1
+  local vm_memory=$2
+  local vm_cpu=$3
+  local vm_ip=
+
+  govc vm.create \
+    -debug \
+    -m ${vm_memory} \
+    -c ${vm_cpu} \
+    -disk ${DISK} \
+    -g ${GUEST_ID} \
+    -link=true \
+    ${vm_name}
+
+  # Retrieve IP first, to confirm the guest operations agent is running.
+  vm_ip=$(govc vm.ip ${vm_name})
+
+  govc guest.mkdir \
+    -vm ${vm_name} \
+    -p \
+    /home/kube/.ssh
+
+  govc guest.upload \
+    -vm ${vm_name} \
+    -f \
+    ${PUBLIC_KEY_FILE} \
+    /home/kube/.ssh/authorized_keys
+}
+
+# Instantiate a kubernetes cluster
+function kube-up {
+  # Build up start up script for master
+  KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
+  trap "rm -rf ${KUBE_TEMP}" EXIT
+
+  get-password
+  echo "Using password: $user:$passwd"
+  echo
+  python $(dirname $0)/../third_party/htpasswd/htpasswd.py -b -c ${KUBE_TEMP}/htpasswd $user $passwd
+  HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd)
+
+  echo "Starting master VM (this can take a minute)..."
+
+  kube-up-vm ${MASTER_NAME} ${MASTER_MEMORY_MB-1024} ${MASTER_CPU-1}
+
+  # Prints master IP, so user can log in for debugging.
+  detect-master
+  echo
+
+  echo "Starting minion VMs (this can take a minute)..."
+
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+    (
+      echo "#! /bin/bash"
+      echo "MY_NAME=${MINION_NAMES[$i]}"
+      grep -v "^#" $(dirname $0)/vsphere/templates/hostname.sh
+      echo "MASTER_NAME=${MASTER_NAME}"
+      echo "MASTER_IP=${KUBE_MASTER_IP}"
+      echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}"
+      grep -v "^#" $(dirname $0)/vsphere/templates/salt-minion.sh
+    ) > ${KUBE_TEMP}/minion-start-${i}.sh
+
+    (
+      kube-up-vm ${MINION_NAMES[$i]} ${MINION_MEMORY_MB-1024} ${MINION_CPU-1}
+
+      MINION_IP=$(govc vm.ip ${MINION_NAMES[$i]})
+
+      govc guest.upload \
+        -vm ${MINION_NAMES[$i]} \
+        -perm 0700 \
+        -f \
+        ${KUBE_TEMP}/minion-start-${i}.sh \
+        /home/kube/minion-start.sh
+
+      # Kickstart start script
+      kube-ssh ${MINION_IP} "nohup sudo ~/minion-start.sh < /dev/null 1> minion-start.out 2> minion-start.err &"
+    ) &
+  done
+
+  FAIL=0
+  for job in `jobs -p`
+  do
+      wait $job || let "FAIL+=1"
+  done
+  if (( $FAIL != 0 )); then
+    echo "${FAIL} commands failed.  Exiting."
+    exit 2
+  fi
+
+  # Print minion IPs, so user can log in for debugging.
+  detect-minions
+  echo
+
+  # Continue provisioning the master.
+
+  (
+    echo "#! /bin/bash"
+    echo "MY_NAME=${MASTER_NAME}"
+    grep -v "^#" $(dirname $0)/vsphere/templates/hostname.sh
+    echo "MASTER_NAME=${MASTER_NAME}"
+    echo "MASTER_HTPASSWD='${HTPASSWD}'"
+    grep -v "^#" $(dirname $0)/vsphere/templates/install-release.sh
+    grep -v "^#" $(dirname $0)/vsphere/templates/salt-master.sh
+  ) > ${KUBE_TEMP}/master-start.sh
+
+  govc guest.upload \
+    -vm ${MASTER_NAME} \
+    -perm 0700 \
+    -f \
+    ${KUBE_TEMP}/master-start.sh \
+    /home/kube/master-start.sh
+
+  govc guest.upload \
+    -vm ${MASTER_NAME} \
+    -f \
+    ./output/release/master-release.tgz \
+    /home/kube/master-release.tgz
+
+  # Kickstart start script
+  kube-ssh ${KUBE_MASTER_IP} "nohup sudo ~/master-start.sh < /dev/null 1> master-start.out 2> master-start.err &"
+
+  echo "Waiting for cluster initialization."
+  echo
+  echo "  This will continually check to see if the API for kubernetes is reachable."
+  echo "  This might loop forever if there was some uncaught error during start up."
+  echo
+
+  until $(curl --insecure --user ${user}:${passwd} --max-time 5 \
+          --fail --output /dev/null --silent https://${KUBE_MASTER_IP}/api/v1beta1/pods); do
+      printf "."
+      sleep 2
+  done
+
+  echo "Kubernetes cluster created."
+  echo
+
+  echo "Sanity checking cluster..."
+
+  sleep 5
+
+  # Don't bail on errors, we want to be able to print some info.
+  set +e
+
+  # Basic sanity checking
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+      # Make sure docker is installed
+      kube-ssh ${KUBE_MINION_IP_ADDRESSES[$i]} which docker > /dev/null
+      if [ "$?" != "0" ]; then
+          echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
+          echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)"
+          exit 1
+      fi
+  done
+
+  echo
+  echo "Kubernetes cluster is running. Access the master at:"
+  echo
+  echo "  https://${user}:${passwd}@${KUBE_MASTER_IP}"
+  echo
+  echo "Security note: The server above uses a self signed certificate."
+  echo "This is subject to \"Man in the middle\" type attacks."
+  echo
+}
+
+# Delete a kubernetes cluster
+function kube-down {
+  govc vm.destroy ${MASTER_NAME} &
+
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
+    govc vm.destroy ${MINION_NAMES[i]} &
+  done
+
+  wait
+
+}
+
+# Update a kubernetes cluster with latest source
+function kube-push {
+  detect-master
+
+  govc guest.upload \
+    -vm ${MASTER_NAME} \
+    -f \
+    ./output/release/master-release.tgz \
+    /home/kube/master-release.tgz
+
+  (
+    grep -v "^#" $(dirname $0)/vsphere/templates/install-release.sh
+    echo "echo Executing configuration"
+    echo "sudo salt '*' mine.update"
+    echo "sudo salt --force-color '*' state.highstate"
+  ) | kube-ssh ${KUBE_MASTER_IP} bash
+
+  get-password
+
+  echo "Kubernetes cluster is updated.  Access the master at:"
+  echo
+  echo "  https://${user}:${passwd}@${KUBE_MASTER_IP}"
+  echo
+}
+
+# Execute prior to running tests to build a release if required for env
+function test-build-release {
+	echo "TODO"
+}
+
+# Execute prior to running tests to initialize required structure
+function test-setup {
+	echo "TODO"
+}
+
+# Execute after running tests to perform any required clean-up
+function test-teardown {
+	echo "TODO"
+}
+
+# Set the {user} and {password} environment values required to interact with provider
+function get-password {
+  file=${HOME}/.kubernetes_auth
+  if [ -e ${file} ]; then
+    user=$(cat $file | python -c 'import json,sys;print(json.load(sys.stdin)["User"])')
+    passwd=$(cat $file | python -c 'import json,sys;print(json.load(sys.stdin)["Password"])')
+    return
+  fi
+  user=admin
+  passwd=$(python -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))')
+
+  # Store password for reuse.
+  cat << EOF > ~/.kubernetes_auth
+{
+  "User": "$user",
+  "Password": "$passwd"
+}
+EOF
+  chmod 0600 ~/.kubernetes_auth
+}

docs/getting-started-guides/vsphere.md

+## Getting started with vSphere
+
+### Prerequisites
+
+1. You need administrator credentials to an ESXi machine or vCenter instance.
+2. You must have Go (version 1.2 or later) installed: [www.golang.org](http://www.golang.org).
+3. You must have your `GOPATH` set up and include `$GOPATH/bin` in your `PATH`.
+
+   ```sh
+   export GOPATH=$HOME/src/go
+   mkdir -p $GOPATH
+   export PATH=$PATH:$GOPATH/bin
+   ```
+
+4. Install the govc tool to interact with ESXi/vCenter:
+
+   ```sh
+   go get github.com/vmware/govmomi/govc
+   ```
+
+5. Install godep:
+
+   ```sh
+   go get github.com/tools/godep
+   ```
+
+6. Get the Kubernetes source:
+
+   ```sh
+   go get github.com/GoogleCloudPlatform/kubernetes
+   cd $GOPATH/src/github.com/GoogleCloudPlatform/kubernetes
+   ```
+
+### Setup
+
+Download a prebuilt Debian VMDK to be used as base image:
+
+```sh
+wget https://storage.googleapis.com/govmomi/vmdk/kube.vmdk.gz{,.md5}
+md5sum -c kube.vmdk.gz.md5
+gzip -d kube.vmdk.gz
+```
+
+Upload this VMDK to your vSphere instance:
+
+```sh
+export GOVC_URL='https://user:pass@hostname/sdk'
+export GOVC_DATASTORE='target datastore'
+export GOVC_RESOURCE_POOL='resource pool with access to datastore'
+
+govc datastore.import kube.vmdk
+```
+
+Verify that the VMDK was correctly uploaded and expanded to 10GiB:
+
+```sh
+govc datastore.ls
+```
+
+Take a look at the file `cluster/vsphere/config-common.sh` fill in the required
+parameters. The guest login for the image that you imported is `kube:kube`.
+
+Now, let's continue with deploying Kubernetes:
+
+```sh
+cd kubernetes
+
+# Build a release
+release/build-release.sh
+
+# Deploy Kubernetes (takes ~5 minutes, provided everything works out)
+export KUBERNETES_PROVIDER=vsphere
+cluster/kube-up.sh
+```
+
+Refer to the top level README and the getting started guide for Google Compute
+Engine. Once you have successfully reached this point, your vSphere Kubernetes
+deployment works just as any other one!
+
+**Enjoy!**