remove trusty GCE kube-up.sh

build/lib/release.sh

@@ -378,7 +378,6 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${salt_dir}/l7-gcp/glbc.manifest" "${gci_dst_dir}"
   cp "${salt_dir}/rescheduler/rescheduler.manifest" "${gci_dst_dir}/"
   cp "${salt_dir}/e2e-image-puller/e2e-image-puller.manifest" "${gci_dst_dir}/"
-  cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${gci_dst_dir}/trusty-configure-helper.sh"
   cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${gci_dst_dir}/gci-configure-helper.sh"
   cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${gci_dst_dir}/gci-mounter"
   cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${gci_dst_dir}/health-monitor.sh"

cluster/gce/BUILD

@@ -9,7 +9,6 @@ pkg_tar(
         "gci/configure-helper.sh",
         "gci/health-monitor.sh",
         "gci/mounter/mounter",
-        "trusty/configure-helper.sh",
     ],
     mode = "0755",
     strip_prefix = ".",

cluster/gce/trusty/configure.sh

-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script contains functions for configuring instances to run kubernetes
-# master and nodes. It is uploaded as GCE instance metadata. The upstart jobs
-# in cluster/gce/trusty/<node.yaml, master.yaml> download it and make use
-# of needed functions. The script itself is not supposed to be executed in
-# other manners.
-
-set_broken_motd() {
-  cat > /etc/motd <<EOF
-Broken (or in progress) Kubernetes node setup! If you are using Ubuntu Trusty,
-check log file /var/log/syslog. If you are using GCI image, use
-"journalctl | grep kube" to find more information.
-EOF
-}
-
-download_kube_env() {
-  # Fetch kube-env from GCE metadata server.
-  readonly tmp_kube_env="/tmp/kube-env.yaml"
-  curl --fail --retry 5 --retry-delay 3 --silent --show-error \
-    -H "X-Google-Metadata-Request: True" \
-    -o "${tmp_kube_env}" \
-    http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
-  # Convert the yaml format file into a shell-style file.
-  eval $(python -c '''
-import pipes,sys,yaml
-for k,v in yaml.load(sys.stdin).iteritems():
-  print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
-''' < "${tmp_kube_env}" > /etc/kube-env)
-  rm -f "${tmp_kube_env}"
-}
-
-validate_hash() {
-  file="$1"
-  expected="$2"
-
-  actual=$(sha1sum ${file} | awk '{ print $1 }') || true
-  if [ "${actual}" != "${expected}" ]; then
-    echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
-    return 1
-  fi
-}
-
-# Retry a download until we get it. Takes a hash and a set of URLs.
-#
-# $1: The sha1 of the URL. Can be "" if the sha1 is unknown, which means
-#     we are downloading a hash file.
-# $2: The temp file containing a list of urls to download.
-download_or_bust() {
-  file_hash="$1"
-  tmpfile_urls="$2"
-
-  while true; do
-    # Read urls from the file one-by-one.
-    while read -r url; do
-      if [ ! -n "${file_hash}" ]; then
-        url="${url/.tar.gz/.tar.gz.sha1}"
-      fi
-      file="${url##*/}"
-      rm -f "${file}"
-      if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
-        echo "== Failed to download ${url}. Retrying. =="
-      elif [ -n "${file_hash}" ] && ! validate_hash "${file}" "${file_hash}"; then
-        echo "== Hash validation of ${url} failed. Retrying. =="
-      else
-        if [ -n "${file_hash}" ]; then
-          echo "== Downloaded ${url} (SHA1 = ${file_hash}) =="
-        else
-          echo "== Downloaded ${url} =="
-        fi
-        return
-      fi
-    done < "${tmpfile_urls}"
-  done
-}
-
-# Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
-# and places them into suitable directories. Files are placed in /home/kubernetes. 
-install_kube_binary_config() {
-  # Upstart does not support shell array well. Put urls in a temp file with one
-  # url at a line, and we will use 'read' command to get them one-by-one.
-  tmp_binary_urls=$(mktemp /tmp/kube-temp.XXXXXX)
-  echo "${SERVER_BINARY_TAR_URL}" | tr "," "\n" > "${tmp_binary_urls}"
-  tmp_manifests_urls=$(mktemp /tmp/kube-temp.XXXXXX)
-  echo "${KUBE_MANIFESTS_TAR_URL}" | tr "," "\n" > "${tmp_manifests_urls}"
-
-  kube_home="/home/kubernetes"
-  mkdir -p "${kube_home}"
-  cd "${kube_home}"
-  read -r server_binary_tar_url < "${tmp_binary_urls}"
-  readonly server_binary_tar="${server_binary_tar_url##*/}"
-  if [ -n "${SERVER_BINARY_TAR_HASH:-}" ]; then
-    readonly server_binary_tar_hash="${SERVER_BINARY_TAR_HASH}"
-  else
-    echo "Downloading binary release sha1 (not found in env)"
-    download_or_bust "" "${tmp_binary_urls}"
-    readonly server_binary_tar_hash=$(cat "${server_binary_tar}.sha1")
-  fi
-  echo "Downloading binary release tar"
-  download_or_bust "${server_binary_tar_hash}" "${tmp_binary_urls}"
-  tar xzf "${kube_home}/${server_binary_tar}" -C "${kube_home}" --overwrite
-  # Copy docker_tag and image files to /home/kubernetes/kube-docker-files.
-  src_dir="${kube_home}/kubernetes/server/bin"
-  dst_dir="${kube_home}/kube-docker-files"
-  mkdir -p "${dst_dir}"
-  cp "${src_dir}/"*.docker_tag "${dst_dir}"
-  if [ "${KUBERNETES_MASTER:-}" = "false" ]; then
-    cp "${src_dir}/kube-proxy.tar" "${dst_dir}"
-  else
-    cp "${src_dir}/kube-apiserver.tar" "${dst_dir}"
-    cp "${src_dir}/kube-controller-manager.tar" "${dst_dir}"
-    cp "${src_dir}/kube-scheduler.tar" "${dst_dir}"
-    cp -r "${kube_home}/kubernetes/addons" "${dst_dir}"
-  fi
-  # Use the binary from the release tarball if they are not preinstalled, or if this is
-  # a test cluster.
-  readonly BIN_PATH="/usr/bin"
-  if ! which kubelet > /dev/null || ! which kubectl > /dev/null; then
-    # This should be the case of trusty.
-    cp "${src_dir}/kubelet" "${BIN_PATH}"
-    cp "${src_dir}/kubectl" "${BIN_PATH}"
-  else
-    # This should be the case of GCI.
-    readonly kube_bin="${kube_home}/bin"
-    mkdir -p "${kube_bin}"
-    mount -o remount,rw,exec "${kube_bin}"
-    cp "${src_dir}/kubelet" "${kube_bin}"
-    cp "${src_dir}/kubectl" "${kube_bin}"
-    chmod 544 "${kube_bin}/kubelet"
-    chmod 544 "${kube_bin}/kubectl"
-    # If the built-in binary version is different from the expected version, we use
-    # the downloaded binary. The simplest implementation is to always use the downloaded
-    # binary without checking the version. But we have another version guardian in GKE.
-    # So, we compare the versions to ensure this run-time binary replacement is only
-    # applied for OSS kubernetes.
-    readonly builtin_version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")"
-    readonly required_version="$(/home/kubernetes/bin/kubelet --version=true | cut -f2 -d " ")"
-    if [ "${TEST_CLUSTER:-}" = "true" ] || [ "${builtin_version}" != "${required_version}" ]; then
-      mount --bind "${kube_bin}/kubelet" "${BIN_PATH}/kubelet"
-      mount --bind "${kube_bin}/kubectl" "${BIN_PATH}/kubectl"
-    else
-      # Remove downloaded binary just to prevent misuse.
-      rm -f "${kube_bin}/kubelet"
-      rm -f "${kube_bin}/kubectl"
-    fi
-  fi
-  cp "${kube_home}/kubernetes/LICENSES" "${kube_home}"
-
-  # Put kube-system pods manifests in /home/kubernetes/kube-manifests/.
-  dst_dir="${kube_home}/kube-manifests"
-  mkdir -p "${dst_dir}"
-  read -r manifests_tar_url < "${tmp_manifests_urls}"
-  readonly manifests_tar="${manifests_tar_url##*/}"
-  if [ -n "${KUBE_MANIFESTS_TAR_HASH:-}" ]; then
-    readonly manifests_tar_hash="${KUBE_MANIFESTS_TAR_HASH}"
-  else
-    echo "Downloading k8s manifests sha1 (not found in env)"
-    download_or_bust "" "${tmp_manifests_urls}"
-    readonly manifests_tar_hash=$(cat "${manifests_tar}.sha1")
-  fi
-  echo "Downloading k8s manifests tar"
-  download_or_bust "${manifests_tar_hash}" "${tmp_manifests_urls}"
-  tar xzf "${kube_home}/${manifests_tar}" -C "${dst_dir}" --overwrite
-  readonly kube_addon_registry="${KUBE_ADDON_REGISTRY:-gcr.io/google_containers}"
-  if [ "${kube_addon_registry}" != "gcr.io/google_containers" ]; then
-    find "${dst_dir}" -name \*.yaml -or -name \*.yaml.in | \
-      xargs sed -ri "s@(image:\s.*)gcr.io/google_containers@\1${kube_addon_registry}@"
-    find "${dst_dir}" -name \*.manifest -or -name \*.json | \
-      xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
-  fi
-  cp "${dst_dir}/kubernetes/gci-trusty/trusty-configure-helper.sh" /etc/kube-configure-helper.sh
-
-  # Clean up.
-  rm -rf "${kube_home}/kubernetes"
-  rm -f "${kube_home}/${server_binary_tar}"
-  rm -f "${kube_home}/${server_binary_tar}.sha1"
-  rm -f "${kube_home}/${manifests_tar}"
-  rm -f "${kube_home}/${manifests_tar}.sha1"
-  rm -f "${tmp_binary_urls}"
-  rm -f "${tmp_manifests_urls}"
-}

cluster/gce/trusty/helper.sh

-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A library of helper functions and constant for ubuntu os distro
-
-# The configuration is based on upstart, which is in Ubuntu up to 14.04 LTS (Trusty).
-# Ubuntu 15.04 and above replaced upstart with systemd as the init system.
-# Consequently, the configuration cannot work on these images. In release-1.2 branch,
-# GCI and Trusty share the configuration code. We have to keep the GCI specific code
-# here as long as the release-1.2 branch has not been deprecated.
-
-# Creates the GCI specific metadata files if they do not exit.
-# Assumed var
-#   KUBE_TEMP
-function ensure-gci-metadata-files {
-  if [[ ! -f "${KUBE_TEMP}/gci-update.txt" ]]; then
-    cat >"${KUBE_TEMP}/gci-update.txt" << EOF
-update_disabled
-EOF
-  fi
-  if [[ ! -f "${KUBE_TEMP}/gci-docker.txt" ]]; then
-    cat >"${KUBE_TEMP}/gci-docker.txt" << EOF
-true
-EOF
-  fi
-}

cluster/gce/trusty/master-helper.sh

-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A library of helper functions and constant for ubuntu os distro
-
-# The configuration is based on upstart, which is in Ubuntu up to 14.04 LTS (Trusty).
-# Ubuntu 15.04 and above replaced upstart with systemd as the init system.
-# Consequently, the configuration cannot work on these images. In release-1.2 branch,
-# GCI and Trusty share the configuration code. We have to keep the GCI specific code
-# here as long as the release-1.2 branch has not been deprecated.
-
-source ./helper.sh
-
-# create-master-instance creates the master instance. If called with
-# an argument, the argument is used as the name to a reserved IP
-# address for the master. (In the case of upgrade/repair, we re-use
-# the same IP.)
-#
-# It requires a whole slew of assumed variables, partially due to to
-# the call to write-master-env. Listing them would be rather
-# futile. Instead, we list the required calls to ensure any additional
-# variables are set:
-#   ensure-temp-dir
-#   detect-project
-#   get-bearer-token
-#
-function create-master-instance {
-  local address_opt=""
-  [[ -n ${1:-} ]] && address_opt="--address ${1}"
-  local image_metadata=""
-  if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then
-    ensure-gci-metadata-files
-    image_metadata=",gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-docker.txt"
-  fi
-
-  write-master-env
-  gcloud compute instances create "${MASTER_NAME}" \
-    ${address_opt} \
-    --project "${PROJECT}" \
-    --zone "${ZONE}" \
-    --machine-type "${MASTER_SIZE}" \
-    --image-project="${MASTER_IMAGE_PROJECT}" \
-    --image "${MASTER_IMAGE}" \
-    --tags "${MASTER_TAG}" \
-    --network "${NETWORK}" \
-    --scopes "storage-ro,compute-rw,monitoring,logging-write" \
-    --can-ip-forward \
-    --metadata-from-file \
-      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/trusty/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/trusty/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt${image_metadata}" \
-    --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
-    --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}"
-}

cluster/gce/trusty/master.yaml

-From nobody Thu May 13 20:33:00 2016
-Content-Type: multipart/mixed; boundary="===================================="
-MIME-Version: 1.0
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-install-master.conf"
-
-#upstart-job
-
-description "Download and install k8s binaries and configurations"
-
-start on cloud-config
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	# Fetch the script for installing master binary and configuration files.
-	curl --fail --retry 5 --retry-delay 3 --silent --show-error \
-		-H "X-Google-Metadata-Request: True" \
-		-o /etc/kube-configure.sh \
-		http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
-	. /etc/kube-configure.sh
-	set_broken_motd
-	echo "Downloading kube-env file"
-	download_kube_env
-	. /etc/kube-env
-	echo "Install kube master binary and configuration files"
-	install_kube_binary_config
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-env.conf"
-
-#upstart-job
-
-description "Prepare kube master environment"
-
-start on stopped kube-install-master
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	. /etc/kube-env
-	echo "Configuring hostname"
-	config_hostname
-	echo "Configuring IP firewall rules"
-	config_ip_firewall
-	echo "Creating required directories"
-	create_dirs
-	echo "Mount master PD"
-	mount_master_pd
-	echo "Creating kubernetes master auth file"
-	create_master_auth
-	echo "Creating master instance kubelet auth file"
-	create_master_kubelet_auth
-	echo "Creating auth files for etcd"
-	create-master-etcd-auth
-	echo "Assemble kubelet command line"
-	# Kubelet command flags will be written in /etc/default/kubelet
-	assemble_kubelet_flags
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-install-packages.conf"
-
-#upstart-job
-
-description "Install packages needed to run kubernetes"
-
-start on stopped kube-install-master
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	install_critical_packages
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-install-additional-packages.conf"
-
-#upstart-job
-
-description "Install additional packages used by kubernetes"
-
-start on stopped kube-install-packages
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	install_additional_packages
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kubelet.conf"
-
-#upstart-job
-
-description "Run kubelet service"
-
-start on stopped kube-install-packages and stopped kube-env
-
-respawn
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	start_kubelet
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
-# Wait for 10s to start kubelet again.
-post-stop exec sleep 10
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-docker.conf"
-
-#upstart-job
-
-description "Restart docker daemon"
-
-start on started kubelet
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	. /etc/kube-env
-	restart_docker_daemon
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-master-components.conf"
-
-#upstart-job
-
-description "Start kube-master components and addons pods"
-
-start on stopped kube-docker
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	. /etc/kube-env
-	start_etcd_servers
-	start_fluentd_static_pod
-	compute_master_manifest_variables
-	start_kube_apiserver
-	start_kube_controller_manager
-	start_kube_scheduler
-	start_kube_addons
-	start_cluster_autoscaler
-	start_rescheduler
-	reset_motd
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-master-health-monitoring.conf"
-
-#upstart-job
-
-description "Kubernetes master health monitoring"
-
-start on stopped kube-docker
-
-respawn
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	# Wait for a minute to let docker and kubelet processes finish initialization.
-	# TODO(andyzheng0831): replace it with a more reliable method if possible.
-	sleep 60
-	. /etc/kube-configure-helper.sh
-	. /etc/kube-env
-	health_monitoring
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
-# Wait for 10s to start it again.
-post-stop exec sleep 10
-
---====================================--

cluster/gce/trusty/node-helper.sh

-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A library of helper functions and constant for ubuntu os distro
-
-# The configuration is based on upstart, which is in Ubuntu up to 14.04 LTS (Trusty).
-# Ubuntu 15.04 and above replaced upstart with systemd as the init system.
-# Consequently, the configuration cannot work on these images. In release-1.2 branch,
-# GCI and Trusty share the configuration code. We have to keep the GCI specific code
-# here as long as the release-1.2 branch has not been deprecated.
-
-source ./helper.sh
-
-# $1: template name (required)
-function create-node-instance-template {
-  local template_name="$1"
-  ensure-gci-metadata-files
-  create-node-template "$template_name" "${scope_flags[*]}" \
-    "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
-    "user-data=${KUBE_ROOT}/cluster/gce/trusty/node.yaml" \
-    "configure-sh=${KUBE_ROOT}/cluster/gce/trusty/configure.sh" \
-    "cluster-name=${KUBE_TEMP}/cluster-name.txt" \
-    "gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \
-    "gci-ensure-gke-docker=${KUBE_TEMP}/gci-docker.txt"
-}

cluster/gce/trusty/node.yaml

-From nobody Thu May 13 20:33:00 2016
-Content-Type: multipart/mixed; boundary="===================================="
-MIME-Version: 1.0
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-install-node.conf"
-
-#upstart-job
-
-description "Download and install k8s binaries and configurations"
-
-start on cloud-config
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	# Fetch the script for installing nodes binary and configuration files.
-	curl --fail --retry 5 --retry-delay 3 --silent --show-error \
-		-H "X-Google-Metadata-Request: True" \
-		-o /etc/kube-configure.sh \
-		http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
-	. /etc/kube-configure.sh
-	set_broken_motd
-	echo "Downloading kube-env file"
-	download_kube_env
-	. /etc/kube-env
-	echo "Install kube nodes binary and configuration files"
-	install_kube_binary_config
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-env.conf"
-
-#upstart-job
-
-description "Prepare kube node environment"
-
-start on stopped kube-install-node
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	. /etc/kube-env
-	echo "Configuring hostname"
-	config_hostname
-	echo "Configuring IP firewall rules"
-	config_ip_firewall
-	echo "Creating required directories"
-	create_dirs
-	echo "Creating kubelet kubeconfig file"
-	create_kubelet_kubeconfig
-	echo "Creating kube-proxy kubeconfig file"
-	create_kubeproxy_kubeconfig
-	echo "Assemble kubelet command line"
-	# Kubelet command flags will be in /etc/default/kubelet
-	assemble_kubelet_flags
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-install-packages.conf"
-
-#upstart-job
-
-description "Install packages needed to run kubernetes"
-
-start on stopped kube-install-node
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	install_critical_packages
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-install-additional-packages.conf"
-
-#upstart-job
-
-description "Install additional packages used by kubernetes"
-
-start on stopped kube-install-packages
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	install_additional_packages
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kubelet.conf"
-
-#upstart-job
-
-description "Run kubelet service"
-
-start on stopped kube-install-packages and stopped kube-env
-
-respawn
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	start_kubelet
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
-# Wait for 10s to start kubelet again.
-post-stop exec sleep 10
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-docker.conf"
-
-#upstart-job
-
-description "Restart docker daemon"
-
-start on started kubelet
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	. /etc/kube-env
-	restart_docker_daemon
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-proxy.conf"
-
-#upstart-job
-
-description "Start kube-proxy static pod"
-
-start on stopped kube-docker
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	. /etc/kube-env
-	prepare_log_file "/var/log/kube-proxy.log"
-	# Load the docker image from file /home/kubernetes/kube-docker-files/kube-proxy.tar.
-	echo "Try to load docker image file kube-proxy.tar"
-	timeout 30 docker load -i /home/kubernetes/kube-docker-files/kube-proxy.tar
-	# Copy the manifest to /tmp to manipulate
-	tmp_file="/tmp/kube-proxy.manifest"
-	cp -f /home/kubernetes/kube-manifests/kubernetes/kube-proxy.manifest ${tmp_file}
-	# Remove the lines of salt configuration and replace variables with values.
-	# NOTE: Changes to variable names in cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
-	# may break this upstart job.
-	sed -i "/^ *{%/d" ${tmp_file}
-	kubeconfig="--kubeconfig=\/var\/lib\/kube-proxy\/kubeconfig"
-	kube_docker_registry="gcr.io\/google_containers"
-	if [ -n "${KUBE_DOCKER_REGISTRY:-}" ]; then
-		kube_docker_registry=${KUBE_DOCKER_REGISTRY}
-	fi
-	kube_proxy_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-proxy.docker_tag)
-	test_args=""
-	if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then
-		test_args="${KUBEPROXY_TEST_ARGS}"
-	fi
-	log_level="--v=2"
-	if [ -n "${KUBEPROXY_TEST_LOG_LEVEL:-}" ]; then
-		log_level="${KUBEPROXY_TEST_LOG_LEVEL}"
-	fi
-	api_servers="--master=https://${KUBERNETES_MASTER_NAME}"
-	sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${tmp_file}
-	sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${tmp_file}
-	sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${tmp_file}
-	sed -i -e "s@{{test_args}}@${test_args}@g" ${tmp_file}
-	sed -i -e "s@{{ cpurequest }}@100m@g" ${tmp_file}
-	sed -i -e "s@{{log_level}}@${log_level}@g" ${tmp_file}
-	sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${tmp_file}
-	if [ -n "${CLUSTER_IP_RANGE:-}" ]; then
-		sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${tmp_file}
-	fi
-
-	mv -f ${tmp_file} /etc/kubernetes/manifests/
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-addons.conf"
-
-#upstart-job
-
-description "Install kubelet add-on manifest files"
-
-start on stopped kube-docker
-
-script
-{
-	set -o errexit
-	set -o nounset
-
-	. /etc/kube-configure-helper.sh
-	. /etc/kube-env
-	# Kube-registry-proxy
-	if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then
-		cp /home/kubernetes/kube-manifests/kubernetes/kube-registry-proxy.yaml /etc/kubernetes/manifests/
-	fi
-	reset_motd
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
---====================================
-MIME-Version: 1.0
-Content-Type: text/upstart-job; charset="us-ascii"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-node-health-monitoring.conf"
-
-description "Kubernetes node health monitoring"
-
-start on stopped kube-docker
-
-respawn
-
-script
-{
-	set -o nounset
-	set -o errexit
-
-	# Wait for a minute to let docker, kubelet, and kube-proxy processes finish initialization.
-	# TODO(andyzheng0831): replace it with a more reliable method if possible.
-	sleep 60
-	. /etc/kube-configure-helper.sh
-	. /etc/kube-env
-	health_monitoring
-} 2>&1 | logger --priority daemon.info -t ${UPSTART_JOB}
-end script
-
-# Wait for 10s to start it again.
-post-stop exec sleep 10
-
---====================================--