Add support for running clusters on GCI

build/common.sh

@@ -885,35 +885,36 @@ function kube::release::package_salt_tarball() {
 }
 
 # This will pack kube-system manifests files for distros without using salt
-# such as Ubuntu Trusty. For Trusty, we directly copy manifests from cluster/addons
-# and cluster/saltbase/salt. The script of cluster initialization will remove
-# the salt configuration and evaluate the variables in the manifests.
+# such as GCI and Ubuntu Trusty. We directly copy manifests from
+# cluster/addons and cluster/saltbase/salt. The script of cluster initialization
+# will remove the salt configuration and evaluate the variables in the manifests.
 function kube::release::package_kube_manifests_tarball() {
   kube::log::status "Building tarball: manifests"
 
   local release_stage="${RELEASE_STAGE}/manifests/kubernetes"
   rm -rf "${release_stage}"
-  mkdir -p "${release_stage}/trusty"
+  local dst_dir="${release_stage}/gci-trusty"
+  mkdir -p "${dst_dir}"
 
   local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
   cp "${salt_dir}/fluentd-es/fluentd-es.yaml" "${release_stage}/"
   cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/"
   cp "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${release_stage}/"
   cp "${salt_dir}/kube-proxy/kube-proxy.manifest" "${release_stage}/"
-  cp "${salt_dir}/etcd/etcd.manifest" "${release_stage}/trusty"
-  cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${release_stage}/trusty"
-  cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${release_stage}/trusty"
-  cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${release_stage}/trusty"
-  cp "${salt_dir}/kube-addons/namespace.yaml" "${release_stage}/trusty"
-  cp "${salt_dir}/kube-addons/kube-addons.sh" "${release_stage}/trusty"
-  cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${release_stage}/trusty"
-  cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${release_stage}/trusty"
-  cp -r "${salt_dir}/kube-admission-controls/limit-range" "${release_stage}/trusty"
+  cp "${salt_dir}/etcd/etcd.manifest" "${dst_dir}"
+  cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${dst_dir}"
+  cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${dst_dir}"
+  cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${dst_dir}"
+  cp "${salt_dir}/kube-addons/namespace.yaml" "${dst_dir}"
+  cp "${salt_dir}/kube-addons/kube-addons.sh" "${dst_dir}"
+  cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${dst_dir}"
+  cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}"
+  cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}"
   local objects
   objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
-  tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${release_stage}/trusty"
+  tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${dst_dir}"
 
-  # This is for coreos only. ContainerVM or Trusty does not use it.
+  # This is for coreos only. ContainerVM, GCI, or Trusty does not use it.
   cp -r "${KUBE_ROOT}/cluster/gce/coreos/kube-manifests"/* "${release_stage}/"
 
   kube::release::clean_cruft
@@ -1104,8 +1105,9 @@ function kube::release::gcs::copy_release_artifacts() {
   # Stage everything in release directory
   kube::release::gcs::stage_and_hash "${RELEASE_DIR}"/* . || return 1
 
-  # Having the configure-vm.sh script and and trusty code from the GCE cluster
+  # Having the configure-vm.sh script and trusty code from the GCE cluster
   # deploy hosted with the release is useful for GKE.
+  # TODO(andyzheng0831): Replace the trusty path with GCI after finshing the GCI code.
   kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/configure-vm.sh" extra/gce || return 1
   kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/trusty/node.yaml" extra/gce || return 1
   kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/trusty/master.yaml" extra/gce || return 1

cluster/common.sh

@@ -335,9 +335,9 @@ function find-release-tars() {
     exit 1
   fi
 
-  # This tarball is only used by Ubuntu Trusty.
+  # This tarball is used by GCI, Ubuntu Trusty, and CoreOS.
   KUBE_MANIFESTS_TAR=
-  if [[ "${KUBE_OS_DISTRIBUTION:-}" == "trusty" || "${KUBE_OS_DISTRIBUTION:-}" == "coreos" ]]; then
+  if [[ "${KUBE_OS_DISTRIBUTION:-}" == "trusty" || "${KUBE_OS_DISTRIBUTION:-}" == "gci" || "${KUBE_OS_DISTRIBUTION:-}" == "coreos" ]]; then
     KUBE_MANIFESTS_TAR="${KUBE_ROOT}/server/kubernetes-manifests.tar.gz"
     if [[ ! -f "${KUBE_MANIFESTS_TAR}" ]]; then
       KUBE_MANIFESTS_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-manifests.tar.gz"
@@ -519,7 +519,7 @@ EOF
 TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD})
 EOF
   fi
-  if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then
+  if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" ]]; then
     cat >>$file <<EOF
 KUBE_MANIFESTS_TAR_URL: $(yaml-quote ${KUBE_MANIFESTS_TAR_URL})
 KUBE_MANIFESTS_TAR_HASH: $(yaml-quote ${KUBE_MANIFESTS_TAR_HASH})

cluster/gce/config-test.sh

@@ -137,7 +137,7 @@ KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false}
 # Optional: setting it to true denotes this is a testing cluster,
 # so that we can use pulled kubernetes binaries, even if binaries
 # are pre-installed in the image. Note that currently this logic
-# is only supported in trusty nodes.
+# is only supported in trusty or GCI.
 TEST_CLUSTER="${TEST_CLUSTER:-true}"
 
 # OpenContrail networking plugin specific settings

cluster/gce/trusty/configure-helper.sh

@@ -357,7 +357,7 @@ create_master_kubelet_auth() {
 # $5: pod name, which should be either etcd or etcd-events
 prepare_etcd_manifest() {
   etcd_temp_file="/tmp/$5"
-  cp /home/kubernetes/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}"
+  cp /home/kubernetes/kube-manifests/kubernetes/gci-trusty/etcd.manifest "${etcd_temp_file}"
   sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}"
   sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}"
   sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}"
@@ -462,7 +462,7 @@ start_kube_apiserver() {
   fi
   readonly kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
 
-  src_file="/home/kubernetes/kube-manifests/kubernetes/trusty/kube-apiserver.manifest"
+  src_file="/home/kubernetes/kube-manifests/kubernetes/gci-trusty/kube-apiserver.manifest"
   remove_salt_config_comments "${src_file}"
   # Evaluate variables
   sed -i -e "s@{{params}}@${params}@g" "${src_file}"
@@ -521,7 +521,7 @@ start_kube_controller_manager() {
   fi
   readonly kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
 
-  src_file="/home/kubernetes/kube-manifests/kubernetes/trusty/kube-controller-manager.manifest"
+  src_file="/home/kubernetes/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest"
   remove_salt_config_comments "${src_file}"
   # Evaluate variables
   sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
@@ -561,7 +561,7 @@ start_kube_scheduler() {
   readonly kube_scheduler_docker_tag=$(cat "${kube_home}/kube-docker-files/kube-scheduler.docker_tag")
 
   # Remove salt comments and replace variables with values
-  src_file="${kube_home}/kube-manifests/kubernetes/trusty/kube-scheduler.manifest"
+  src_file="${kube_home}/kube-manifests/kubernetes/gci-trusty/kube-scheduler.manifest"
   remove_salt_config_comments "${src_file}"
   sed -i -e "s@{{params}}@${params}@g" "${src_file}"
   sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
@@ -585,7 +585,7 @@ start_fluentd() {
 # $1: addon category under /etc/kubernetes
 # $2: manifest source dir
 setup_addon_manifests() {
-  src_dir="/home/kubernetes/kube-manifests/kubernetes/trusty/$2"
+  src_dir="/home/kubernetes/kube-manifests/kubernetes/gci-trusty/$2"
   dst_dir="/etc/kubernetes/$1/$2"
   if [ ! -d "${dst_dir}" ]; then
     mkdir -p "${dst_dir}"
@@ -609,7 +609,7 @@ setup_addon_manifests() {
 
 # Prepares the manifests of k8s addons static pods.
 prepare_kube_addons() {
-  addon_src_dir="/home/kubernetes/kube-manifests/kubernetes/trusty"
+  addon_src_dir="/home/kubernetes/kube-manifests/kubernetes/gci-trusty"
   addon_dst_dir="/etc/kubernetes/addons"
   # Set up manifests of other addons.
   if [ "${ENABLE_CLUSTER_MONITORING:-}" = "influxdb" ] || \

cluster/gce/trusty/configure.sh

@@ -157,7 +157,7 @@ install_kube_binary_config() {
     find "${dst_dir}" -name \*.manifest -or -name \*.json | \
       xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
   fi
-  cp "${dst_dir}/kubernetes/trusty/configure-helper.sh" /etc/kube-configure-helper.sh
+  cp "${dst_dir}/kubernetes/gci-trusty/configure-helper.sh" /etc/kube-configure-helper.sh
 
   # Clean up.
   rm -rf "${kube_home}/kubernetes"

cluster/gce/util.sh

@@ -25,6 +25,18 @@ source "${KUBE_ROOT}/cluster/lib/util.sh"
 
 if [[ "${OS_DISTRIBUTION}" == "debian" || "${OS_DISTRIBUTION}" == "coreos" || "${OS_DISTRIBUTION}" == "trusty" ]]; then
   source "${KUBE_ROOT}/cluster/gce/${OS_DISTRIBUTION}/helper.sh"
+elif [[ "${OS_DISTRIBUTION}" == "gci" ]]; then
+  # TODO(andyzheng0831): Switch to use the GCI specific code.
+  source "${KUBE_ROOT}/cluster/gce/trusty/helper.sh"
+  MASTER_IMAGE_PROJECT="google-containers"
+  # If choosing "gci" disto, at least the cluster master needs to run on GCI image.
+  # If the user does not set a GCI image for master, we run both master and nodes
+  # using the latest GCI dev image.
+  if [[ "${MASTER_IMAGE}" != gci* ]]; then
+    MASTER_IMAGE=$(gcloud compute images list | grep "gci-dev" | cut -d ' ' -f1)
+    NODE_IMAGE="${MASTER_IMAGE}"
+    NODE_IMAGE_PROJECT="${MASTER_IMAGE_PROJECT}"
+  fi
 else
   echo "Cannot operate on cluster using os distro: ${OS_DISTRIBUTION}" >&2
   exit 1
@@ -169,8 +181,6 @@ function set-preferred-region() {
 
 # Take the local tar files and upload them to Google Storage.  They will then be
 # downloaded by the master as part of the start up script for the master.
-# If running on Ubuntu trusty, we also pack the dir cluster/gce/trusty/kube-manifest
-# and upload it to Google Storage.
 #
 # Assumed vars:
 #   PROJECT
@@ -208,7 +218,7 @@ function upload-server-tars() {
 
   SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}")
   SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}")
-  if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
+  if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
     KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}")
   fi
 
@@ -241,7 +251,7 @@ function upload-server-tars() {
     server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
     salt_tar_urls+=("${salt_gs_url/gs:\/\//https://storage.googleapis.com/}")
 
-    if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
+    if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
       local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
       copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
       # Convert from gs:// URL to an https:// URL
@@ -257,7 +267,7 @@ function upload-server-tars() {
   else
     SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}")
     SALT_TAR_URL=$(join_csv "${salt_tar_urls[@]}")
-    if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then
+    if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" ]]; then
       KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}")
     fi
   fi
@@ -695,8 +705,8 @@ function create-nodes-template() {
 
   local template_name="${NODE_INSTANCE_PREFIX}-template"
 
-  # For master on trusty, we support running nodes on ContainerVM or trusty.
-  if [[ "${OS_DISTRIBUTION}" == "trusty" ]] && \
+  # For master on GCI or trusty, we support the hybrid mode with nodes on ContainerVM.
+  if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" ]] && \
      [[ "${NODE_IMAGE}" == container* ]]; then
     source "${KUBE_ROOT}/cluster/gce/debian/helper.sh"
   fi

test/e2e/addon_update.go

@@ -214,7 +214,7 @@ var _ = framework.KubeDescribe("Addon update", func() {
 		switch framework.TestContext.OSDistro {
 		case "debian":
 			sshExecAndVerify(sshClient, "sudo TEST_ADDON_CHECK_INTERVAL_SEC=1 /etc/init.d/kube-addons restart")
-		case "trusty":
+		case "trusty", "gci":
 			sshExecAndVerify(sshClient, "sudo initctl restart kube-addons TEST_ADDON_CHECK_INTERVAL_SEC=1")
 		case "coreos":
 			sshExecAndVerify(sshClient, "sudo systemctl set-environment TEST_ADDON_CHECK_INTERVAL_SEC=1")
@@ -230,7 +230,7 @@ var _ = framework.KubeDescribe("Addon update", func() {
 			switch framework.TestContext.OSDistro {
 			case "debian":
 				sshExec(sshClient, "sudo /etc/init.d/kube-addons restart")
-			case "trusty":
+			case "trusty", "gci":
 				sshExec(sshClient, "sudo initctl restart kube-addons")
 			case "coreos":
 				sshExec(sshClient, "sudo systemctl unset-environment TEST_ADDON_CHECK_INTERVAL_SEC")