Merge branch 'master' into master

OWNERS_ALIASES

@@ -3,11 +3,13 @@ aliases:
     - davidopp
     - timothysc
     - wojtek-t
+    - k82cn
   sig-scheduling:
     - davidopp
+    - bsalamat
     - timothysc
     - wojtek-t
-    - k82cn 
+    - k82cn
     - jayunit100
   sig-cli-maintainers:
     - adohe
@@ -52,7 +54,7 @@ aliases:
     - pmorie
     - resouer
     - sjpotter
-    - timstclair
+    - tallclair
     - tmrts
     - vishh
     - yifan-gu

api/OWNERS

@@ -17,7 +17,6 @@ reviewers:
 - mikedanese
 - liggitt
 - nikhiljindal
-- bprashanth
 - gmarek
 - erictune
 - davidopp
@@ -33,7 +32,7 @@ reviewers:
 - pwittrock
 - roberthbailey
 - ncdc
-- timstclair
+- tallclair
 - yifan-gu
 - eparis
 - mwielgus

build/BUILD

@@ -21,36 +21,23 @@ filegroup(
     tags = ["automanaged"],
 )
 
-docker_build(
-    name = "busybox",
-    debs = [
-        "@busybox_deb//file",
-    ],
-    symlinks = {
-        "/bin/sh": "/bin/busybox",
-        "/usr/bin/busybox": "/bin/busybox",
-        "/usr/sbin/busybox": "/bin/busybox",
-        "/sbin/busybox": "/bin/busybox",
-    },
-)
-
 # This list should roughly match kube::build::get_docker_wrapped_binaries()
 # in build/common.sh.
 DOCKERIZED_BINARIES = {
     "cloud-controller-manager": {
-        "base": ":busybox",
+        "base": "@official_busybox//image:image.tar",
         "target": "//cmd/cloud-controller-manager:cloud-controller-manager",
     },
     "kube-apiserver": {
-        "base": ":busybox",
+        "base": "@official_busybox//image:image.tar",
         "target": "//cmd/kube-apiserver:kube-apiserver",
     },
     "kube-controller-manager": {
-        "base": ":busybox",
+        "base": "@official_busybox//image:image.tar",
         "target": "//cmd/kube-controller-manager:kube-controller-manager",
     },
     "kube-scheduler": {
-        "base": ":busybox",
+        "base": "@official_busybox//image:image.tar",
         "target": "//plugin/cmd/kube-scheduler:kube-scheduler",
     },
     "kube-proxy": {

build/common.sh

@@ -384,7 +384,13 @@ function kube::build::short_hash() {
 # a workaround for bug https://github.com/docker/docker/issues/3968.
 function kube::build::destroy_container() {
   "${DOCKER[@]}" kill "$1" >/dev/null 2>&1 || true
-  "${DOCKER[@]}" wait "$1" >/dev/null 2>&1 || true
+  if [[ $("${DOCKER[@]}" version --format '{{.Server.Version}}') = 17.06.0* ]]; then
+    # Workaround https://github.com/moby/moby/issues/33948.
+    # TODO: remove when 17.06.0 is not relevant anymore
+    DOCKER_API_VERSION=v1.29 "${DOCKER[@]}" wait "$1" >/dev/null 2>&1 || true
+  else
+    "${DOCKER[@]}" wait "$1" >/dev/null 2>&1 || true
+  fi
   "${DOCKER[@]}" rm -f -v "$1" >/dev/null 2>&1 || true
 }
 

build/root/Makefile

@@ -126,6 +126,21 @@ verify: verify_generated_files
 	KUBE_VERIFY_GIT_BRANCH=$(BRANCH) hack/make-rules/verify.sh -v
 endif
 
+define QUICK_VERIFY_HELP_INFO
+# Runs only the presubmission verifications that aren't slow.
+#
+# Example:
+#   make quick-verify
+endef
+.PHONY: quick-verify
+ifeq ($(PRINT_HELP),y)
+quick-verify:
+	@echo "$$QUICK_VERIFY_HELP_INFO"
+else
+quick-verify: verify_generated_files
+	hack/make-rules/verify.sh -v -Q
+endif
+
 define UPDATE_HELP_INFO
 # Runs all the generated updates.
 #
@@ -240,6 +255,11 @@ define TEST_E2E_NODE_HELP_INFO
 #  IMAGE_SERVICE_ENDPOINT: remote image endpoint to connect to, to prepull images.
 #   Used when RUNTIME is set to "remote".
 #  IMAGE_CONFIG_FILE: path to a file containing image configuration.
+#  SYSTEM_SPEC_NAME: The name of the system spec to be used for validating the
+#    image in the node conformance test. The specs are located at
+#    test/e2e_node/system/specs/. For example, "SYSTEM_SPEC_NAME=gke" will use
+#    the spec at test/e2e_node/system/specs/gke.yaml. If unspecified, the
+#    default built-in spec (system.DefaultSpec) will be used.
 #
 # Example:
 #   make test-e2e-node FOCUS=Kubelet SKIP=container
@@ -524,6 +544,20 @@ bazel-test:
 endif
 
 ifeq ($(PRINT_HELP),y)
+define BAZEL_TEST_INTEGRATION_HELP_INFO
+# Integration test with bazel
+#
+# Example:
+# make bazel-test-integration
+endef
+bazel-test-integration:
+	@echo "$$BAZEL_TEST_INTEGRATION_HELP_INFO"
+else
+bazel-test-integration:
+	bazel test //test/integration/...
+endif
+
+ifeq ($(PRINT_HELP),y)
 define BAZEL_BUILD_HELP_INFO
 # Build release tars with bazel
 #

build/root/WORKSPACE

@@ -12,6 +12,16 @@ http_archive(
     urls = ["https://github.com/kubernetes/repo-infra/archive/9dedd5f4093884c133ad5ea73695b28338b954ab.tar.gz"],
 )
 
+ETCD_VERSION = "3.0.17"
+
+new_http_archive(
+    name = "com_coreos_etcd",
+    build_file = "third_party/etcd.BUILD",
+    sha256 = "274c46a7f8d26f7ae99d6880610f54933cbcf7f3beafa19236c52eb5df8c7a0b",
+    strip_prefix = "etcd-v%s-linux-amd64" % ETCD_VERSION,
+    urls = ["https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-amd64.tar.gz" % (ETCD_VERSION, ETCD_VERSION)],
+)
+
 # This contains a patch to not prepend ./ to tarfiles produced by pkg_tar.
 # When merged upstream, we'll no longer need to use ixdy's fork:
 # https://bazel-review.googlesource.com/#/c/10390/
@@ -24,9 +34,9 @@ http_archive(
 
 http_archive(
     name = "io_bazel_rules_docker",
-    sha256 = "261fbd8fda1d06a12a0479019b46acd302c6aaa8df8e49383dc37917f20492a1",
-    strip_prefix = "rules_docker-52d9faf209ff6d16eb850b6b66d03483735e0633",
-    urls = ["https://github.com/bazelbuild/rules_docker/archive/52d9faf209ff6d16eb850b6b66d03483735e0633.tar.gz"],
+    sha256 = "40d780165c0b9fbb3ddca858df7347381af0e87e430c74863e4ce9d6f6441023",
+    strip_prefix = "rules_docker-8359263f35227a3634ea023ff4ae163189eb4b26",
+    urls = ["https://github.com/bazelbuild/rules_docker/archive/8359263f35227a3634ea023ff4ae163189eb4b26.tar.gz"],
 )
 
 load("@io_bazel_rules_go//go:def.bzl", "go_repositories")
@@ -38,22 +48,6 @@ go_repositories(
 
 docker_repositories()
 
-# for building docker base images
-debs = (
-    (
-        "busybox_deb",
-        "5f81f140777454e71b9e5bfdce9c89993de5ddf4a7295ea1cfda364f8f630947",
-        "http://ftp.us.debian.org/debian/pool/main/b/busybox/busybox-static_1.22.0-19+b3_amd64.deb",
-        "https://storage.googleapis.com/kubernetes-release/debs/busybox-static_1.22.0-19+b3_amd64.deb",
-    ),
-)
-
-[http_file(
-    name = name,
-    sha256 = sha256,
-    url = url,
-) for name, sha256, origin, url in debs]
-
 http_file(
     name = "kubernetes_cni",
     sha256 = "05ab3937bc68562e989dc143362ec4d4275262ba9f359338aed720fc914457a5",
@@ -62,7 +56,16 @@ http_file(
 
 docker_pull(
     name = "debian-iptables-amd64",
-    digest = "sha256:bc20977ac38abfb43071b4c61c4b7edb30af894c05eb06758dd61d05118d2842",  # v7
+    digest = "sha256:bc20977ac38abfb43071b4c61c4b7edb30af894c05eb06758dd61d05118d2842",
     registry = "gcr.io",
     repository = "google-containers/debian-iptables-amd64",
+    tag = "v7",  # ignored, but kept here for documentation
+)
+
+docker_pull(
+    name = "official_busybox",
+    digest = "sha256:be3c11fdba7cfe299214e46edc642e09514dbb9bbefcd0d3836c05a1e0cd0642",
+    registry = "index.docker.io",
+    repository = "library/busybox",
+    tag = "latest",  # ignored, but kept here for documentation
 )

build/visible_to/BUILD

@@ -49,6 +49,7 @@ package_group(
     packages = [
         "//test/e2e",
         "//test/e2e/framework",
+        "//test/e2e/kubectl",
         "//test/e2e/workload",
         "//test/integration/etcd",
         "//test/integration/framework",
@@ -72,7 +73,7 @@ package_group(
     packages = [
         "//cmd/kubeadm/app",
         "//cmd/kubeadm/app/cmd",
-        "//cmd/kubeadm/app/master",
+        "//cmd/kubeadm/app/phases/controlplane",
     ],
 )
 

cluster/addons/cluster-loadbalancing/glbc/README.md

@@ -42,44 +42,43 @@ spec:
 ```
 
 * time, t=0
-```console
-$ kubectl get ing
-NAME           RULE      BACKEND                   ADDRESS
-test-ingress   -         default-http-backend:80
-$ kubectl describe ing
-No events.
-```
+  ```console
+  $ kubectl get ing
+  NAME           RULE      BACKEND                   ADDRESS
+  test-ingress   -         default-http-backend:80
+  $ kubectl describe ing
+  No events.
+  ```
 
 * time, t=1m
-```console
-$ kubectl get ing
-NAME           RULE      BACKEND                   ADDRESS
-test-ingress   -         default-http-backend:80   130.211.5.27
-
-$ kubectl describe ing
-target-proxy:		k8s-tp-default-test-ingress
-url-map:		    k8s-um-default-test-ingress
-backends:		    {"k8s-be-32342":"UNKNOWN"}
-forwarding-rule:	k8s-fw-default-test-ingress
-Events:
-  FirstSeen	LastSeen	Count	From				SubobjectPath	Reason	Message
-  ─────────	────────	─────	────				─────────────	──────	───────
-  46s		46s		1	{loadbalancer-controller }	Success	Created loadbalancer 130.211.5.27
-```
+  ```console
+  $ kubectl get ing
+  NAME           RULE      BACKEND                   ADDRESS
+  test-ingress   -         default-http-backend:80   130.211.5.27
+
+  $ kubectl describe ing
+  target-proxy:		k8s-tp-default-test-ingress
+  url-map:		    k8s-um-default-test-ingress
+  backends:		    {"k8s-be-32342":"UNKNOWN"}
+  forwarding-rule:	k8s-fw-default-test-ingress
+  Events:
+    FirstSeen	LastSeen	Count	From				SubobjectPath	Reason	Message
+    ─────────	────────	─────	────				─────────────	──────	───────
+    46s		46s		1	{loadbalancer-controller }	Success	Created loadbalancer 130.211.5.27
+  ```
 
 * time, t=5m
-```console
-$ kubectl describe ing
-target-proxy:		k8s-tp-default-test-ingress
-url-map:		    k8s-um-default-test-ingress
-backends:		    {"k8s-be-32342":"HEALTHY"}
-forwarding-rule:	k8s-fw-default-test-ingress
-Events:
-  FirstSeen	LastSeen	Count	From				SubobjectPath	Reason	Message
-  ─────────	────────	─────	────				─────────────	──────	───────
-  46s		46s		1	{loadbalancer-controller }	Success	Created loadbalancer 130.211.5.27
-
-```
+  ```console
+  $ kubectl describe ing
+  target-proxy:		k8s-tp-default-test-ingress
+  url-map:		    k8s-um-default-test-ingress
+  backends:		    {"k8s-be-32342":"HEALTHY"}
+  forwarding-rule:	k8s-fw-default-test-ingress
+  Events:
+    FirstSeen	LastSeen	Count	From				SubobjectPath	Reason	Message
+    ─────────	────────	─────	────				─────────────	──────	───────
+    46s		46s		1	{loadbalancer-controller }	Success	Created loadbalancer 130.211.5.27
+  ```
 
 ## Disabling GLBC
 
@@ -87,20 +86,20 @@ Since GLBC runs as a cluster addon, you cannot simply delete the RC. The easiest
 
 * IFF you want to tear down existing L7 loadbalancers, hit the /delete-all-and-quit endpoint on the pod:
 
-```console
-$ kubectl get pods --namespace=kube-system
-NAME                                               READY     STATUS    RESTARTS   AGE
-l7-lb-controller-7bb21                             1/1       Running   0          1h
-$ kubectl exec l7-lb-controller-7bb21 -c l7-lb-controller curl http://localhost:8081/delete-all-and-quit --namespace=kube-system
-$ kubectl logs l7-lb-controller-7b221 -c l7-lb-controller --follow
-...
-I1007 00:30:00.322528       1 main.go:160] Handled quit, awaiting pod deletion.
-```
+  ```console
+  $ kubectl get pods --namespace=kube-system
+  NAME                                               READY     STATUS    RESTARTS   AGE
+  l7-lb-controller-7bb21                             1/1       Running   0          1h
+  $ kubectl exec l7-lb-controller-7bb21 -c l7-lb-controller curl http://localhost:8081/delete-all-and-quit --namespace=kube-system
+  $ kubectl logs l7-lb-controller-7b221 -c l7-lb-controller --follow
+  ...
+  I1007 00:30:00.322528       1 main.go:160] Handled quit, awaiting pod deletion.
+  ```
 
 * Nullify the RC (but don't delete it or the addon controller will "fix" it for you)
-```console
-$ kubectl scale rc l7-lb-controller --replicas=0 --namespace=kube-system
-```
+  ```console
+  $ kubectl scale rc l7-lb-controller --replicas=0 --namespace=kube-system
+  ```
 
 ## Limitations
 

cluster/addons/cluster-monitoring/google/heapster-controller.yaml

@@ -23,29 +23,29 @@ metadata:
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: heapster-v1.4.0-beta.0
+  name: heapster-v1.4.0
   namespace: kube-system
   labels:
     k8s-app: heapster
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
-    version: v1.4.0-beta.0
+    version: v1.4.0
 spec:
   replicas: 1
   selector:
     matchLabels:
       k8s-app: heapster
-      version: v1.4.0-beta.0
+      version: v1.4.0
   template:
     metadata:
       labels:
         k8s-app: heapster
-        version: v1.4.0-beta.0
+        version: v1.4.0
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
-        - image: gcr.io/google_containers/heapster-amd64:v1.4.0-beta.0
+        - image: gcr.io/google_containers/heapster-amd64:v1.4.0
           name: heapster
           livenessProbe:
             httpGet:
@@ -65,7 +65,7 @@ spec:
             - name: usr-ca-certs
               mountPath: /usr/share/ca-certificates
               readOnly: true
-        - image: gcr.io/google_containers/heapster-amd64:v1.4.0-beta.0
+        - image: gcr.io/google_containers/heapster-amd64:v1.4.0
           name: eventer
           command:
             - /eventer
@@ -78,7 +78,7 @@ spec:
             - name: usr-ca-certs
               mountPath: /usr/share/ca-certificates
               readOnly: true
-        - image: gcr.io/google_containers/addon-resizer:1.7
+        - image: gcr.io/google_containers/addon-resizer:2.0
           name: heapster-nanny
           resources:
             limits:
@@ -102,12 +102,10 @@ spec:
             - --extra-cpu={{ metrics_cpu_per_node }}m
             - --memory={{ base_metrics_memory }}
             - --extra-memory={{metrics_memory_per_node}}Mi
-            - --threshold=5
-            - --deployment=heapster-v1.4.0-beta.0
+            - --deployment=heapster-v1.4.0
             - --container=heapster
             - --poll-period=300000
-            - --estimator=exponential
-        - image: gcr.io/google_containers/addon-resizer:1.7
+        - image: gcr.io/google_containers/addon-resizer:2.0
           name: eventer-nanny
           resources:
             limits:
@@ -131,11 +129,9 @@ spec:
             - --extra-cpu=0m
             - --memory={{base_eventer_memory}}
             - --extra-memory={{eventer_memory_per_node}}Ki
-            - --threshold=5
-            - --deployment=heapster-v1.4.0-beta.0
+            - --deployment=heapster-v1.4.0
             - --container=eventer
             - --poll-period=300000
-            - --estimator=exponential
       volumes:
         - name: ssl-certs
           hostPath:

cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml

@@ -23,29 +23,29 @@ metadata:
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: heapster-v1.4.0-beta.0
+  name: heapster-v1.4.0
   namespace: kube-system
   labels:
     k8s-app: heapster
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
-    version: v1.4.0-beta.0
+    version: v1.4.0
 spec:
   replicas: 1
   selector:
     matchLabels:
       k8s-app: heapster
-      version: v1.4.0-beta.0
+      version: v1.4.0
   template:
     metadata:
       labels:
         k8s-app: heapster
-        version: v1.4.0-beta.0
+        version: v1.4.0
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
-        - image: gcr.io/google_containers/heapster-amd64:v1.4.0-beta.0
+        - image: gcr.io/google_containers/heapster-amd64:v1.4.0
           name: heapster
           livenessProbe:
             httpGet:
@@ -66,7 +66,7 @@ spec:
             - name: usr-ca-certs
               mountPath: /usr/share/ca-certificates
               readOnly: true
-        - image: gcr.io/google_containers/heapster-amd64:v1.4.0-beta.0
+        - image: gcr.io/google_containers/heapster-amd64:v1.4.0
           name: eventer
           command:
             - /eventer
@@ -79,7 +79,7 @@ spec:
             - name: usr-ca-certs
               mountPath: /usr/share/ca-certificates
               readOnly: true
-        - image: gcr.io/google_containers/addon-resizer:1.7
+        - image: gcr.io/google_containers/addon-resizer:2.0
           name: heapster-nanny
           resources:
             limits:
@@ -103,12 +103,10 @@ spec:
             - --extra-cpu={{ metrics_cpu_per_node }}m
             - --memory={{ base_metrics_memory }}
             - --extra-memory={{ metrics_memory_per_node }}Mi
-            - --threshold=5
-            - --deployment=heapster-v1.4.0-beta.0
+            - --deployment=heapster-v1.4.0
             - --container=heapster
             - --poll-period=300000
-            - --estimator=exponential
-        - image: gcr.io/google_containers/addon-resizer:1.7
+        - image: gcr.io/google_containers/addon-resizer:2.0
           name: eventer-nanny
           resources:
             limits:
@@ -132,11 +130,9 @@ spec:
             - --extra-cpu=0m
             - --memory={{ base_eventer_memory }}
             - --extra-memory={{ eventer_memory_per_node }}Ki
-            - --threshold=5
-            - --deployment=heapster-v1.4.0-beta.0
+            - --deployment=heapster-v1.4.0
             - --container=eventer
             - --poll-period=300000
-            - --estimator=exponential
       volumes:
         - name: ssl-certs
           hostPath:

cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml

@@ -23,29 +23,29 @@ metadata:
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: heapster-v1.4.0-beta.0
+  name: heapster-v1.4.0
   namespace: kube-system
   labels:
     k8s-app: heapster
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
-    version: v1.4.0-beta.0
+    version: v1.4.0
 spec:
   replicas: 1
   selector:
     matchLabels:
       k8s-app: heapster
-      version: v1.4.0-beta.0
+      version: v1.4.0
   template:
     metadata:
       labels:
         k8s-app: heapster
-        version: v1.4.0-beta.0
+        version: v1.4.0
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
-        - image: gcr.io/google_containers/heapster-amd64:v1.4.0-beta.0
+        - image: gcr.io/google_containers/heapster-amd64:v1.4.0
           name: heapster
           livenessProbe:
             httpGet:
@@ -58,13 +58,13 @@ spec:
             - /heapster
             - --source=kubernetes.summary_api:''
             - --sink=influxdb:http://monitoring-influxdb:8086
-        - image: gcr.io/google_containers/heapster-amd64:v1.4.0-beta.0
+        - image: gcr.io/google_containers/heapster-amd64:v1.4.0
           name: eventer
           command:
             - /eventer
             - --source=kubernetes:''
             - --sink=influxdb:http://monitoring-influxdb:8086
-        - image: gcr.io/google_containers/addon-resizer:1.7
+        - image: gcr.io/google_containers/addon-resizer:2.0
           name: heapster-nanny
           resources:
             limits:
@@ -88,12 +88,10 @@ spec:
             - --extra-cpu={{ metrics_cpu_per_node }}m
             - --memory={{ base_metrics_memory }}
             - --extra-memory={{ metrics_memory_per_node }}Mi
-            - --threshold=5
-            - --deployment=heapster-v1.4.0-beta.0
+            - --deployment=heapster-v1.4.0
             - --container=heapster
             - --poll-period=300000
-            - --estimator=exponential
-        - image: gcr.io/google_containers/addon-resizer:1.7
+        - image: gcr.io/google_containers/addon-resizer:2.0
           name: eventer-nanny
           resources:
             limits:
@@ -117,11 +115,9 @@ spec:
             - --extra-cpu=0m
             - --memory={{ base_eventer_memory }}
             - --extra-memory={{ eventer_memory_per_node }}Ki
-            - --threshold=5
-            - --deployment=heapster-v1.4.0-beta.0
+            - --deployment=heapster-v1.4.0
             - --container=eventer
             - --poll-period=300000
-            - --estimator=exponential
       serviceAccountName: heapster
       tolerations:
         - key: "CriticalAddonsOnly"

cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml

@@ -21,29 +21,29 @@ metadata:
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: heapster-v1.4.0-beta.0
+  name: heapster-v1.4.0
   namespace: kube-system
   labels:
     k8s-app: heapster
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
-    version: v1.4.0-beta.0
+    version: v1.4.0
 spec:
   replicas: 1
   selector:
     matchLabels:
       k8s-app: heapster
-      version: v1.4.0-beta.0
+      version: v1.4.0
   template:
     metadata:
       labels:
         k8s-app: heapster
-        version: v1.4.0-beta.0
+        version: v1.4.0
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
-        - image: gcr.io/google_containers/heapster-amd64:v1.4.0-beta.0
+        - image: gcr.io/google_containers/heapster-amd64:v1.4.0
           name: heapster
           livenessProbe:
             httpGet:
@@ -55,7 +55,7 @@ spec:
           command:
             - /heapster
             - --source=kubernetes.summary_api:''
-            - --sink=stackdriver
+            - --sink=stackdriver:?cluster_name={{ cluster_name }}
               # TODO: add --disable_export when it's merged into Heapster release
           volumeMounts:
             - name: ssl-certs
@@ -64,7 +64,7 @@ spec:
             - name: usr-ca-certs
               mountPath: /usr/share/ca-certificates
               readOnly: true
-        - image: gcr.io/google_containers/addon-resizer:1.7
+        - image: gcr.io/google_containers/addon-resizer:2.0
           name: heapster-nanny
           resources:
             limits:
@@ -88,11 +88,9 @@ spec:
             - --extra-cpu={{ metrics_cpu_per_node }}m
             - --memory={{ base_metrics_memory }}
             - --extra-memory={{metrics_memory_per_node}}Mi
-            - --threshold=5
-            - --deployment=heapster-v1.4.0-beta.0
+            - --deployment=heapster-v1.4.0
             - --container=heapster
             - --poll-period=300000
-            - --estimator=exponential
       volumes:
         - name: ssl-certs
           hostPath:

cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml

@@ -21,29 +21,29 @@ metadata:
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: heapster-v1.4.0-beta.0
+  name: heapster-v1.4.0
   namespace: kube-system
   labels:
     k8s-app: heapster
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
-    version: v1.4.0-beta.0
+    version: v1.4.0
 spec:
   replicas: 1
   selector:
     matchLabels:
       k8s-app: heapster
-      version: v1.4.0-beta.0
+      version: v1.4.0
   template:
     metadata:
       labels:
         k8s-app: heapster
-        version: v1.4.0-beta.0
+        version: v1.4.0
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
-        - image: gcr.io/google_containers/heapster-amd64:v1.4.0-beta.0
+        - image: gcr.io/google_containers/heapster-amd64:v1.4.0
           name: heapster
           livenessProbe:
             httpGet:
@@ -55,7 +55,7 @@ spec:
           command:
             - /heapster
             - --source=kubernetes.summary_api:''
-        - image: gcr.io/google_containers/addon-resizer:1.7
+        - image: gcr.io/google_containers/addon-resizer:2.0
           name: heapster-nanny
           resources:
             limits:
@@ -79,11 +79,9 @@ spec:
             - --extra-cpu={{ metrics_cpu_per_node }}m
             - --memory={{ base_metrics_memory }}
             - --extra-memory={{ metrics_memory_per_node }}Mi
-            - --threshold=5
-            - --deployment=heapster-v1.4.0-beta.0
+            - --deployment=heapster-v1.4.0
             - --container=heapster
             - --poll-period=300000
-            - --estimator=exponential
       serviceAccountName: heapster
       tolerations:
         - key: "CriticalAddonsOnly"

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
-  name: fluentd-es-v1.22
+  name: fluentd-es-v1.24
   namespace: kube-system
   labels:
     k8s-app: fluentd-es
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
-    version: v1.22
+    version: v1.24
 spec:
   template:
     metadata:
       labels:
         k8s-app: fluentd-es
         kubernetes.io/cluster-service: "true"
-        version: v1.22
+        version: v1.24
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -24,7 +24,7 @@ spec:
       serviceAccountName: fluentd-es
       containers:
       - name: fluentd-es
-        image: gcr.io/google_containers/fluentd-elasticsearch:1.23
+        image: gcr.io/google_containers/fluentd-elasticsearch:1.24
         command:
           - '/bin/sh'
           - '-c'

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -16,7 +16,7 @@
 
 PREFIX = gcr.io/google_containers
 IMAGE = fluentd-elasticsearch
-TAG = 1.23
+TAG = 1.24
 
 build:
 	docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) .

cluster/addons/fluentd-elasticsearch/fluentd-es-image/build.sh

@@ -32,6 +32,7 @@ sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/in
 # http://docs.fluentd.org/articles/plugin-management
 td-agent-gem install --no-document fluent-plugin-kubernetes_metadata_filter -v 0.27.0
 td-agent-gem install --no-document fluent-plugin-elasticsearch -v 1.9.5
+td-agent-gem install --no-document fluent-plugin-prometheus -v 0.3.0
 
 # Remove docs and postgres references
 rm -rf /opt/td-agent/embedded/share/doc \

cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf

@@ -283,6 +283,44 @@
   type kubernetes_metadata
 </filter>
 
+# Prometheus Exporter Plugin
+# input plugin that exports metrics
+<source>
+  type prometheus
+</source>
+
+<source>
+  type monitor_agent
+</source>
+
+<source>
+  type forward
+</source>
+
+# input plugin that collects metrics from MonitorAgent
+<source>
+  @type prometheus_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+# input plugin that collects metrics for output plugin
+<source>
+  @type prometheus_output_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+# input plugin that collects metrics for in_tail plugin
+<source>
+  @type prometheus_tail_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
 <match **>
    type elasticsearch
    log_level info

cluster/addons/fluentd-gcp/event-exporter.yaml

@@ -46,7 +46,7 @@ spec:
       containers:
       # TODO: Add resources in 1.8
       - name: event-exporter
-        image: gcr.io/google-containers/event-exporter:v0.1.0-r2
+        image: gcr.io/google-containers/event-exporter:v0.1.4
         command:
         - '/event-exporter'
       - name: prometheus-to-sd-exporter

cluster/addons/fluentd-gcp/fluentd-gcp-configmap.yaml

@@ -70,27 +70,14 @@ data:
 
     # Detect exceptions in the log output and forward them as one log entry.
     <match raw.kubernetes.**>
-      @type copy
-
-      <store>
-        @type prometheus
-
-        <metric>
-          type counter
-          name logging_line_count
-          desc Total number of lines generated by application containers
-        </metric>
-      </store>
-      <store>
-        @type detect_exceptions
-
-        remove_tag_prefix raw
-        message log
-        stream stream
-        multiline_flush_interval 5
-        max_bytes 500000
-        max_lines 1000
-      </store>
+      @type detect_exceptions
+
+      remove_tag_prefix raw
+      message log
+      stream stream
+      multiline_flush_interval 5
+      max_bytes 500000
+      max_lines 1000
     </match>
   system.input.conf: |-
     # Example:
@@ -291,6 +278,14 @@ data:
       read_from_head true
       tag kubelet
     </source>
+
+    <source>
+      type systemd
+      filters [{ "_SYSTEMD_UNIT": "node-problem-detector.service" }]
+      pos_file /var/log/gcp-journald-node-problem-detector.pos
+      read_from_head true
+      tag node-problem-detector
+    </source>
   monitoring.conf: |-
     # Prometheus monitoring
     <source>
@@ -342,77 +337,50 @@ data:
     # compute.googleapis.com service rather than container.googleapis.com to keep
     # them separate since most users don't care about the node logs.
     <match kubernetes.**>
-      @type copy
-
-      <store>
-        @type google_cloud
-
-        # Set the buffer type to file to improve the reliability and reduce the memory consumption
-        buffer_type file
-        buffer_path /var/log/fluentd-buffers/kubernetes.containers.buffer
-        # Set queue_full action to block because we want to pause gracefully
-        # in case of the off-the-limits load instead of throwing an exception
-        buffer_queue_full_action block
-        # Set the chunk limit conservatively to avoid exceeding the GCL limit
-        # of 10MiB per write request.
-        buffer_chunk_limit 2M
-        # Cap the combined memory usage of this buffer and the one below to
-        # 2MiB/chunk * (6 + 2) chunks = 16 MiB
-        buffer_queue_limit 6
-        # Never wait more than 5 seconds before flushing logs in the non-error case.
-        flush_interval 5s
-        # Never wait longer than 30 seconds between retries.
-        max_retry_wait 30
-        # Disable the limit on the number of retries (retry forever).
-        disable_retry_limit
-        # Use multiple threads for processing.
-        num_threads 2
-      </store>
-      <store>
-        @type prometheus
-
-        <metric>
-          type counter
-          name logging_entry_count
-          desc Total number of log entries generated by either application containers or system components
-          <labels>
-            component container
-          </labels>
-        </metric>
-      </store>
+      @type google_cloud
+
+      # Collect metrics in Prometheus registry about plugin activity.
+      enable_monitoring true
+      monitoring_type prometheus
+      # Set the buffer type to file to improve the reliability and reduce the memory consumption
+      buffer_type file
+      buffer_path /var/log/fluentd-buffers/kubernetes.containers.buffer
+      # Set queue_full action to block because we want to pause gracefully
+      # in case of the off-the-limits load instead of throwing an exception
+      buffer_queue_full_action block
+      # Set the chunk limit conservatively to avoid exceeding the GCL limit
+      # of 10MiB per write request.
+      buffer_chunk_limit 2M
+      # Cap the combined memory usage of this buffer and the one below to
+      # 2MiB/chunk * (6 + 2) chunks = 16 MiB
+      buffer_queue_limit 6
+      # Never wait more than 5 seconds before flushing logs in the non-error case.
+      flush_interval 5s
+      # Never wait longer than 30 seconds between retries.
+      max_retry_wait 30
+      # Disable the limit on the number of retries (retry forever).
+      disable_retry_limit
+      # Use multiple threads for processing.
+      num_threads 2
     </match>
 
     # Keep a smaller buffer here since these logs are less important than the user's
     # container logs.
     <match **>
-      @type copy
-
-      <store>
-        @type google_cloud
-
-        detect_subservice false
-        buffer_type file
-        buffer_path /var/log/fluentd-buffers/kubernetes.system.buffer
-        buffer_queue_full_action block
-        buffer_chunk_limit 2M
-        buffer_queue_limit 2
-        flush_interval 5s
-        max_retry_wait 30
-        disable_retry_limit
-        num_threads 2
-      </store>
-      <store>
-        @type prometheus
-
-        <metric>
-          type counter
-          name logging_entry_count
-          desc Total number of log entries generated by either application containers or system components
-          <labels>
-            component system
-          </labels>
-        </metric>
-      </store>
+      @type google_cloud
+
+      enable_monitoring true
+      monitoring_type prometheus
+      detect_subservice false
+      buffer_type file
+      buffer_path /var/log/fluentd-buffers/kubernetes.system.buffer
+      buffer_queue_full_action block
+      buffer_chunk_limit 2M
+      buffer_queue_limit 2
+      flush_interval 5s
+      max_retry_wait 30
+      disable_retry_limit
+      num_threads 2
     </match>
 metadata:
   name: fluentd-gcp-config-v1.1

cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml

@@ -27,7 +27,7 @@ spec:
       hostNetwork: true
       containers:
       - name: fluentd-gcp
-        image: gcr.io/google-containers/fluentd-gcp:2.0.7
+        image: gcr.io/google-containers/fluentd-gcp:2.0.8
         # If fluentd consumes its own logs, the following situation may happen:
         # fluentd fails to send a chunk to the server => writes it to the log =>
         # tries to send this message to the server => fails to send a chunk and so on.
@@ -90,13 +90,13 @@ spec:
                 exit 1;
               fi;
       - name: prometheus-to-sd-exporter
-        image: gcr.io/google-containers/prometheus-to-sd:v0.1.0
+        image: gcr.io/google-containers/prometheus-to-sd:v0.1.3
         command:
           - /monitor
           - --component=fluentd
           - --target-port=31337
           - --stackdriver-prefix=container.googleapis.com/internal/addons
-          - --whitelisted-metrics=logging_line_count,logging_entry_count
+          - --whitelisted-metrics=stackdriver_successful_requests_count,stackdriver_failed_requests_count,stackdriver_ingested_entries_count,stackdriver_dropped_entries_count
         volumeMounts:
         - name: ssl-certs
           mountPath: /etc/ssl/certs
@@ -107,6 +107,9 @@ spec:
         effect: "NoSchedule"
       - operator: "Exists"
         effect: "NoExecute"
+      #TODO: remove this toleration once #44445 is properly fixed.
+      - operator: "Exists"
+        effect: "NoSchedule"
       terminationGracePeriodSeconds: 30
       volumes:
       - name: varlog

cluster/addons/fluentd-gcp/fluentd-gcp-image/README.md

 # Collecting Docker Log Files with Fluentd and sending to GCP.
 
-The image was moved to the the
+The image was moved to the
 [new location](https://github.com/kubernetes/contrib/tree/master/fluentd/fluentd-gcp-image).
 
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/fluentd-gcp/fluentd-gcp-image/README.md?pixel)]()

cluster/addons/node-problem-detector/npd.yaml

@@ -80,3 +80,5 @@ spec:
       tolerations:
       - operator: "Exists"
         effect: "NoExecute"
+      - key: "CriticalAddonsOnly"
+        operator: "Exists"

cluster/common.sh

@@ -605,6 +605,7 @@ function build-kube-env {
 
   rm -f ${file}
   cat >$file <<EOF
+CLUSTER_NAME: $(yaml-quote ${CLUSTER_NAME})
 ENV_TIMESTAMP: $(yaml-quote $(date -u +%Y-%m-%dT%T%z))
 INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX})
 NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX})
@@ -664,7 +665,9 @@ ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote ${ENABLE_DEFAULT_STORAGE_CLASS:-})
 ENABLE_APISERVER_BASIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_BASIC_AUDIT:-})
 ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-})
 ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false})
+ENABLE_PATCH_CONVERSION_DETECTOR: $(yaml-quote ${ENABLE_PATCH_CONVERSION_DETECTOR:-false})
 ADVANCED_AUDIT_BACKEND: $(yaml-quote ${ADVANCED_AUDIT_BACKEND:-log})
+GCE_API_ENDPOINT: $(yaml-quote ${GCE_API_ENDPOINT:-})
 EOF
   if [ -n "${KUBELET_PORT:-}" ]; then
     cat >>$file <<EOF
@@ -743,6 +746,12 @@ EOF
 FEATURE_GATES: $(yaml-quote ${FEATURE_GATES})
 EOF
   fi
+  if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] ||
+     [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then
+    cat >>$file <<EOF
+VOLUME_PLUGIN_DIR: $(yaml-quote ${VOLUME_PLUGIN_DIR:-/home/kubernetes/flexvolume})
+EOF
+  fi
 
   if [ -n "${PROVIDER_VARS:-}" ]; then
     local var_name

cluster/gce/config-common.sh

@@ -33,6 +33,9 @@ function get-master-size {
   if [[ "${NUM_NODES}" -gt "500" ]]; then
     suggested_master_size=32
   fi
+  if [[ "${NUM_NODES}" -gt "3000" ]]; then
+    suggested_master_size=64
+  fi
   echo "${suggested_master_size}"
 }
 

cluster/gce/config-default.sh

@@ -19,6 +19,9 @@
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 source "${KUBE_ROOT}/cluster/gce/config-common.sh"
 
+# Specifying KUBE_GCE_API_ENDPOINT will override the default GCE Compute API endpoint (https://www.googleapis.com/compute/v1/).
+# This endpoint has to be pointing to v1 api. For example, https://www.googleapis.com/compute/staging_v1/
+GCE_API_ENDPOINT=${KUBE_GCE_API_ENDPOINT:-}
 GCLOUD=gcloud
 ZONE=${KUBE_GCE_ZONE:-us-central1-b}
 REGION=${ZONE%-*}
@@ -42,7 +45,7 @@ KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true}
 KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false}
 
 MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
-NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-debian}}
+NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
 if [[ "${MASTER_OS_DISTRIBUTION}" == "coreos" ]]; then
     MASTER_OS_DISTRIBUTION="container-linux"
 fi
@@ -63,15 +66,16 @@ if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
     NODE_ACCELERATORS=""
 fi
 
-# By default a cluster will be started with the master on GCI and nodes on
-# containervm. If you are updating the containervm version, update this
-# variable. Also please update corresponding image for node e2e at:
+# By default a cluster will be started with the master and nodes
+# on Container-optimized OS (cos, previously known as gci). If
+# you are updating the os image versions, update this variable.
+# Also please update corresponding image for node e2e at:
 # https://github.com/kubernetes/kubernetes/blob/master/test/e2e_node/jenkins/image-config.yaml
-CVM_VERSION=${CVM_VERSION:-container-vm-v20170214}
+CVM_VERSION=${CVM_VERSION:-container-vm-v20170627}
 GCI_VERSION=${KUBE_GCI_VERSION:-cos-stable-59-9460-64-0}
 MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-}
 MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud}
-NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}}
+NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}}
 NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-cos-cloud}
 CONTAINER_RUNTIME=${KUBE_CONTAINER_RUNTIME:-docker}
 RKT_VERSION=${KUBE_RKT_VERSION:-1.23.0}
@@ -118,6 +122,10 @@ ENABLE_L7_LOADBALANCING="${KUBE_ENABLE_L7_LOADBALANCING:-glbc}"
 #   standalone     - Heapster only. Metrics available via Heapster REST API.
 ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}"
 
+# One special node out of NUM_NODES would be created of this type if specified.
+# Useful for scheduling heapster in large clusters with nodes of small size.
+HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}"
+
 # Historically fluentd was a manifest pod and then was migrated to DaemonSet.
 # To avoid situation during cluster upgrade when there are two instances
 # of fluentd running on a node, kubelet need to mark node on which
@@ -125,7 +133,7 @@ ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}"
 # TODO(piosz): remove this in 1.8
 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}"
 
-# To avoid running Calico on a node that is not configured appropriately, 
+# To avoid running Calico on a node that is not configured appropriately,
 # label each Node so that the DaemonSet can run the Pods only on ready Nodes.
 if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then
 	NODE_LABELS="${NODE_LABELS},projectcalico.org/ds-ready=true"

cluster/gce/config-test.sh

@@ -19,6 +19,9 @@
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 source "${KUBE_ROOT}/cluster/gce/config-common.sh"
 
+# Specifying KUBE_GCE_API_ENDPOINT will override the default GCE Compute API endpoint (https://www.googleapis.com/compute/v1/).
+# This endpoint has to be pointing to v1 api. For example, https://www.googleapis.com/compute/staging_v1/
+GCE_API_ENDPOINT=${KUBE_GCE_API_ENDPOINT:-}
 GCLOUD=gcloud
 ZONE=${KUBE_GCE_ZONE:-us-central1-b}
 REGION=${ZONE%-*}
@@ -62,11 +65,13 @@ if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
     NODE_ACCELERATORS=""
 fi
 
-# By default a cluster will be started with the master on GCI and nodes on
-# containervm. If you are updating the containervm version, update this
-# variable. Also please update corresponding image for node e2e at:
+# By default a cluster will be started with the master and nodes
+# on Container-VM, the deprecated OS. Some tests assume container-VM,
+# and only when that is fixed can we use Container-Optimized OS
+# (cos, gci) as we do in config-default.sh.
+# Also please update corresponding image for node e2e at:
 # https://github.com/kubernetes/kubernetes/blob/master/test/e2e_node/jenkins/image-config.yaml
-CVM_VERSION=${CVM_VERSION:-container-vm-v20170214}
+CVM_VERSION=${CVM_VERSION:-container-vm-v20170627}
 GCI_VERSION=${KUBE_GCI_VERSION:-cos-stable-59-9460-64-0}
 MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-}
 MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud}
@@ -128,6 +133,10 @@ ENABLE_L7_LOADBALANCING="${KUBE_ENABLE_L7_LOADBALANCING:-glbc}"
 #   standalone     - Heapster only. Metrics available via Heapster REST API.
 ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}"
 
+# One special node out of NUM_NODES would be created of this type if specified.
+# Useful for scheduling heapster in large clusters with nodes of small size.
+HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}"
+
 # Set etcd image (e.g. 3.0.17-alpha.1) and version (e.g. 3.0.17) if you need
 # non-default version.
 ETCD_IMAGE="${TEST_ETCD_IMAGE:-}"

cluster/gce/container-linux/configure-helper.sh

@@ -224,6 +224,11 @@ function create-master-auth {
   cat <<EOF >/etc/gce.conf
 [global]
 EOF
+  if [[ -n "${GCE_API_ENDPOINT:-}" ]]; then
+    cat <<EOF >>/etc/gce.conf
+api-endpoint = ${GCE_API_ENDPOINT}
+EOF
+  fi
   if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then
     use_cloud_config="true"
     cat <<EOF >>/etc/gce.conf
@@ -232,6 +237,11 @@ token-body = ${TOKEN_BODY}
 project-id = ${PROJECT_ID}
 network-name = ${NODE_NETWORK}
 EOF
+    if [[ -n "${NETWORK_PROJECT_ID:-}" ]]; then
+      cat <<EOF >>/etc/gce.conf
+network-project-id = ${NETWORK_PROJECT_ID}
+EOF
+    fi
     if [[ -n "${NODE_SUBNETWORK:-}" ]]; then
       cat <<EOF >>/etc/gce.conf
 subnetwork-name = ${NODE_SUBNETWORK}
@@ -983,7 +993,16 @@ function start-kube-apiserver {
 
   local container_env=""
   if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
-    container_env="\"env\":[{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}],"
+    container_env="\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
+  fi
+  if [[ -n "${ENABLE_PATCH_CONVERSION_DETECTOR:-}" ]]; then
+    if [[ -n "${container_env}" ]]; then
+      container_env="${container_env}, "
+    fi
+    container_env="\"name\": \"KUBE_PATCH_CONVERSION_DETECTOR\", \"value\": \"${ENABLE_PATCH_CONVERSION_DETECTOR}\""
+  fi
+  if [[ -n "${container_env}" ]]; then
+    container_env="\"env\":[{${container_env}}],"
   fi
 
   src_file="${src_dir}/kube-apiserver.manifest"
@@ -1168,6 +1187,8 @@ function setup-addon-manifests {
 }
 
 # Prepares the manifests of k8s addons, and starts the addon manager.
+# Vars assumed:
+#   CLUSTER_NAME
 function start-kube-addons {
   echo "Prepare kube-addons manifests and start kube addon manager"
   local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty"
@@ -1205,6 +1226,7 @@ function start-kube-addons {
       controller_yaml="${controller_yaml}/heapster-controller.yaml"
     fi
     remove-salt-config-comments "${controller_yaml}"
+    sed -i -e "s@{{ cluster_name }}@${CLUSTER_NAME}@g" "${controller_yaml}"
     sed -i -e "s@{{ *base_metrics_memory *}}@${base_metrics_memory}@g" "${controller_yaml}"
     sed -i -e "s@{{ *base_metrics_cpu *}}@${base_metrics_cpu}@g" "${controller_yaml}"
     sed -i -e "s@{{ *base_eventer_memory *}}@${base_eventer_memory}@g" "${controller_yaml}"

cluster/gce/container-linux/node-helper.sh

@@ -17,14 +17,19 @@
 # A library of helper functions and constant for the Container Linux distro.
 source "${KUBE_ROOT}/cluster/gce/container-linux/helper.sh"
 
+function get-node-instance-metadata {
+  local metadata=""
+  metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml,"
+  metadata+="user-data=${KUBE_ROOT}/cluster/gce/container-linux/node.yaml,"
+  metadata+="configure-sh=${KUBE_ROOT}/cluster/gce/container-linux/configure.sh,"
+  metadata+="cluster-name=${KUBE_TEMP}/cluster-name.txt"
+  echo "${metadata}"
+}
+
 # $1: template name (required).
 function create-node-instance-template {
   local template_name="$1"
 
-  create-node-template "$template_name" "${scope_flags[*]}" \
-    "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
-    "user-data=${KUBE_ROOT}/cluster/gce/container-linux/node.yaml" \
-    "configure-sh=${KUBE_ROOT}/cluster/gce/container-linux/configure.sh" \
-    "cluster-name=${KUBE_TEMP}/cluster-name.txt"
+  create-node-template "$template_name" "${scope_flags[*]}" "$(get-node-instance-metadata)"
   # TODO(euank): We should include update-strategy here. We should also switch to ignition
 }

cluster/gce/debian/node-helper.sh

@@ -16,12 +16,17 @@
 
 # A library of helper functions and constant for debian os distro
 
+function get-node-instance-metadata {
+	local metadata=""
+	metadata+="startup-script=${KUBE_TEMP}/configure-vm.sh,"
+	metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml,"
+	metadata+="cluster-name=${KUBE_TEMP}/cluster-name.txt"
+	echo "${metadata}"
+}
+
 # $1: template name (required)
 function create-node-instance-template {
   local template_name="$1"
   prepare-startup-script
-  create-node-template "$template_name" "${scope_flags}" \
-    "startup-script=${KUBE_TEMP}/configure-vm.sh" \
-    "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
-    "cluster-name=${KUBE_TEMP}/cluster-name.txt"
+  create-node-template "$template_name" "${scope_flags}" "$(get-node-instance-metadata)"
 }

cluster/gce/gci/configure-helper.sh

@@ -379,6 +379,11 @@ function create-master-auth {
   cat <<EOF >/etc/gce.conf
 [global]
 EOF
+  if [[ -n "${GCE_API_ENDPOINT:-}" ]]; then
+    cat <<EOF >>/etc/gce.conf
+api-endpoint = ${GCE_API_ENDPOINT}
+EOF
+  fi
   if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then
     use_cloud_config="true"
     cat <<EOF >>/etc/gce.conf
@@ -387,6 +392,11 @@ token-body = ${TOKEN_BODY}
 project-id = ${PROJECT_ID}
 network-name = ${NODE_NETWORK}
 EOF
+    if [[ -n "${NETWORK_PROJECT_ID:-}" ]]; then
+        cat <<EOF >>/etc/gce.conf
+network-project-id = ${NETWORK_PROJECT_ID}
+EOF
+    fi
     if [[ -n "${NODE_SUBNETWORK:-}" ]]; then
       cat <<EOF >>/etc/gce.conf
 subnetwork-name = ${NODE_SUBNETWORK}
@@ -912,7 +922,11 @@ function start-kubelet {
     flags+=" --cni-bin-dir=/home/kubernetes/bin"
     if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
       # Calico uses CNI always.
-      flags+=" --network-plugin=cni"
+      if [[ "${KUBERNETES_PRIVATE_MASTER:-}" == "true" ]]; then
+        flags+=" --network-plugin=${NETWORK_PROVIDER}"
+      else
+        flags+=" --network-plugin=cni"
+      fi
     else
       # Otherwise use the configured value.
       flags+=" --network-plugin=${NETWORK_PROVIDER}"
@@ -1182,6 +1196,7 @@ function prepare-mounter-rootfs {
   mount --make-rshared "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
   mount --bind -o ro /proc "${CONTAINERIZED_MOUNTER_ROOTFS}/proc"
   mount --bind -o ro /dev "${CONTAINERIZED_MOUNTER_ROOTFS}/dev"
+  mount --bind -o ro /etc/resolv.conf "${CONTAINERIZED_MOUNTER_ROOTFS}/etc/resolv.conf"
 }
 
 # A helper function for removing salt configuration and comments from a file.
@@ -1398,7 +1413,16 @@ function start-kube-apiserver {
 
   local container_env=""
   if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
-    container_env="\"env\":[{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}],"
+    container_env="\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
+  fi
+  if [[ -n "${ENABLE_PATCH_CONVERSION_DETECTOR:-}" ]]; then
+    if [[ -n "${container_env}" ]]; then
+      container_env="${container_env}, "
+    fi
+    container_env="\"name\": \"KUBE_PATCH_CONVERSION_DETECTOR\", \"value\": \"${ENABLE_PATCH_CONVERSION_DETECTOR}\""
+  fi
+  if [[ -n "${container_env}" ]]; then
+    container_env="\"env\":[{${container_env}}],"
   fi
 
   if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
@@ -1597,6 +1621,8 @@ function setup-addon-manifests {
 }
 
 # Prepares the manifests of k8s addons, and starts the addon manager.
+# Vars assumed:
+#   CLUSTER_NAME
 function start-kube-addons {
   echo "Prepare kube-addons manifests and start kube addon manager"
   local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty"
@@ -1634,6 +1660,7 @@ function start-kube-addons {
       controller_yaml="${controller_yaml}/heapster-controller.yaml"
     fi
     remove-salt-config-comments "${controller_yaml}"
+    sed -i -e "s@{{ cluster_name }}@${CLUSTER_NAME}@g" "${controller_yaml}"
     sed -i -e "s@{{ *base_metrics_memory *}}@${base_metrics_memory}@g" "${controller_yaml}"
     sed -i -e "s@{{ *base_metrics_cpu *}}@${base_metrics_cpu}@g" "${controller_yaml}"
     sed -i -e "s@{{ *base_eventer_memory *}}@${base_eventer_memory}@g" "${controller_yaml}"
@@ -1701,7 +1728,7 @@ function start-kube-addons {
     sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}"
     sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}"
   else
-    # If not configured to use Calico, the set the typha replica count to 0, but only if the 
+    # If not configured to use Calico, the set the typha replica count to 0, but only if the
     # addon is present.
     local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml"
     if [[ -e $typha_dep_file ]]; then

cluster/gce/gci/node-helper.sh

@@ -17,16 +17,21 @@
 # A library of helper functions and constant for GCI distro
 source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
 
+function get-node-instance-metadata {
+  local metadata=""
+  metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml,"
+  metadata+="user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml,"
+  metadata+="configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,"
+  metadata+="cluster-name=${KUBE_TEMP}/cluster-name.txt,"
+  metadata+="gci-update-strategy=${KUBE_TEMP}/gci-update.txt,"
+  metadata+="gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,"
+  metadata+="gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt"
+  echo "${metadata}"
+}
+
 # $1: template name (required).
 function create-node-instance-template {
   local template_name="$1"
   ensure-gci-metadata-files
-  create-node-template "$template_name" "${scope_flags[*]}" \
-    "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
-    "user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml" \
-    "configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" \
-    "cluster-name=${KUBE_TEMP}/cluster-name.txt" \
-    "gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \
-    "gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" \
-    "gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt"
+  create-node-template "$template_name" "${scope_flags[*]}" "$(get-node-instance-metadata)"
 }

cluster/gce/util.sh

@@ -345,6 +345,11 @@ function detect-node-names() {
         --format='value(instance)'))
     done
   fi
+  # Add heapster node name to the list too (if it exists).
+  if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
+    NODE_NAMES+=("${NODE_INSTANCE_PREFIX}-heapster")
+  fi
+
   echo "INSTANCE_GROUPS=${INSTANCE_GROUPS[*]:-}" >&2
   echo "NODE_NAMES=${NODE_NAMES[*]:-}" >&2
 }
@@ -510,7 +515,7 @@ function make-gcloud-network-argument() {
     ret="${ret},aliases=pods-default:${alias_size}"
     ret="${ret} --no-can-ip-forward"
   else
-    if [[ ${PREEXISTING_NETWORK} = "true" && "${PREEXISTING_NETWORK_MODE}" != "custom" ]]; then
+    if [[ ${ENABLE_BIG_CLUSTER_SUBNETS} != "true" || (${PREEXISTING_NETWORK} = "true" && "${PREEXISTING_NETWORK_MODE}" != "custom") ]]; then
       ret="--network ${network}"
     else
       ret="--subnet=${network}"
@@ -533,7 +538,7 @@ function get-template-name-from-version() {
 # Robustly try to create an instance template.
 # $1: The name of the instance template.
 # $2: The scopes flag.
-# $3 and others: Metadata entries (must all be from a file).
+# $3: String of comma-separated metadata entries (must all be from a file).
 function create-node-template() {
   detect-project
   local template_name="$1"
@@ -600,7 +605,7 @@ function create-node-template() {
       ${network} \
       ${preemptible_minions} \
       $2 \
-      --metadata-from-file $(echo ${@:3} | tr ' ' ',') >&2; then
+      --metadata-from-file $3 >&2; then
         if (( attempt > 5 )); then
           echo -e "${color_red}Failed to create instance template $template_name ${color_norm}" >&2
           exit 2
@@ -1237,21 +1242,24 @@ function create-nodes-firewall() {
   }
 }
 
-function create-nodes-template() {
-  echo "Creating minions."
-
-  # TODO(zmerlynn): Refactor setting scope flags.
+function get-scope-flags() {
   local scope_flags=
   if [[ -n "${NODE_SCOPES}" ]]; then
     scope_flags="--scopes ${NODE_SCOPES}"
   else
     scope_flags="--no-scopes"
   fi
+  echo "${scope_flags}"
+}
+
+function create-nodes-template() {
+  echo "Creating nodes."
+
+  local scope_flags=$(get-scope-flags)
 
   write-node-env
 
   local template_name="${NODE_INSTANCE_PREFIX}-template"
-
   create-node-instance-template $template_name
 }
 
@@ -1279,7 +1287,13 @@ function set_num_migs() {
 function create-nodes() {
   local template_name="${NODE_INSTANCE_PREFIX}-template"
 
-  local instances_left=${NUM_NODES}
+  if [[ -z "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
+    local -r nodes="${NUM_NODES}"
+  else
+    local -r nodes=$(( NUM_NODES - 1 ))
+  fi
+
+  local instances_left=${nodes}
 
   #TODO: parallelize this loop to speed up the process
   for ((i=1; i<=${NUM_MIGS}; i++)); do
@@ -1305,6 +1319,47 @@ function create-nodes() {
         --zone "${ZONE}" \
         --project "${PROJECT}" || true;
   done
+
+  if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
+    echo "Creating a special node for heapster with machine-type ${HEAPSTER_MACHINE_TYPE}"
+    create-heapster-node
+  fi
+}
+
+# Assumes:
+# - NODE_INSTANCE_PREFIX
+# - PROJECT
+# - ZONE
+# - HEAPSTER_MACHINE_TYPE
+# - NODE_DISK_TYPE
+# - NODE_DISK_SIZE
+# - NODE_IMAGE_PROJECT
+# - NODE_IMAGE
+# - NODE_TAG
+# - NETWORK
+# - ENABLE_IP_ALIASES
+# - IP_ALIAS_SUBNETWORK
+# - IP_ALIAS_SIZE
+function create-heapster-node() {
+  local network=$(make-gcloud-network-argument \
+      "${NETWORK}" "" \
+      "${ENABLE_IP_ALIASES:-}" \
+      "${IP_ALIAS_SUBNETWORK:-}" \
+      "${IP_ALIAS_SIZE:-}")
+
+  gcloud compute instances \
+      create "${NODE_INSTANCE_PREFIX}-heapster" \
+      --project "${PROJECT}" \
+      --zone "${ZONE}" \
+      --machine-type="${HEAPSTER_MACHINE_TYPE}" \
+      --boot-disk-type "${NODE_DISK_TYPE}" \
+      --boot-disk-size "${NODE_DISK_SIZE}" \
+      --image-project="${NODE_IMAGE_PROJECT}" \
+      --image "${NODE_IMAGE}" \
+      --tags "${NODE_TAG}" \
+      ${network} \
+      $(get-scope-flags) \
+      --metadata-from-file "$(get-node-instance-metadata)"
 }
 
 # Assumes:
@@ -1505,6 +1560,20 @@ function kube-down() {
           "${template}"
       fi
     done
+
+    # Delete the special heapster node (if it exists).
+    if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
+      local -r heapster_machine_name="${NODE_INSTANCE_PREFIX}-heapster"
+      if gcloud compute instances describe "${heapster_machine_name}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
+        # Now we can safely delete the VM.
+        gcloud compute instances delete \
+          --project "${PROJECT}" \
+          --quiet \
+          --delete-disks all \
+          --zone "${ZONE}" \
+          "${heapster_machine_name}"
+      fi
+    fi
   fi
 
   local -r REPLICA_NAME="${KUBE_REPLICA_NAME:-$(get-replica-name)}"
@@ -1875,13 +1944,7 @@ function prepare-push() {
   if [[ "${node}" == "true" ]]; then
     write-node-env
 
-    # TODO(zmerlynn): Refactor setting scope flags.
-    local scope_flags=
-    if [[ -n "${NODE_SCOPES}" ]]; then
-      scope_flags="--scopes ${NODE_SCOPES}"
-    else
-      scope_flags="--no-scopes"
-    fi
+    local scope_flags=$(get-scope-flags)
 
     # Ugly hack: Since it is not possible to delete instance-template that is currently
     # being used, create a temp one, then delete the old one and recreate it once again.

cluster/juju/layers/kubeapi-load-balancer/reactive/load_balancer.py

@@ -29,6 +29,7 @@ from charms.layer import nginx
 from subprocess import Popen
 from subprocess import PIPE
 from subprocess import STDOUT
+from subprocess import CalledProcessError
 
 
 @when('certificates.available')
@@ -49,6 +50,16 @@ def request_server_certificates(tls):
     tls.request_server_cert(common_name, sans, certificate_name)
 
 
+@when('config.changed.port')
+def close_old_port():
+    config = hookenv.config()
+    old_port = config.previous('port')
+    try:
+        hookenv.close_port(old_port)
+    except CalledProcessError:
+        hookenv.log('Port %d already closed, skipping.' % old_port)
+
+
 @when('nginx.available', 'apiserver.available',
       'certificates.server.cert.available')
 def install_load_balancer(apiserver, tls):
@@ -63,20 +74,23 @@ def install_load_balancer(apiserver, tls):
     if cert_exists and key_exists:
         # At this point the cert and key exist, and they are owned by root.
         chown = ['chown', 'www-data:www-data', server_cert_path]
+
         # Change the owner to www-data so the nginx process can read the cert.
         subprocess.call(chown)
         chown = ['chown', 'www-data:www-data', server_key_path]
+
         # Change the owner to www-data so the nginx process can read the key.
         subprocess.call(chown)
 
-        hookenv.open_port(hookenv.config('port'))
+        port = hookenv.config('port')
+        hookenv.open_port(port)
         services = apiserver.services()
         nginx.configure_site(
                 'apilb',
                 'apilb.conf',
                 server_name='_',
                 services=services,
-                port=hookenv.config('port'),
+                port=port,
                 server_certificate=server_cert_path,
                 server_key=server_key_path,
         )

cluster/juju/layers/kubeapi-load-balancer/templates/apilb.conf

@@ -8,7 +8,7 @@ upstream target_service {
 
 
 server {
-    listen 443 ssl http2;
+    listen {{ port }} ssl http2;
     server_name {{ server_name }};
 
     access_log /var/log/nginx.access.log;
@@ -33,9 +33,6 @@ server {
       proxy_set_header        Connection $http_connection;
       proxy_set_header        X-Stream-Protocol-Version $http_x_stream_protocol_version;
 
-      proxy_ssl_certificate   {{ server_certificate }};
-      proxy_ssl_certificate_key {{ server_key }};
-
       add_header              X-Stream-Protocol-Version $upstream_http_x_stream_protocol_version;
 
       proxy_pass              https://target_service;

cluster/juju/layers/kubernetes-e2e/README.md

@@ -21,7 +21,8 @@ and then relate the `kubernetes-e2e` charm.
 ```shell
 juju deploy kubernetes-core
 juju deploy cs:~containers/kubernetes-e2e
-juju add-relation kubernetes-e2e kubernetes-master
+juju add-relation kubernetes-e2e:kube-control kubernetes-master:kube-control
+juju add-relation kubernetes-e2e:kubernetes-master kubernetes-master:kube-api-endpoint
 juju add-relation kubernetes-e2e easyrsa
 ```
 

cluster/juju/layers/kubernetes-e2e/layer.yaml

@@ -4,6 +4,7 @@ includes:
   - layer:tls-client
   - layer:snap
   - interface:http
+  - interface:kube-control
 options:
   tls-client:
     ca_certificate_path: '/srv/kubernetes/ca.crt'

cluster/juju/layers/kubernetes-e2e/metadata.yaml

@@ -14,6 +14,8 @@ series:
 requires:
   kubernetes-master:
     interface: http
+  kube-control:
+    interface: kube-control
 resources:
   kubectl:
     type: file
@@ -23,3 +25,4 @@ resources:
     type: file
     filename: kubernetes-test.snap
     description: kubernetes-test snap
+

cluster/juju/layers/kubernetes-e2e/reactive/kubernetes_e2e.py

@@ -38,15 +38,22 @@ def reset_delivery_states():
 
 
 @when('kubernetes-e2e.installed')
+def report_status():
+    ''' Report the status of the charm. '''
+    messaging()
+
+
 def messaging():
     ''' Probe our relations to determine the propper messaging to the
     end user '''
 
     missing_services = []
     if not is_state('kubernetes-master.available'):
-        missing_services.append('kubernetes-master')
+        missing_services.append('kubernetes-master:http')
     if not is_state('certificates.available'):
         missing_services.append('certificates')
+    if not is_state('kubeconfig.ready'):
+        missing_services.append('kubernetes-master:kube-control')
 
     if missing_services:
         if len(missing_services) > 1:
@@ -80,16 +87,15 @@ def install_snaps():
 
 @when('tls_client.ca.saved', 'tls_client.client.certificate.saved',
       'tls_client.client.key.saved', 'kubernetes-master.available',
-      'kubernetes-e2e.installed')
+      'kubernetes-e2e.installed', 'kube-control.auth.available')
 @when_not('kubeconfig.ready')
-def prepare_kubeconfig_certificates(master):
+def prepare_kubeconfig_certificates(master, kube_control):
     ''' Prepare the data to feed to create the kubeconfig file. '''
 
     layer_options = layer.options('tls-client')
     # Get all the paths to the tls information required for kubeconfig.
     ca = layer_options.get('ca_certificate_path')
-    key = layer_options.get('client_key_path')
-    cert = layer_options.get('client_certificate_path')
+    creds = kube_control.get_auth_credentials()
 
     servers = get_kube_api_servers(master)
 
@@ -97,17 +103,28 @@ def prepare_kubeconfig_certificates(master):
     kubeconfig_path = '/home/ubuntu/.kube/config'
 
     # Create kubernetes configuration in the default location for ubuntu.
-    create_kubeconfig('/root/.kube/config', servers[0], ca, key, cert,
-                      user='root')
-    create_kubeconfig(kubeconfig_path, servers[0], ca, key, cert,
-                      user='ubuntu')
+    create_kubeconfig('/root/.kube/config', servers[0], ca,
+                      token=creds['client_token'], user='root')
+    create_kubeconfig(kubeconfig_path, servers[0], ca,
+                      token=creds['client_token'], user='ubuntu')
     # Set permissions on the ubuntu users kubeconfig to ensure a consistent UX
     cmd = ['chown', 'ubuntu:ubuntu', kubeconfig_path]
     check_call(cmd)
-
+    messaging()
     set_state('kubeconfig.ready')
 
 
+@when('kube-control.connected')
+def request_credentials(kube_control):
+    """ Request authorization creds."""
+
+    # The kube-cotrol interface is created to support RBAC.
+    # At this point we might as well do the right thing and return the hostname
+    # even if it will only be used when we enable RBAC
+    user = 'system:masters'
+    kube_control.set_auth_request(user)
+
+
 @when('kubernetes-e2e.installed', 'kubeconfig.ready')
 def set_app_version():
     ''' Declare the application version to juju '''
@@ -124,19 +141,40 @@ def set_app_version():
     hookenv.application_version_set(version_from.rstrip())
 
 
-def create_kubeconfig(kubeconfig, server, ca, key, certificate, user='ubuntu',
-                      context='juju-context', cluster='juju-cluster'):
+def create_kubeconfig(kubeconfig, server, ca, key=None, certificate=None,
+                      user='ubuntu', context='juju-context',
+                      cluster='juju-cluster', password=None, token=None):
     '''Create a configuration for Kubernetes based on path using the supplied
     arguments for values of the Kubernetes server, CA, key, certificate, user
     context and cluster.'''
+    if not key and not certificate and not password and not token:
+        raise ValueError('Missing authentication mechanism.')
+
+    # token and password are mutually exclusive. Error early if both are
+    # present. The developer has requested an impossible situation.
+    # see: kubectl config set-credentials --help
+    if token and password:
+        raise ValueError('Token and Password are mutually exclusive.')
     # Create the config file with the address of the master server.
     cmd = 'kubectl config --kubeconfig={0} set-cluster {1} ' \
           '--server={2} --certificate-authority={3} --embed-certs=true'
     check_call(split(cmd.format(kubeconfig, cluster, server, ca)))
+    # Delete old users
+    cmd = 'kubectl config --kubeconfig={0} unset users'
+    check_call(split(cmd.format(kubeconfig)))
     # Create the credentials using the client flags.
-    cmd = 'kubectl config --kubeconfig={0} set-credentials {1} ' \
-          '--client-key={2} --client-certificate={3} --embed-certs=true'
-    check_call(split(cmd.format(kubeconfig, user, key, certificate)))
+    cmd = 'kubectl config --kubeconfig={0} ' \
+          'set-credentials {1} '.format(kubeconfig, user)
+
+    if key and certificate:
+        cmd = '{0} --client-key={1} --client-certificate={2} '\
+              '--embed-certs=true'.format(cmd, key, certificate)
+    if password:
+        cmd = "{0} --username={1} --password={2}".format(cmd, user, password)
+    # This is mutually exclusive from password. They will not work together.
+    if token:
+        cmd = "{0} --token={1}".format(cmd, token)
+    check_call(split(cmd))
     # Create a default context with the cluster.
     cmd = 'kubectl config --kubeconfig={0} set-context {1} ' \
           '--cluster={2} --user={3}'

cluster/juju/layers/kubernetes-master/actions/namespace-create

 #!/usr/bin/env python3
-
+import os
 from yaml import safe_load as load
 from charmhelpers.core.hookenv import (
     action_get,
@@ -11,6 +11,9 @@ from charms.templating.jinja2 import render
 from subprocess import check_output
 
 
+os.environ['PATH'] += os.pathsep + os.path.join(os.sep, 'snap', 'bin')
+
+
 def kubectl(args):
     cmd = ['kubectl'] + args
     return check_output(cmd)

cluster/juju/layers/kubernetes-master/config.yaml

@@ -26,3 +26,8 @@ options:
     default: "stable"
     description: |
       Snap channel to install Kubernetes master services from
+  client_password:
+    type: string
+    default: ""
+    description: |
+      Password to be used for admin user (leave empty for random password).

cluster/juju/layers/kubernetes-worker/layer.yaml

@@ -22,6 +22,7 @@ options:
       - 'ceph-common'
       - 'nfs-common'
       - 'socat'
+      - 'virt-what'
   tls-client:
     ca_certificate_path: '/root/cdk/ca.crt'
     server_certificate_path: '/root/cdk/server.crt'

cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py

@@ -149,6 +149,7 @@ def install_snaps():
     hookenv.status_set('maintenance', 'Installing kube-proxy snap')
     snap.install('kube-proxy', channel=channel, classic=True)
     set_state('kubernetes-worker.snaps.installed')
+    set_state('kubernetes-worker.restart-needed')
     remove_state('kubernetes-worker.snaps.upgrade-needed')
     remove_state('kubernetes-worker.snaps.upgrade-specified')
 
@@ -157,15 +158,15 @@ def install_snaps():
 def shutdown():
     ''' When this unit is destroyed:
         - delete the current node
-        - stop the kubelet service
-        - stop the kube-proxy service
-        - remove the 'kubernetes-worker.cni-plugins.installed' state
+        - stop the worker services
     '''
-    if os.path.isfile(kubeconfig_path):
-        kubectl('delete', 'node', gethostname())
-    service_stop('kubelet')
-    service_stop('kube-proxy')
-    remove_state('kubernetes-worker.cni-plugins.installed')
+    try:
+        if os.path.isfile(kubeconfig_path):
+            kubectl('delete', 'node', gethostname())
+    except CalledProcessError:
+        hookenv.log('Failed to unregister node.')
+    service_stop('snap.kubelet.daemon')
+    service_stop('snap.kube-proxy.daemon')
 
 
 @when('docker.available')
@@ -303,9 +304,10 @@ def watch_for_changes(kube_api, kube_control, cni):
 @when('kubernetes-worker.snaps.installed', 'kube-api-endpoint.available',
       'tls_client.ca.saved', 'tls_client.client.certificate.saved',
       'tls_client.client.key.saved', 'tls_client.server.certificate.saved',
-      'tls_client.server.key.saved', 'kube-control.dns.available',
+      'tls_client.server.key.saved',
+      'kube-control.dns.available', 'kube-control.auth.available',
       'cni.available', 'kubernetes-worker.restart-needed')
-def start_worker(kube_api, kube_control, cni):
+def start_worker(kube_api, kube_control, auth_control, cni):
     ''' Start kubelet using the provided API and DNS info.'''
     servers = get_kube_api_servers(kube_api)
     # Note that the DNS server doesn't necessarily exist at this point. We know
@@ -320,10 +322,13 @@ def start_worker(kube_api, kube_control, cni):
         hookenv.log('Waiting for cluster cidr.')
         return
 
+    creds = kube_control.get_auth_credentials()
+    data_changed('kube-control.creds', creds)
+
     # set --allow-privileged flag for kubelet
     set_privileged()
 
-    create_config(random.choice(servers))
+    create_config(random.choice(servers), creds)
     configure_worker_services(servers, dns, cluster_cidr)
     set_state('kubernetes-worker.config.created')
     restart_unit_services()
@@ -429,27 +434,25 @@ def arch():
     return architecture
 
 
-def create_config(server):
+def create_config(server, creds):
     '''Create a kubernetes configuration for the worker unit.'''
     # Get the options from the tls-client layer.
     layer_options = layer.options('tls-client')
     # Get all the paths to the tls information required for kubeconfig.
     ca = layer_options.get('ca_certificate_path')
-    key = layer_options.get('client_key_path')
-    cert = layer_options.get('client_certificate_path')
 
     # Create kubernetes configuration in the default location for ubuntu.
-    create_kubeconfig('/home/ubuntu/.kube/config', server, ca, key, cert,
-                      user='ubuntu')
+    create_kubeconfig('/home/ubuntu/.kube/config', server, ca,
+                      token=creds['client_token'], user='ubuntu')
     # Make the config dir readable by the ubuntu users so juju scp works.
     cmd = ['chown', '-R', 'ubuntu:ubuntu', '/home/ubuntu/.kube']
     check_call(cmd)
     # Create kubernetes configuration in the default location for root.
-    create_kubeconfig('/root/.kube/config', server, ca, key, cert,
-                      user='root')
+    create_kubeconfig('/root/.kube/config', server, ca,
+                      token=creds['client_token'], user='root')
     # Create kubernetes configuration for kubelet, and kube-proxy services.
-    create_kubeconfig(kubeconfig_path, server, ca, key, cert,
-                      user='kubelet')
+    create_kubeconfig(kubeconfig_path, server, ca,
+                      token=creds['kubelet_token'], user='kubelet')
 
 
 def configure_worker_services(api_servers, dns, cluster_cidr):
@@ -464,7 +467,6 @@ def configure_worker_services(api_servers, dns, cluster_cidr):
     kubelet_opts.add('require-kubeconfig', 'true')
     kubelet_opts.add('kubeconfig', kubeconfig_path)
     kubelet_opts.add('network-plugin', 'cni')
-    kubelet_opts.add('logtostderr', 'true')
     kubelet_opts.add('v', '0')
     kubelet_opts.add('address', '0.0.0.0')
     kubelet_opts.add('port', '10250')
@@ -474,6 +476,7 @@ def configure_worker_services(api_servers, dns, cluster_cidr):
     kubelet_opts.add('client-ca-file', ca_cert_path)
     kubelet_opts.add('tls-cert-file', server_cert_path)
     kubelet_opts.add('tls-private-key-file', server_key_path)
+    kubelet_opts.add('logtostderr', 'true')
 
     kube_proxy_opts = FlagManager('kube-proxy')
     kube_proxy_opts.add('cluster-cidr', cluster_cidr)
@@ -482,25 +485,49 @@ def configure_worker_services(api_servers, dns, cluster_cidr):
     kube_proxy_opts.add('v', '0')
     kube_proxy_opts.add('master', random.choice(api_servers), strict=True)
 
+    if b'lxc' in check_output('virt-what', shell=True):
+        kube_proxy_opts.add('conntrack-max-per-core', '0')
+
     cmd = ['snap', 'set', 'kubelet'] + kubelet_opts.to_s().split(' ')
     check_call(cmd)
     cmd = ['snap', 'set', 'kube-proxy'] + kube_proxy_opts.to_s().split(' ')
     check_call(cmd)
 
 
-def create_kubeconfig(kubeconfig, server, ca, key, certificate, user='ubuntu',
-                      context='juju-context', cluster='juju-cluster'):
+def create_kubeconfig(kubeconfig, server, ca, key=None, certificate=None,
+                      user='ubuntu', context='juju-context',
+                      cluster='juju-cluster', password=None, token=None):
     '''Create a configuration for Kubernetes based on path using the supplied
     arguments for values of the Kubernetes server, CA, key, certificate, user
     context and cluster.'''
+    if not key and not certificate and not password and not token:
+        raise ValueError('Missing authentication mechanism.')
+
+    # token and password are mutually exclusive. Error early if both are
+    # present. The developer has requested an impossible situation.
+    # see: kubectl config set-credentials --help
+    if token and password:
+        raise ValueError('Token and Password are mutually exclusive.')
     # Create the config file with the address of the master server.
     cmd = 'kubectl config --kubeconfig={0} set-cluster {1} ' \
           '--server={2} --certificate-authority={3} --embed-certs=true'
     check_call(split(cmd.format(kubeconfig, cluster, server, ca)))
+    # Delete old users
+    cmd = 'kubectl config --kubeconfig={0} unset users'
+    check_call(split(cmd.format(kubeconfig)))
     # Create the credentials using the client flags.
-    cmd = 'kubectl config --kubeconfig={0} set-credentials {1} ' \
-          '--client-key={2} --client-certificate={3} --embed-certs=true'
-    check_call(split(cmd.format(kubeconfig, user, key, certificate)))
+    cmd = 'kubectl config --kubeconfig={0} ' \
+          'set-credentials {1} '.format(kubeconfig, user)
+
+    if key and certificate:
+        cmd = '{0} --client-key={1} --client-certificate={2} '\
+              '--embed-certs=true'.format(cmd, key, certificate)
+    if password:
+        cmd = "{0} --username={1} --password={2}".format(cmd, user, password)
+    # This is mutually exclusive from password. They will not work together.
+    if token:
+        cmd = "{0} --token={1}".format(cmd, token)
+    check_call(split(cmd))
     # Create a default context with the cluster.
     cmd = 'kubectl config --kubeconfig={0} set-context {1} ' \
           '--cluster={2} --user={3}'
@@ -762,6 +789,26 @@ def notify_master_gpu_not_enabled(kube_control):
     kube_control.set_gpu(False)
 
 
+@when('kube-control.connected')
+def request_kubelet_and_proxy_credentials(kube_control):
+    """ Request kubelet node authorization with a well formed kubelet user.
+    This also implies that we are requesting kube-proxy auth. """
+
+    # The kube-cotrol interface is created to support RBAC.
+    # At this point we might as well do the right thing and return the hostname
+    # even if it will only be used when we enable RBAC
+    nodeuser = 'system:node:{}'.format(gethostname())
+    kube_control.set_auth_request(nodeuser)
+
+
+@when('kube-control.auth.available')
+def catch_change_in_creds(kube_control):
+    """Request a service restart in case credential updates were detected."""
+    creds = kube_control.get_auth_credentials()
+    if data_changed('kube-control.creds', creds):
+        set_state('kubernetes-worker.restart-needed')
+
+
 @when_not('kube-control.connected')
 def missing_kube_control():
     """Inform the operator they need to add the kube-control relation.

cluster/kubemark/gce/config-default.sh

@@ -35,7 +35,7 @@ REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false}
 PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false}
 
 MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-gci}
-NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-debian}
+NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-gci}
 MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-cos-stable-59-9460-64-0}
 MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud}
 

cluster/kubemark/pre-existing/config-default.sh

+#!/bin/bash
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Configuration for landing a Kubemark cluster on a pre-existing Kubernetes
+# cluster.
+
+# Pre-existing provider expects a MASTER_IP.
+# If you need to specify a port that's not the default (443), add it to MASTER_IP.
+#
+# Example: Connect to the Master on the secure port 6443
+#          MASTER_IP=192.168.122.5:6443
+#
+MASTER_IP="${MASTER_IP:-}"
+
+# The container registry and project given to the kubemark container:
+#   $CONTAINER_REGISTRY/$PROJECT/kubemark
+#
+CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-}"
+PROJECT="${PROJECT:-}"
+
+NUM_NODES="${NUM_NODES:-1}"
+
+TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
+KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-}"
+KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-}"
+MASTER_NAME="${MASTER_NAME:-}"
+USE_REAL_PROXIER="${USE_REAL_PROXIER:-true}"

cluster/log-dump.sh

@@ -224,8 +224,19 @@ function dump_nodes() {
     return
   fi
 
+  nodes_selected_for_logs=()
+  if [[ -n "${LOGDUMP_ONLY_N_RANDOM_NODES:-}" ]]; then
+    # We randomly choose 'LOGDUMP_ONLY_N_RANDOM_NODES' many nodes for fetching logs.
+    for index in `shuf -i 0-$(( ${#node_names[*]} - 1 )) -n ${LOGDUMP_ONLY_N_RANDOM_NODES}`
+    do
+      nodes_selected_for_logs+=("${node_names[$index]}")
+    done
+  else
+    nodes_selected_for_logs=( "${node_names[@]}" )
+  fi
+
   proc=${max_scp_processes}
-  for node_name in "${node_names[@]}"; do
+  for node_name in "${nodes_selected_for_logs[@]}"; do
     node_dir="${report_dir}/${node_name}"
     mkdir -p "${node_dir}"
     # Save logs in the background. This speeds up things when there are

cluster/log-dump/logexporter-daemonset.yaml

+# Template job config for running the log exporter on the cluster as a daemonset.
+# Creates everything within 'logexporter' namespace.
+#
+# Note: Since daemonsets have "AlwaysRestart" policy for pods, we provide a long
+# sleep-duration (24 hr) to the logexporter pods so they don't finish the work and
+# get restarted while some pods are still running. So it is your duty to detect
+# the work has been done (or use some timeout) and delete the daemonset yourself.
+
+apiVersion: v1
+kind: Namespace
+metadata:
+    name: logexporter
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: google-service-account
+  namespace: logexporter
+type: Opaque
+data:
+  service-account.json: {{.ServiceAccountCredentials}}
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: logexporter
+  namespace: logexporter
+spec:
+  template:
+    metadata:
+      labels:
+        app: logexporter
+    spec:
+      containers:
+      - name: logexporter-test
+        image: gcr.io/google-containers/logexporter:v0.1.0
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        command:
+        - logexporter
+        - --node-name=$(NODE_NAME)
+        - --cloud-provider={{.CloudProvider}}
+        - --gcs-path={{.GCSPath}}
+        - --gcloud-auth-file-path=/etc/service-account/service-account.json
+        - --enable-hollow-node-logs={{.EnableHollowNodeLogs}}
+        - --sleep-duration=24h
+        - --alsologtostderr
+        volumeMounts:
+        - mountPath: /etc/service-account
+          name: service
+          readOnly: true
+        - mountPath: /var/log
+          name: varlog
+          readOnly: true
+        - mountPath: /workspace/etc
+          name: hostetc
+          readOnly: true
+        resources:
+          requests:
+            cpu: 10m
+            memory: 10Mi
+      volumes:
+      - name: service
+        secret:
+          secretName: google-service-account
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: hostetc
+        hostPath:
+          path: /etc

cluster/openstack-heat/config-default.sh

@@ -43,6 +43,9 @@ CLUSTER_IP_RANGE=${CLUSTER_IP_RANGE:-10.244.0.0/16}
 
 SWIFT_SERVER_URL=${SWIFT_SERVER_URL:-}
 
+# The name of the object store container to use
+SWIFT_OBJECT_STORE=${SWIFT_OBJECT_STORE:-kubernetes}
+
 # Flag indicates if new image must be created. If 'false' then image with IMAGE_ID will be used.
 # If 'true' then new image will be created from file config-image.sh
 CREATE_IMAGE=${CREATE_IMAGE:-true} # use "true" for devstack

cluster/openstack-heat/util.sh

@@ -108,7 +108,7 @@ function create-stack() {
 #   ROOT
 #   KUBERNETES_RELEASE_TAR
 function upload-resources() {
-  swift post kubernetes --read-acl '.r:*,.rlistings'
+  swift post ${SWIFT_OBJECT_STORE} --read-acl '.r:*,.rlistings'
 
   locations=(
     "${ROOT}/../../_output/release-tars/${KUBERNETES_RELEASE_TAR}"
@@ -119,11 +119,11 @@ function upload-resources() {
   RELEASE_TAR_PATH=$(dirname ${RELEASE_TAR_LOCATION})
 
   echo "[INFO] Uploading ${KUBERNETES_RELEASE_TAR}"
-  swift upload kubernetes ${RELEASE_TAR_PATH}/${KUBERNETES_RELEASE_TAR} \
+  swift upload ${SWIFT_OBJECT_STORE} ${RELEASE_TAR_PATH}/${KUBERNETES_RELEASE_TAR} \
     --object-name kubernetes-server.tar.gz
 
   echo "[INFO] Uploading kubernetes-salt.tar.gz"
-  swift upload kubernetes ${RELEASE_TAR_PATH}/kubernetes-salt.tar.gz \
+  swift upload ${SWIFT_OBJECT_STORE} ${RELEASE_TAR_PATH}/kubernetes-salt.tar.gz \
     --object-name kubernetes-salt.tar.gz
 }
 
@@ -196,7 +196,7 @@ function run-heat-script() {
     fi
     SWIFT_SERVER_URL=$(openstack catalog show object-store --format value | egrep -o "$rgx" | cut -d" " -f2 | head -n 1)
   fi
-  local swift_repo_url="${SWIFT_SERVER_URL}/kubernetes"
+  local swift_repo_url="${SWIFT_SERVER_URL}/${SWIFT_OBJECT_STORE}"
 
   if [ $CREATE_IMAGE = true ]; then
     echo "[INFO] Retrieve new image ID"

cluster/pre-existing/util.sh

+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A library of helper functions for landing kubemark containers on a
+# pre-existing Kubernetes master. See test/kubemark/pre-existing/README.md
+# for me details on using a pre-existing provider.
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+
+source "${KUBE_ROOT}/cluster/common.sh"
+source "${KUBE_ROOT}/hack/lib/util.sh"
+
+function detect-project() {
+  if [[ -z "${MASTER_IP:-}" ]]; then
+    echo "Set 'MASTER_IP' to the instance assigned to be the Kubernetes master" 1>&2
+    exit 1
+  fi
+
+  if [[ -z "${PROJECT:-}" ]]; then
+    echo "Set 'PROJECT' to the name of the container project: $CONTAINER_REGISTRY/$PROJECT/kubemark" >&2
+    exit 1
+  fi
+
+  if [[ -z "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
+    cluster_range=$(echo "${MASTER_IP}" | awk -F '.' '{printf("%d.%d.%d.0", $1, $2, $3)}')
+    SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-$cluster_range/16}"
+  fi
+}
+
+function create-certs {
+  rm /tmp/kubeconfig
+
+  execute-cmd-on-pre-existing-master-with-retries "sudo cat /etc/kubernetes/admin.conf" > /tmp/kubeconfig
+  CA_CERT_BASE64=$(cat /tmp/kubeconfig | grep certificate-authority | awk '{print $2}' | head -n 1)
+  KUBELET_CERT_BASE64=$(cat /tmp/kubeconfig | grep client-certificate-data | awk '{print $2}' | head -n 1)
+  KUBELET_KEY_BASE64=$(cat /tmp/kubeconfig | grep client-key-data | awk '{print $2}' | head -n 1)
+
+  # Local kubeconfig.kubemark vars
+  KUBECFG_CERT_BASE64="${KUBELET_CERT_BASE64}"
+  KUBECFG_KEY_BASE64="${KUBELET_KEY_BASE64}"
+
+  # The pre-existing Kubernetes master already has these setup
+  # Set these vars but don't use them
+  CA_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+  MASTER_CERT_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+  MASTER_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+  KUBEAPISERVER_CERT_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+  KUBEAPISERVER_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+}

cluster/vagrant/OWNERS

@@ -35,4 +35,3 @@ reviewers:
 - k82cn
 - caseydavenport
 - johscheuer
-- rjnagal

cluster/validate-cluster.sh

@@ -57,7 +57,7 @@ if [[ "${KUBERNETES_PROVIDER:-}" == "gce" ]]; then
   # In multizone mode we need to add instances for all nodes in the region.
   if [[ "${MULTIZONE:-}" == "true" ]]; then
     EXPECTED_NUM_NODES=$(gcloud -q compute instances list --project="${PROJECT}" --format=[no-heading] --regexp="${NODE_INSTANCE_PREFIX}.*" \
-      --zones=$(gcloud -q compute zones list --project="${PROJECT}" --filter=region=${REGION} --format=[no-heading]\(name\) | tr "\n" "," | sed  "s/,$//") | wc -l)
+      --zones=$(gcloud -q compute zones list --project="${PROJECT}" --filter=region=${REGION} --format=csv[no-heading]\(name\) | tr "\n" "," | sed  "s/,$//") | wc -l)
     echo "Computing number of nodes, NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX}, REGION=${REGION}, EXPECTED_NUM_NODES=${EXPECTED_NUM_NODES}"
   fi
 fi

cmd/cloud-controller-manager/app/BUILD

@@ -16,8 +16,6 @@ go_library(
         "//pkg/api:go_default_library",
         "//pkg/client/clientset_generated/clientset:go_default_library",
         "//pkg/client/informers/informers_generated/externalversions:go_default_library",
-        "//pkg/client/leaderelection:go_default_library",
-        "//pkg/client/leaderelection/resourcelock:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//pkg/controller:go_default_library",
         "//pkg/controller/cloud:go_default_library",
@@ -32,9 +30,12 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+        "//vendor/k8s.io/client-go/tools/leaderelection:go_default_library",
+        "//vendor/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library",
         "//vendor/k8s.io/client-go/tools/record:go_default_library",
     ],
 )

cmd/cloud-controller-manager/app/controllermanager.go

@@ -30,16 +30,17 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/client-go/kubernetes"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions"
-	"k8s.io/kubernetes/pkg/client/leaderelection"
-	"k8s.io/kubernetes/pkg/client/leaderelection/resourcelock"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/controller"
 	nodecontroller "k8s.io/kubernetes/pkg/controller/cloud"
@@ -102,7 +103,7 @@ func Run(s *options.CloudControllerManagerServer, cloud cloudprovider.Interface)
 	if err != nil {
 		glog.Fatalf("Invalid API configuration: %v", err)
 	}
-	leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election"))
+	leaderElectionClient := kubernetes.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election"))
 
 	// Start the external controller manager server
 	go func() {
@@ -253,9 +254,8 @@ func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restc
 
 	// If apiserver is not running we should wait for some time and fail only then. This is particularly
 	// important when we start apiserver and controller manager at the same time.
-	var versionStrings []string
 	err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
-		if versionStrings, err = restclient.ServerAPIVersions(kubeconfig); err == nil {
+		if _, err = restclient.ServerAPIVersions(kubeconfig); err == nil {
 			return true, nil
 		}
 		glog.Errorf("Failed to get api versions from server: %v", err)

cmd/cloud-controller-manager/app/options/BUILD

@@ -13,7 +13,7 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//pkg/apis/componentconfig:go_default_library",
-        "//pkg/client/leaderelection:go_default_library",
+        "//pkg/client/leaderelectionconfig:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/master/ports:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",

cmd/cloud-controller-manager/app/options/options.go

@@ -22,7 +22,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
-	"k8s.io/kubernetes/pkg/client/leaderelection"
+	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
 	"k8s.io/kubernetes/pkg/master/ports"
 
 	// add the kubernetes feature gates
@@ -56,7 +56,7 @@ func NewCloudControllerManagerServer() *CloudControllerManagerServer {
 			ContentType:             "application/vnd.kubernetes.protobuf",
 			KubeAPIQPS:              20.0,
 			KubeAPIBurst:            30,
-			LeaderElection:          leaderelection.DefaultLeaderElectionConfiguration(),
+			LeaderElection:          leaderelectionconfig.DefaultLeaderElectionConfiguration(),
 			ControllerStartInterval: metav1.Duration{Duration: 0 * time.Second},
 		},
 		NodeStatusUpdateFrequency: metav1.Duration{Duration: 5 * time.Minute},
@@ -90,7 +90,7 @@ func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
 	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
 	fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
 
-	leaderelection.BindFlags(&s.LeaderElection, fs)
+	leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
 
 	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }

cmd/genkubedocs/BUILD

@@ -19,6 +19,7 @@ go_library(
     srcs = ["gen_kube_docs.go"],
     tags = ["automanaged"],
     deps = [
+        "//cmd/cloud-controller-manager/app:go_default_library",
         "//cmd/genutils:go_default_library",
         "//cmd/kube-apiserver/app:go_default_library",
         "//cmd/kube-controller-manager/app:go_default_library",

cmd/genkubedocs/gen_kube_docs.go

@@ -21,6 +21,7 @@ import (
 	"os"
 
 	"github.com/spf13/cobra/doc"
+	ccmapp "k8s.io/kubernetes/cmd/cloud-controller-manager/app"
 	"k8s.io/kubernetes/cmd/genutils"
 	apiservapp "k8s.io/kubernetes/cmd/kube-apiserver/app"
 	cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
@@ -56,6 +57,10 @@ func main() {
 		// generate docs for kube-controller-manager
 		controllermanager := cmapp.NewControllerManagerCommand()
 		doc.GenMarkdownTree(controllermanager, outDir)
+	case "cloud-controller-manager":
+		// generate docs for cloud-controller-manager
+		cloudcontrollermanager := ccmapp.NewCloudControllerManagerCommand()
+		doc.GenMarkdownTree(cloudcontrollermanager, outDir)
 	case "kube-proxy":
 		// generate docs for kube-proxy
 		proxy := proxyapp.NewProxyCommand()

cmd/genman/BUILD

@@ -19,6 +19,7 @@ go_library(
     srcs = ["gen_kube_man.go"],
     tags = ["automanaged"],
     deps = [
+        "//cmd/cloud-controller-manager/app:go_default_library",
         "//cmd/genutils:go_default_library",
         "//cmd/kube-apiserver/app:go_default_library",
         "//cmd/kube-controller-manager/app:go_default_library",

cmd/genman/gen_kube_man.go

@@ -26,6 +26,7 @@ import (
 	mangen "github.com/cpuguy83/go-md2man/md2man"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	ccmapp "k8s.io/kubernetes/cmd/cloud-controller-manager/app"
 	"k8s.io/kubernetes/cmd/genutils"
 	apiservapp "k8s.io/kubernetes/cmd/kube-apiserver/app"
 	cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
@@ -73,6 +74,13 @@ func main() {
 		for _, c := range controllermanager.Commands() {
 			genMarkdown(c, "kube-controller-manager", outDir)
 		}
+	case "cloud-controller-manager":
+		//generate manpage for cloud-controller-manager
+		controllermanager := ccmapp.NewCloudControllerManagerCommand()
+		genMarkdown(controllermanager, "", outDir)
+		for _, c := range controllermanager.Commands() {
+			genMarkdown(c, "cloud-controller-manager", outDir)
+		}
 	case "kube-proxy":
 		// generate manpage for kube-proxy
 		proxy := proxyapp.NewProxyCommand()

cmd/kube-apiserver/app/BUILD

@@ -39,7 +39,7 @@ go_library(
         "//pkg/kubeapiserver/options:go_default_library",
         "//pkg/kubeapiserver/server:go_default_library",
         "//pkg/master:go_default_library",
-        "//pkg/master/thirdparty:go_default_library",
+        "//pkg/master/controller/crdregistration:go_default_library",
         "//pkg/master/tunneler:go_default_library",
         "//pkg/quota/install:go_default_library",
         "//pkg/registry/cachesize:go_default_library",
@@ -89,7 +89,6 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/filters:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
@@ -119,6 +118,7 @@ filegroup(
         ":package-srcs",
         "//cmd/kube-apiserver/app/options:all-srcs",
         "//cmd/kube-apiserver/app/preflight:all-srcs",
+        "//cmd/kube-apiserver/app/testing:all-srcs",
     ],
     tags = ["automanaged"],
 )

cmd/kube-apiserver/app/aggregator.go

@@ -41,8 +41,7 @@ import (
 	apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion"
 	"k8s.io/kube-aggregator/pkg/controllers/autoregister"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
-	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
-	"k8s.io/kubernetes/pkg/master/thirdparty"
+	"k8s.io/kubernetes/pkg/master/controller/crdregistration"
 )
 
 func createAggregatorConfig(kubeAPIServerConfig genericapiserver.Config, commandOptions *options.ServerRunOptions, externalInformers kubeexternalinformers.SharedInformerFactory, serviceResolver aggregatorapiserver.ServiceResolver, proxyTransport *http.Transport) (*aggregatorapiserver.Config, error) {
@@ -85,7 +84,7 @@ func createAggregatorConfig(kubeAPIServerConfig genericapiserver.Config, command
 	return aggregatorConfig, nil
 }
 
-func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget, kubeInformers informers.SharedInformerFactory, apiExtensionInformers apiextensionsinformers.SharedInformerFactory) (*aggregatorapiserver.APIAggregator, error) {
+func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget, apiExtensionInformers apiextensionsinformers.SharedInformerFactory) (*aggregatorapiserver.APIAggregator, error) {
 	aggregatorServer, err := aggregatorConfig.Complete().NewWithDelegate(delegateAPIServer)
 	if err != nil {
 		return nil, err
@@ -98,14 +97,13 @@ func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delega
 	}
 	autoRegistrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().InternalVersion().APIServices(), apiRegistrationClient)
 	apiServices := apiServicesToRegister(delegateAPIServer, autoRegistrationController)
-	tprRegistrationController := thirdparty.NewAutoRegistrationController(
-		kubeInformers.Extensions().InternalVersion().ThirdPartyResources(),
+	crdRegistrationController := crdregistration.NewAutoRegistrationController(
 		apiExtensionInformers.Apiextensions().InternalVersion().CustomResourceDefinitions(),
 		autoRegistrationController)
 
 	aggregatorServer.GenericAPIServer.AddPostStartHook("kube-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error {
 		go autoRegistrationController.Run(5, context.StopCh)
-		go tprRegistrationController.Run(5, context.StopCh)
+		go crdRegistrationController.Run(5, context.StopCh)
 		return nil
 	})
 	aggregatorServer.GenericAPIServer.AddHealthzChecks(healthz.NamedCheck("autoregister-completion", func(r *http.Request) error {

cmd/kube-apiserver/app/options/options.go

@@ -143,7 +143,7 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
 		"Amount of time to retain events.")
 
 	fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged,
-		"If true, allow privileged containers.")
+		"If true, allow privileged containers. [default=false]")
 
 	fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler,
 		"If true, install a /logs handler for the apiserver logs.")

cmd/kube-apiserver/app/options/validation.go

@@ -63,6 +63,9 @@ func (options *ServerRunOptions) Validate() []error {
 	if errs := options.Authentication.Validate(); len(errs) > 0 {
 		errors = append(errors, errs...)
 	}
+	if errs := options.Audit.Validate(); len(errs) > 0 {
+		errors = append(errors, errs...)
+	}
 	if errs := options.InsecureServing.Validate("insecure-port"); len(errs) > 0 {
 		errors = append(errors, errs...)
 	}

cmd/kube-apiserver/app/server.go

@@ -47,7 +47,6 @@ import (
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
-	genericregistry "k8s.io/apiserver/pkg/registry/generic"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/filters"
 	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
@@ -111,30 +110,39 @@ func Run(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) error {
 	// To help debugging, immediately log version
 	glog.Infof("Version: %+v", version.Get())
 
-	nodeTunneler, proxyTransport, err := CreateNodeDialer(runOptions)
+	server, err := CreateServerChain(runOptions, stopCh)
 	if err != nil {
 		return err
 	}
 
+	return server.PrepareRun().Run(stopCh)
+}
+
+// CreateServerChain creates the apiservers connected via delegation.
+func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) (*genericapiserver.GenericAPIServer, error) {
+	nodeTunneler, proxyTransport, err := CreateNodeDialer(runOptions)
+	if err != nil {
+		return nil, err
+	}
 	kubeAPIServerConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, err := CreateKubeAPIServerConfig(runOptions, nodeTunneler, proxyTransport)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	// TPRs are enabled and not yet beta, since this these are the successor, they fall under the same enablement rule
 	// If additional API servers are added, they should be gated.
 	apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, runOptions)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.EmptyDelegate)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer, sharedInformers, apiExtensionsConfig.CRDRESTOptionsGetter)
+	kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer, sharedInformers)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	// if we're starting up a hacked up version of this API server for a weird test case,
@@ -143,11 +151,11 @@ func Run(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) error {
 		if insecureServingOptions != nil {
 			insecureHandlerChain := kubeserver.BuildInsecureHandlerChain(kubeAPIServer.GenericAPIServer.UnprotectedHandler(), kubeAPIServerConfig.GenericConfig)
 			if err := kubeserver.NonBlockingRun(insecureServingOptions, insecureHandlerChain, stopCh); err != nil {
-				return err
+				return nil, err
 			}
 		}
 
-		return kubeAPIServer.GenericAPIServer.PrepareRun().Run(stopCh)
+		return kubeAPIServer.GenericAPIServer, nil
 	}
 
 	// otherwise go down the normal path of standing the aggregator up in front of the API server
@@ -157,29 +165,29 @@ func Run(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) error {
 	// aggregator comes last in the chain
 	aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, runOptions, versionedInformers, serviceResolver, proxyTransport)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	aggregatorConfig.ProxyTransport = proxyTransport
 	aggregatorConfig.ServiceResolver = serviceResolver
-	aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, sharedInformers, apiExtensionsServer.Informers)
+	aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers)
 	if err != nil {
 		// we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
-		return err
+		return nil, err
 	}
 
 	if insecureServingOptions != nil {
 		insecureHandlerChain := kubeserver.BuildInsecureHandlerChain(aggregatorServer.GenericAPIServer.UnprotectedHandler(), kubeAPIServerConfig.GenericConfig)
 		if err := kubeserver.NonBlockingRun(insecureServingOptions, insecureHandlerChain, stopCh); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
-	return aggregatorServer.GenericAPIServer.PrepareRun().Run(stopCh)
+	return aggregatorServer.GenericAPIServer, nil
 }
 
 // CreateKubeAPIServer creates and wires a workable kube-apiserver
-func CreateKubeAPIServer(kubeAPIServerConfig *master.Config, delegateAPIServer genericapiserver.DelegationTarget, sharedInformers informers.SharedInformerFactory, crdRESTOptionsGetter genericregistry.RESTOptionsGetter) (*master.Master, error) {
-	kubeAPIServer, err := kubeAPIServerConfig.Complete().New(delegateAPIServer, crdRESTOptionsGetter)
+func CreateKubeAPIServer(kubeAPIServerConfig *master.Config, delegateAPIServer genericapiserver.DelegationTarget, sharedInformers informers.SharedInformerFactory) (*master.Master, error) {
+	kubeAPIServer, err := kubeAPIServerConfig.Complete().New(delegateAPIServer)
 	if err != nil {
 		return nil, err
 	}
@@ -256,8 +264,10 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
 		return nil, nil, nil, nil, nil, err
 	}
 
-	if err := utilwait.PollImmediate(etcdRetryInterval, etcdRetryLimit*etcdRetryInterval, preflight.EtcdConnection{ServerList: s.Etcd.StorageConfig.ServerList}.CheckEtcdServers); err != nil {
-		return nil, nil, nil, nil, nil, fmt.Errorf("error waiting for etcd connection: %v", err)
+	if _, port, err := net.SplitHostPort(s.Etcd.StorageConfig.ServerList[0]); err == nil && port != "0" && len(port) != 0 {
+		if err := utilwait.PollImmediate(etcdRetryInterval, etcdRetryLimit*etcdRetryInterval, preflight.EtcdConnection{ServerList: s.Etcd.StorageConfig.ServerList}.CheckEtcdServers); err != nil {
+			return nil, nil, nil, nil, nil, fmt.Errorf("error waiting for etcd connection: %v", err)
+		}
 	}
 
 	capabilities.Initialize(capabilities.Capabilities{

pkg/registry/extensions/thirdpartyresourcedata/storage/BUILD → cmd/kube-apiserver/app/testing/BUILD

@@ -10,39 +10,35 @@ load(
 
 go_test(
     name = "go_default_test",
-    srcs = ["storage_test.go"],
+    srcs = ["server_test.go"],
     library = ":go_default_library",
     tags = ["automanaged"],
     deps = [
-        "//pkg/apis/extensions:go_default_library",
-        "//pkg/registry/registrytest:go_default_library",
-        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/api/apps/v1beta1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/networking/v1:go_default_library",
+        "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/dynamic:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
     ],
 )
 
 go_library(
     name = "go_default_library",
-    srcs = ["storage.go"],
+    srcs = ["testserver.go"],
     tags = ["automanaged"],
     deps = [
+        "//cmd/kube-apiserver/app:go_default_library",
+        "//cmd/kube-apiserver/app/options:go_default_library",
         "//pkg/api:go_default_library",
-        "//pkg/apis/extensions:go_default_library",
-        "//pkg/registry/cachesize:go_default_library",
-        "//pkg/registry/extensions/thirdpartyresourcedata:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/internalversion:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/registry/rest:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
     ],
 )
 

cmd/kube-apiserver/app/testing/server_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	appsv1beta1 "k8s.io/api/apps/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+)
+
+func TestRun(t *testing.T) {
+	config, tearDown := StartTestServerOrDie(t)
+	defer tearDown()
+
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	// test whether the server is really healthy after /healthz told us so
+	t.Logf("Creating Deployment directly after being healthy")
+	var replicas int32 = 1
+	_, err = client.AppsV1beta1().Deployments("default").Create(&appsv1beta1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1beta1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "test",
+		},
+		Spec: appsv1beta1.DeploymentSpec{
+			Replicas: &replicas,
+			Strategy: appsv1beta1.DeploymentStrategy{
+				Type: appsv1beta1.RollingUpdateDeploymentStrategyType,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"foo": "bar"},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "foo",
+							Image: "foo",
+						},
+					},
+				},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create deployment: %v", err)
+	}
+}
+
+func TestCRDShadowGroup(t *testing.T) {
+	config, tearDown := StartTestServerOrDie(t)
+	defer tearDown()
+
+	kubeclient, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	apiextensionsclient, err := apiextensionsclientset.NewForConfig(config)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	t.Logf("Creating a NetworkPolicy")
+	nwPolicy, err := kubeclient.NetworkingV1().NetworkPolicies("default").Create(&networkingv1.NetworkPolicy{
+		ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
+		Spec: networkingv1.NetworkPolicySpec{
+			PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}},
+			Ingress:     []networkingv1.NetworkPolicyIngressRule{},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create NetworkPolicy: %v", err)
+	}
+
+	t.Logf("Trying to shadow networking group")
+	crd := &apiextensionsv1beta1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "foos." + networkingv1.GroupName,
+		},
+		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+			Group:   networkingv1.GroupName,
+			Version: networkingv1.SchemeGroupVersion.Version,
+			Scope:   apiextensionsv1beta1.ClusterScoped,
+			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+				Plural: "foos",
+				Kind:   "Foo",
+			},
+		},
+	}
+	if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
+		t.Fatalf("Failed to create networking group CRD: %v", err)
+	}
+	if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil {
+		t.Fatalf("Failed to establish networking group CRD: %v", err)
+	}
+	// wait to give aggregator time to update
+	time.Sleep(2 * time.Second)
+
+	t.Logf("Checking that we still see the NetworkPolicy")
+	_, err = kubeclient.NetworkingV1().NetworkPolicies(nwPolicy.Namespace).Get(nwPolicy.Name, metav1.GetOptions{})
+	if err != nil {
+		t.Errorf("Failed to get NetworkPolocy: %v", err)
+	}
+
+	t.Logf("Checking that crd resource does not show up in networking group")
+	found, err := crdExistsInDiscovery(apiextensionsclient, crd)
+	if err != nil {
+		t.Fatalf("unexpected discovery error: %v", err)
+	}
+	if found {
+		t.Errorf("CRD resource shows up in discovery, but shouldn't.")
+	}
+}
+
+func TestCRD(t *testing.T) {
+	config, tearDown := StartTestServerOrDie(t)
+	defer tearDown()
+
+	apiextensionsclient, err := apiextensionsclientset.NewForConfig(config)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	t.Logf("Trying to create a custom resource without conflict")
+	crd := &apiextensionsv1beta1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "foos.cr.bar.com",
+		},
+		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+			Group:   "cr.bar.com",
+			Version: "v1",
+			Scope:   apiextensionsv1beta1.NamespaceScoped,
+			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+				Plural: "foos",
+				Kind:   "Foo",
+			},
+		},
+	}
+	if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
+		t.Fatalf("Failed to create foos.cr.bar.com CRD; %v", err)
+	}
+	if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil {
+		t.Fatalf("Failed to establish foos.cr.bar.com CRD: %v", err)
+	}
+	if err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) {
+		return crdExistsInDiscovery(apiextensionsclient, crd)
+	}); err != nil {
+		t.Fatalf("Failed to see foos.cr.bar.com in discovery: %v", err)
+	}
+
+	t.Logf("Trying to access foos.cr.bar.com with dynamic client")
+	barComConfig := *config
+	barComConfig.GroupVersion = &schema.GroupVersion{Group: "cr.bar.com", Version: "v1"}
+	barComConfig.APIPath = "/apis"
+	barComClient, err := dynamic.NewClient(&barComConfig)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	_, err = barComClient.Resource(&metav1.APIResource{Name: "foos", Namespaced: true}, "default").List(metav1.ListOptions{})
+	if err != nil {
+		t.Errorf("Failed to list foos.cr.bar.com instances: %v", err)
+	}
+}
+
+func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
+	return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
+		crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, cond := range crd.Status.Conditions {
+			switch cond.Type {
+			case apiextensionsv1beta1.Established:
+				if cond.Status == apiextensionsv1beta1.ConditionTrue {
+					return true, err
+				}
+			case apiextensionsv1beta1.NamesAccepted:
+				if cond.Status == apiextensionsv1beta1.ConditionFalse {
+					fmt.Printf("Name conflict: %v\n", cond.Reason)
+				}
+			}
+		}
+		return false, nil
+	})
+}
+
+func crdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) (bool, error) {
+	resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version)
+	if err != nil {
+		return false, nil
+	}
+	for _, resource := range resourceList.APIResources {
+		if resource.Name == crd.Spec.Names.Plural {
+			return true, nil
+		}
+	}
+	return false, nil
+}

cmd/kube-apiserver/app/testing/testserver.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+	"k8s.io/kubernetes/pkg/api"
+)
+
+// TearDownFunc is to be called to tear down a test server.
+type TearDownFunc func()
+
+// StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func
+// are returned.
+//
+// Note: we return a tear-down func instead of a stop channel because the later will leak temporariy
+// 		 files that becaues Golang testing's call to os.Exit will not give a stop channel go routine
+// 		 enough time to remove temporariy files.
+func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller TearDownFunc, err error) {
+	var tmpDir string
+	var etcdServer *etcdtesting.EtcdTestServer
+	stopCh := make(chan struct{})
+	tearDown := func() {
+		close(stopCh)
+		if etcdServer != nil {
+			etcdServer.Terminate(t)
+		}
+		if len(tmpDir) != 0 {
+			os.RemoveAll(tmpDir)
+		}
+	}
+	defer func() {
+		if tearDownForCaller == nil {
+			tearDown()
+		}
+	}()
+
+	t.Logf("Starting etcd...")
+	etcdServer, storageConfig := etcdtesting.NewUnsecuredEtcd3TestClientServer(t, api.Scheme)
+
+	tmpDir, err = ioutil.TempDir("", "kubernetes-kube-apiserver")
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to create temp dir: %v", err)
+	}
+
+	s := options.NewServerRunOptions()
+	s.InsecureServing.BindPort = 0
+	s.SecureServing.BindPort = freePort()
+	s.SecureServing.ServerCert.CertDirectory = tmpDir
+	s.ServiceClusterIPRange.IP = net.IPv4(10, 0, 0, 0)
+	s.ServiceClusterIPRange.Mask = net.CIDRMask(16, 32)
+	s.Etcd.StorageConfig = *storageConfig
+	s.Etcd.DefaultStorageMediaType = "application/json"
+	s.Admission.PluginNames = strings.Split("Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds", ",")
+
+	t.Logf("Starting kube-apiserver...")
+	runErrCh := make(chan error, 1)
+	server, err := app.CreateServerChain(s, stopCh)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to create server chain: %v", err)
+	}
+	go func(stopCh <-chan struct{}) {
+		if err := server.PrepareRun().Run(stopCh); err != nil {
+			t.Logf("kube-apiserver exited uncleanly: %v", err)
+			runErrCh <- err
+		}
+	}(stopCh)
+
+	t.Logf("Waiting for /healthz to be ok...")
+	client, err := kubernetes.NewForConfig(server.LoopbackClientConfig)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to create a client: %v", err)
+	}
+	err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) {
+		select {
+		case err := <-runErrCh:
+			return false, err
+		default:
+		}
+
+		result := client.CoreV1Client.RESTClient().Get().AbsPath("/healthz").Do()
+		status := 0
+		result.StatusCode(&status)
+		if status == 200 {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to wait for /healthz to return ok: %v", err)
+	}
+
+	// from here the caller must call tearDown
+	return server.LoopbackClientConfig, tearDown, nil
+}
+
+// StartTestServerOrDie calls StartTestServer with up to 5 retries on bind error and dies with
+// t.Fatal if it does not succeed.
+func StartTestServerOrDie(t *testing.T) (*restclient.Config, TearDownFunc) {
+	// retry test because the bind might fail due to a race with another process
+	// binding to the port. We cannot listen to :0 (then the kernel would give us
+	// a port which is free for sure), so we need this workaround.
+	for retry := 0; retry < 5 && !t.Failed(); retry++ {
+		config, td, err := StartTestServer(t)
+		if err == nil {
+			return config, td
+		}
+		if err != nil && !strings.Contains(err.Error(), "bind") {
+			break
+		}
+		t.Logf("Bind error, retrying...")
+	}
+
+	t.Fatalf("Failed to launch server")
+	return nil, nil
+}
+
+func freePort() int {
+	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
+	if err != nil {
+		panic(err)
+	}
+
+	l, err := net.ListenTCP("tcp", addr)
+	if err != nil {
+		panic(err)
+	}
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port
+}

cmd/kube-controller-manager/OWNERS

@@ -5,7 +5,6 @@ approvers:
 reviewers:
 - '249043822'
 - a-robinson
-- bprashanth
 - brendandburns
 - caesarxuchao
 - cjcullen

cmd/kube-controller-manager/app/BUILD

@@ -30,8 +30,6 @@ go_library(
         "//pkg/apis/componentconfig:go_default_library",
         "//pkg/client/clientset_generated/clientset:go_default_library",
         "//pkg/client/informers/informers_generated/externalversions:go_default_library",
-        "//pkg/client/leaderelection:go_default_library",
-        "//pkg/client/leaderelection/resourcelock:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//pkg/cloudprovider/providers:go_default_library",
         "//pkg/cloudprovider/providers/aws:go_default_library",
@@ -105,9 +103,12 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/discovery:go_default_library",
         "//vendor/k8s.io/client-go/dynamic:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+        "//vendor/k8s.io/client-go/tools/leaderelection:go_default_library",
+        "//vendor/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library",
         "//vendor/k8s.io/client-go/tools/record:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
         "//vendor/k8s.io/metrics/pkg/client/clientset_generated/clientset/typed/metrics/v1alpha1:go_default_library",

cmd/kube-controller-manager/app/controllermanager.go

@@ -40,18 +40,19 @@ import (
 
 	clientv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/kubernetes"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
 	certutil "k8s.io/client-go/util/cert"
 
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions"
-	"k8s.io/kubernetes/pkg/client/leaderelection"
-	"k8s.io/kubernetes/pkg/client/leaderelection/resourcelock"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/controller"
 	serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
@@ -127,7 +128,7 @@ func Run(s *options.CMServer) error {
 	if err != nil {
 		glog.Fatalf("Invalid API configuration: %v", err)
 	}
-	leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election"))
+	leaderElectionClient := kubernetes.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election"))
 
 	go func() {
 		mux := http.NewServeMux()

cmd/kube-controller-manager/app/extensions.go

@@ -24,7 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/kubernetes/pkg/controller/daemon"
 	"k8s.io/kubernetes/pkg/controller/deployment"
-	replicaset "k8s.io/kubernetes/pkg/controller/replicaset"
+	"k8s.io/kubernetes/pkg/controller/replicaset"
 )
 
 func startDaemonSetController(ctx ControllerContext) (bool, error) {

cmd/kube-controller-manager/app/options/BUILD

@@ -13,7 +13,7 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//pkg/apis/componentconfig:go_default_library",
-        "//pkg/client/leaderelection:go_default_library",
+        "//pkg/client/leaderelectionconfig:go_default_library",
         "//pkg/controller/garbagecollector:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/master/ports:go_default_library",

cmd/kube-controller-manager/app/options/options.go

@@ -28,7 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
-	"k8s.io/kubernetes/pkg/client/leaderelection"
+	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
 	"k8s.io/kubernetes/pkg/controller/garbagecollector"
 	"k8s.io/kubernetes/pkg/master/ports"
 
@@ -106,7 +106,7 @@ func NewCMServer() *CMServer {
 			ContentType:                           "application/vnd.kubernetes.protobuf",
 			KubeAPIQPS:                            20.0,
 			KubeAPIBurst:                          30,
-			LeaderElection:                        leaderelection.DefaultLeaderElectionConfiguration(),
+			LeaderElection:                        leaderelectionconfig.DefaultLeaderElectionConfiguration(),
 			ControllerStartInterval:               metav1.Duration{Duration: 0 * time.Second},
 			EnableGarbageCollector:                true,
 			ConcurrentGCSyncs:                     20,
@@ -225,9 +225,9 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled
 	fs.BoolVar(&s.DisableAttachDetachReconcilerSync, "disable-attach-detach-reconcile-sync", false, "Disable volume attach detach reconciler sync. Disabling this may cause volumes to be mismatched with pods. Use wisely.")
 	fs.DurationVar(&s.ReconcilerSyncLoopPeriod.Duration, "attach-detach-reconcile-sync-period", s.ReconcilerSyncLoopPeriod.Duration, "The reconciler sync wait time between volume attach detach. This duration must be larger than one second, and increasing this value from the default may allow for volumes to be mismatched with pods.")
 	fs.BoolVar(&s.EnableTaintManager, "enable-taint-manager", s.EnableTaintManager, "WARNING: Beta feature. If set to true enables NoExecute Taints and will evict all not-tolerating Pod running on Nodes tainted with this kind of Taints.")
-	fs.BoolVar(&s.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", s.HorizontalPodAutoscalerUseRESTClients, "WARNING: alpha feature.  If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy.  This is required for custom metrics support in the horizonal pod autoscaler.")
+	fs.BoolVar(&s.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", s.HorizontalPodAutoscalerUseRESTClients, "WARNING: alpha feature.  If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy.  This is required for custom metrics support in the horizontal pod autoscaler.")
 
-	leaderelection.BindFlags(&s.LeaderElection, fs)
+	leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
 
 	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }

cmd/kube-proxy/app/BUILD

@@ -23,6 +23,7 @@ go_library(
         "//pkg/client/informers/informers_generated/internalversion:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
         "//pkg/kubelet/qos:go_default_library",
+        "//pkg/master/ports:go_default_library",
         "//pkg/proxy:go_default_library",
         "//pkg/proxy/config:go_default_library",
         "//pkg/proxy/healthcheck:go_default_library",
@@ -72,10 +73,13 @@ go_test(
     deps = [
         "//pkg/api:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/apis/componentconfig/v1alpha1:go_default_library",
         "//pkg/util:go_default_library",
+        "//pkg/util/configz:go_default_library",
         "//pkg/util/iptables:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
     ],
 )

cmd/kube-proxy/app/server.go

@@ -53,6 +53,7 @@ import (
 	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
 	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
 	"k8s.io/kubernetes/pkg/kubelet/qos"
+	"k8s.io/kubernetes/pkg/master/ports"
 	"k8s.io/kubernetes/pkg/proxy"
 	proxyconfig "k8s.io/kubernetes/pkg/proxy/config"
 	"k8s.io/kubernetes/pkg/proxy/healthcheck"
@@ -129,6 +130,7 @@ func AddFlags(options *Options, fs *pflag.FlagSet) {
 	fs.StringVar(&options.master, "master", options.master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
 	fs.Int32Var(&options.healthzPort, "healthz-port", options.healthzPort, "The port to bind the health check server. Use 0 to disable.")
 	fs.Var(componentconfig.IPVar{Val: &options.config.HealthzBindAddress}, "healthz-bind-address", "The IP address and port for the health check server to serve on (set to 0.0.0.0 for all interfaces)")
+	fs.Var(componentconfig.IPVar{Val: &options.config.MetricsBindAddress}, "metrics-bind-address", "The IP address and port for the metrics server to serve on (set to 0.0.0.0 for all interfaces)")
 	fs.Int32Var(options.config.OOMScoreAdj, "oom-score-adj", util.Int32PtrDerefOr(options.config.OOMScoreAdj, int32(qos.KubeProxyOOMScoreAdj)), "The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]")
 	fs.StringVar(&options.config.ResourceContainer, "resource-container", options.config.ResourceContainer, "Absolute name of the resource-only container to create and run the Kube-proxy in (Default: /kube-proxy).")
 	fs.MarkDeprecated("resource-container", "This feature will be removed in a later release.")
@@ -166,7 +168,7 @@ func AddFlags(options *Options, fs *pflag.FlagSet) {
 func NewOptions() (*Options, error) {
 	o := &Options{
 		config:      new(componentconfig.KubeProxyConfiguration),
-		healthzPort: 10256,
+		healthzPort: ports.ProxyHealthzPort,
 	}
 
 	o.scheme = runtime.NewScheme()
@@ -447,7 +449,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 
 	// We omit creation of pretty much everything if we run in cleanup mode
 	if cleanupAndExit {
-		return &ProxyServer{IptInterface: iptInterface}, nil
+		return &ProxyServer{IptInterface: iptInterface, CleanupAndExit: cleanupAndExit}, nil
 	}
 
 	client, eventClient, err := createClients(config.ClientConnection, master)
@@ -627,7 +629,9 @@ func (s *ProxyServer) Run() error {
 		}
 	}
 
-	s.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: s.EventClient.Events("")})
+	if s.Broadcaster != nil && s.EventClient != nil {
+		s.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: s.EventClient.Events("")})
+	}
 
 	// Start up a healthz server if requested
 	if s.HealthzServer != nil {

cmd/kube-proxy/app/server_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package app
 
 import (
+	"errors"
 	"fmt"
 	"reflect"
 	"runtime"
@@ -27,10 +28,13 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sRuntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/diff"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
 	"k8s.io/kubernetes/pkg/util"
+	"k8s.io/kubernetes/pkg/util/configz"
 	"k8s.io/kubernetes/pkg/util/iptables"
 )
 
@@ -134,23 +138,67 @@ func Test_getProxyMode(t *testing.T) {
 	}
 }
 
-// This test verifies that Proxy Server does not crash when CleanupAndExit is true.
-func TestProxyServerWithCleanupAndExit(t *testing.T) {
-	options, err := NewOptions()
-	if err != nil {
-		t.Fatal(err)
+// TestNewOptionsFailures tests failure modes for NewOptions()
+func TestNewOptionsFailures(t *testing.T) {
+
+	// Create a fake scheme builder that generates an error
+	errString := fmt.Sprintf("Simulated error")
+	genError := func(scheme *k8sRuntime.Scheme) error {
+		return errors.New(errString)
 	}
+	fakeSchemeBuilder := k8sRuntime.NewSchemeBuilder(genError)
+
+	simulatedErrorTest := func(target string) {
+		var addToScheme *func(s *k8sRuntime.Scheme) error
+		if target == "componentconfig" {
+			addToScheme = &componentconfig.AddToScheme
+		} else {
+			addToScheme = &v1alpha1.AddToScheme
+		}
+		restoreValue := *addToScheme
+		restore := func() {
+			*addToScheme = restoreValue
+		}
+		defer restore()
+		*addToScheme = fakeSchemeBuilder.AddToScheme
+		_, err := NewOptions()
+		assert.Error(t, err, fmt.Sprintf("Simulated error in component %s", target))
+	}
+
+	// Simulate errors in calls to AddToScheme()
+	faultTargets := []string{"componentconfig", "v1alpha1"}
+	for _, target := range faultTargets {
+		simulatedErrorTest(target)
+	}
+}
 
-	options.config = &componentconfig.KubeProxyConfiguration{
-		BindAddress: "0.0.0.0",
+// This test verifies that NewProxyServer does not crash when CleanupAndExit is true.
+func TestProxyServerWithCleanupAndExit(t *testing.T) {
+	// Each bind address below is a separate test case
+	bindAddresses := []string{
+		"0.0.0.0",
+		"2001:db8::1",
 	}
-	options.CleanupAndExit = true
+	for _, addr := range bindAddresses {
+		options, err := NewOptions()
+		if err != nil {
+			t.Fatalf("Unexpected error with address %s: %v", addr, err)
+		}
+
+		options.config = &componentconfig.KubeProxyConfiguration{
+			BindAddress: addr,
+		}
+		options.CleanupAndExit = true
+
+		proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.scheme, options.master)
 
-	proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.scheme, options.master)
+		assert.Nil(t, err, "unexpected error in NewProxyServer, addr: %s", addr)
+		assert.NotNil(t, proxyserver, "nil proxy server obj, addr: %s", addr)
+		assert.NotNil(t, proxyserver.IptInterface, "nil iptables intf, addr: %s", addr)
 
-	assert.Nil(t, err)
-	assert.NotNil(t, proxyserver)
-	assert.NotNil(t, proxyserver.IptInterface)
+		// Clean up config for next test case
+		configz.Delete("componentconfig")
+	}
 }
 
 func TestGetConntrackMax(t *testing.T) {
@@ -211,16 +259,18 @@ func TestGetConntrackMax(t *testing.T) {
 	}
 }
 
+// TestLoadConfig tests proper operation of loadConfig()
 func TestLoadConfig(t *testing.T) {
-	yaml := `apiVersion: componentconfig/v1alpha1
-bindAddress: 9.8.7.6
+
+	yamlTemplate := `apiVersion: componentconfig/v1alpha1
+bindAddress: %s
 clientConnection:
   acceptContentTypes: "abc"
   burst: 100
   contentType: content-type
   kubeconfig: "/path/to/kubeconfig"
   qps: 7
-clusterCIDR: "1.2.3.0/24"
+clusterCIDR: "%s"
 configSyncPeriod: 15s
 conntrack:
   max: 4
@@ -229,7 +279,7 @@ conntrack:
   tcpCloseWaitTimeout: 10s
   tcpEstablishedTimeout: 20s
 featureGates: "all"
-healthzBindAddress: 1.2.3.4:12345
+healthzBindAddress: "%s"
 hostnameOverride: "foo"
 iptables:
   masqueradeAll: true
@@ -237,7 +287,7 @@ iptables:
   minSyncPeriod: 10s
   syncPeriod: 60s
 kind: KubeProxyConfiguration
-metricsBindAddress: 2.3.4.5:23456
+metricsBindAddress: "%s"
 mode: "iptables"
 oomScoreAdj: 17
 portRange: "2-7"
@@ -245,47 +295,104 @@ resourceContainer: /foo
 udpTimeoutMilliseconds: 123ms
 `
 
-	expected := &componentconfig.KubeProxyConfiguration{
-		BindAddress: "9.8.7.6",
-		ClientConnection: componentconfig.ClientConnectionConfiguration{
-			AcceptContentTypes: "abc",
-			Burst:              100,
-			ContentType:        "content-type",
-			KubeConfigFile:     "/path/to/kubeconfig",
-			QPS:                7,
-		},
-		ClusterCIDR:      "1.2.3.0/24",
-		ConfigSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
-		Conntrack: componentconfig.KubeProxyConntrackConfiguration{
-			Max:                   4,
-			MaxPerCore:            2,
-			Min:                   1,
-			TCPCloseWaitTimeout:   metav1.Duration{Duration: 10 * time.Second},
-			TCPEstablishedTimeout: metav1.Duration{Duration: 20 * time.Second},
+	testCases := []struct {
+		name               string
+		bindAddress        string
+		clusterCIDR        string
+		healthzBindAddress string
+		metricsBindAddress string
+	}{
+		{
+			name:               "IPv4 config",
+			bindAddress:        "9.8.7.6",
+			clusterCIDR:        "1.2.3.0/24",
+			healthzBindAddress: "1.2.3.4:12345",
+			metricsBindAddress: "2.3.4.5:23456",
 		},
-		FeatureGates:       "all",
-		HealthzBindAddress: "1.2.3.4:12345",
-		HostnameOverride:   "foo",
-		IPTables: componentconfig.KubeProxyIPTablesConfiguration{
-			MasqueradeAll: true,
-			MasqueradeBit: util.Int32Ptr(17),
-			MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
-			SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
+		{
+			name:               "IPv6 config",
+			bindAddress:        "2001:db8::1",
+			clusterCIDR:        "fd00:1::0/64",
+			healthzBindAddress: "[fd00:1::5]:12345",
+			metricsBindAddress: "[fd00:2::5]:23456",
 		},
-		MetricsBindAddress: "2.3.4.5:23456",
-		Mode:               "iptables",
-		OOMScoreAdj:        util.Int32Ptr(17),
-		PortRange:          "2-7",
-		ResourceContainer:  "/foo",
-		UDPIdleTimeout:     metav1.Duration{Duration: 123 * time.Millisecond},
 	}
 
-	options, err := NewOptions()
-	assert.NoError(t, err)
+	for _, tc := range testCases {
+		expected := &componentconfig.KubeProxyConfiguration{
+			BindAddress: tc.bindAddress,
+			ClientConnection: componentconfig.ClientConnectionConfiguration{
+				AcceptContentTypes: "abc",
+				Burst:              100,
+				ContentType:        "content-type",
+				KubeConfigFile:     "/path/to/kubeconfig",
+				QPS:                7,
+			},
+			ClusterCIDR:      tc.clusterCIDR,
+			ConfigSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
+			Conntrack: componentconfig.KubeProxyConntrackConfiguration{
+				Max:                   4,
+				MaxPerCore:            2,
+				Min:                   1,
+				TCPCloseWaitTimeout:   metav1.Duration{Duration: 10 * time.Second},
+				TCPEstablishedTimeout: metav1.Duration{Duration: 20 * time.Second},
+			},
+			FeatureGates:       "all",
+			HealthzBindAddress: tc.healthzBindAddress,
+			HostnameOverride:   "foo",
+			IPTables: componentconfig.KubeProxyIPTablesConfiguration{
+				MasqueradeAll: true,
+				MasqueradeBit: util.Int32Ptr(17),
+				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
+				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
+			},
+			MetricsBindAddress: tc.metricsBindAddress,
+			Mode:               "iptables",
+			OOMScoreAdj:        util.Int32Ptr(17),
+			PortRange:          "2-7",
+			ResourceContainer:  "/foo",
+			UDPIdleTimeout:     metav1.Duration{Duration: 123 * time.Millisecond},
+		}
+
+		options, err := NewOptions()
+		assert.NoError(t, err, "unexpected error for %s: %v", tc.name, err)
+
+		yaml := fmt.Sprintf(
+			yamlTemplate, tc.bindAddress, tc.clusterCIDR,
+			tc.healthzBindAddress, tc.metricsBindAddress)
+		config, err := options.loadConfig([]byte(yaml))
+		assert.NoError(t, err, "unexpected error for %s: %v", tc.name, err)
+		if !reflect.DeepEqual(expected, config) {
+			t.Fatalf("unexpected config for %s test, diff = %s", tc.name, diff.ObjectDiff(config, expected))
+		}
+	}
+}
 
-	config, err := options.loadConfig([]byte(yaml))
-	assert.NoError(t, err)
-	if !reflect.DeepEqual(expected, config) {
-		t.Fatalf("unexpected config, diff = %s", diff.ObjectDiff(config, expected))
+// TestLoadConfigFailures tests failure modes for loadConfig()
+func TestLoadConfigFailures(t *testing.T) {
+	testCases := []struct {
+		name   string
+		config string
+		expErr string
+	}{
+		{
+			name:   "Decode error test",
+			config: "Twas bryllyg, and ye slythy toves",
+			expErr: "could not find expected ':'",
+		},
+		{
+			name:   "Bad config type test",
+			config: "kind: KubeSchedulerConfiguration",
+			expErr: "unexpected config type",
+		},
+	}
+	version := "apiVersion: componentconfig/v1alpha1"
+	for _, tc := range testCases {
+		options, _ := NewOptions()
+		config := fmt.Sprintf("%s\n%s", version, tc.config)
+		_, err := options.loadConfig([]byte(config))
+		if assert.Error(t, err, tc.name) {
+			assert.Contains(t, err.Error(), tc.expErr, tc.name)
+		}
 	}
 }

cmd/kubeadm/OWNERS

@@ -12,3 +12,4 @@ reviewers:
 - lukemarsden
 - dmmcquay
 - krousey
+- timothysc

cmd/kubeadm/app/BUILD

@@ -35,12 +35,13 @@ filegroup(
         "//cmd/kubeadm/app/constants:all-srcs",
         "//cmd/kubeadm/app/discovery:all-srcs",
         "//cmd/kubeadm/app/images:all-srcs",
-        "//cmd/kubeadm/app/master:all-srcs",
         "//cmd/kubeadm/app/node:all-srcs",
         "//cmd/kubeadm/app/phases/addons:all-srcs",
         "//cmd/kubeadm/app/phases/apiconfig:all-srcs",
         "//cmd/kubeadm/app/phases/certs:all-srcs",
+        "//cmd/kubeadm/app/phases/controlplane:all-srcs",
         "//cmd/kubeadm/app/phases/kubeconfig:all-srcs",
+        "//cmd/kubeadm/app/phases/selfhosting:all-srcs",
         "//cmd/kubeadm/app/phases/token:all-srcs",
         "//cmd/kubeadm/app/preflight:all-srcs",
         "//cmd/kubeadm/app/util:all-srcs",

cmd/kubeadm/app/apis/kubeadm/BUILD

@@ -14,7 +14,6 @@ go_library(
         "env.go",
         "register.go",
         "types.go",
-        "well_known_labels.go",
     ],
     tags = ["automanaged"],
     deps = [

cmd/kubeadm/app/apis/kubeadm/env.go

@@ -25,15 +25,10 @@ import (
 
 var GlobalEnvParams = SetEnvParams()
 
-// TODO(phase1+) Move these paramaters to the API group
-// we need some params for testing etc, let's keep these hidden for now
 func SetEnvParams() *EnvParams {
 
 	envParams := map[string]string{
-		"kubernetes_dir":  "/etc/kubernetes",
-		"hyperkube_image": "",
-		"repo_prefix":     "gcr.io/google_containers",
-		"etcd_image":      "",
+		"kubernetes_dir": "/etc/kubernetes",
 	}
 
 	for k := range envParams {
@@ -43,9 +38,6 @@ func SetEnvParams() *EnvParams {
 	}
 
 	return &EnvParams{
-		KubernetesDir:    path.Clean(envParams["kubernetes_dir"]),
-		HyperkubeImage:   envParams["hyperkube_image"],
-		RepositoryPrefix: envParams["repo_prefix"],
-		EtcdImage:        envParams["etcd_image"],
+		KubernetesDir: path.Clean(envParams["kubernetes_dir"]),
 	}
 }

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -36,7 +36,10 @@ func KubeadmFuzzerFuncs(t apitesting.TestingCommon) []interface{} {
 			obj.CertificatesDir = "foo"
 			obj.APIServerCertSANs = []string{}
 			obj.Token = "foo"
+			obj.Etcd.Image = "foo"
 			obj.Etcd.DataDir = "foo"
+			obj.ImageRepository = "foo"
+			obj.UnifiedControlPlaneImage = "foo"
 		},
 		func(obj *kubeadm.NodeConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -23,10 +23,7 @@ import (
 )
 
 type EnvParams struct {
-	KubernetesDir    string
-	HyperkubeImage   string
-	RepositoryPrefix string
-	EtcdImage        string
+	KubernetesDir string
 }
 
 type MasterConfiguration struct {
@@ -37,6 +34,7 @@ type MasterConfiguration struct {
 	Networking         Networking
 	KubernetesVersion  string
 	CloudProvider      string
+	NodeName           string
 	AuthorizationModes []string
 
 	Token    string
@@ -55,6 +53,11 @@ type MasterConfiguration struct {
 	APIServerCertSANs []string
 	// CertificatesDir specifies where to store or look for all required certificates
 	CertificatesDir string
+
+	// ImageRepository what container registry to pull control plane images from
+	ImageRepository string
+	// UnifiedControlPlaneImage specifies if a specific container image should be used for all control plane components
+	UnifiedControlPlaneImage string
 }
 
 type API struct {
@@ -83,6 +86,8 @@ type Etcd struct {
 	KeyFile   string
 	DataDir   string
 	ExtraArgs map[string]string
+	// Image specifies which container image to use for running etcd. If empty, automatically populated by kubeadm using the image repository and default etcd version
+	Image string
 }
 
 type NodeConfiguration struct {
@@ -93,6 +98,7 @@ type NodeConfiguration struct {
 	DiscoveryToken string
 	// Currently we only pay attention to one api server but hope to support >1 in the future
 	DiscoveryTokenAPIServers []string
+	NodeName                 string
 	TLSBootstrapToken        string
 	Token                    string
 }

cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go

@@ -33,6 +33,7 @@ const (
 	DefaultCACertPath         = "/etc/kubernetes/pki/ca.crt"
 	DefaultCertificatesDir    = "/etc/kubernetes/pki"
 	DefaultEtcdDataDir        = "/var/lib/etcd"
+	DefaultImageRepository    = "gcr.io/google_containers"
 )
 
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
@@ -68,6 +69,10 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 		obj.TokenTTL = constants.DefaultTokenDuration
 	}
 
+	if obj.ImageRepository == "" {
+		obj.ImageRepository = DefaultImageRepository
+	}
+
 	if obj.Etcd.DataDir == "" {
 		obj.Etcd.DataDir = DefaultEtcdDataDir
 	}

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -30,6 +30,7 @@ type MasterConfiguration struct {
 	Networking         Networking `json:"networking"`
 	KubernetesVersion  string     `json:"kubernetesVersion"`
 	CloudProvider      string     `json:"cloudProvider"`
+	NodeName           string     `json:"nodeName"`
 	AuthorizationModes []string   `json:"authorizationModes"`
 
 	Token    string        `json:"token"`
@@ -48,6 +49,11 @@ type MasterConfiguration struct {
 	APIServerCertSANs []string `json:"apiServerCertSANs"`
 	// CertificatesDir specifies where to store or look for all required certificates
 	CertificatesDir string `json:"certificatesDir"`
+
+	// ImageRepository what container registry to pull control plane images from
+	ImageRepository string `json:"imageRepository"`
+	// UnifiedControlPlaneImage specifies if a specific container image should be used for all control plane components
+	UnifiedControlPlaneImage string `json:"unifiedControlPlaneImage"`
 }
 
 type API struct {
@@ -76,6 +82,8 @@ type Etcd struct {
 	KeyFile   string            `json:"keyFile"`
 	DataDir   string            `json:"dataDir"`
 	ExtraArgs map[string]string `json:"extraArgs"`
+	// Image specifies which container image to use for running etcd. If empty, automatically populated by kubeadm using the image repository and default etcd version
+	Image string `json:"image"`
 }
 
 type NodeConfiguration struct {
@@ -85,6 +93,7 @@ type NodeConfiguration struct {
 	DiscoveryFile            string   `json:"discoveryFile"`
 	DiscoveryToken           string   `json:"discoveryToken"`
 	DiscoveryTokenAPIServers []string `json:"discoveryTokenAPIServers"`
+	NodeName                 string   `json:"nodeName"`
 	TLSBootstrapToken        string   `json:"tlsBootstrapToken"`
 	Token                    string   `json:"token"`
 }

cmd/kubeadm/app/apis/kubeadm/validation/BUILD

@@ -15,6 +15,7 @@ go_test(
     tags = ["automanaged"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
     ],
 )
@@ -30,6 +31,8 @@ go_library(
         "//pkg/api/validation:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",
         "//pkg/registry/core/service/ipallocator:go_default_library",
+        "//pkg/util/node:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
     ],

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -24,6 +24,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/spf13/pflag"
+
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
@@ -32,6 +34,7 @@ import (
 	apivalidation "k8s.io/kubernetes/pkg/api/validation"
 	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
+	"k8s.io/kubernetes/pkg/util/node"
 )
 
 // TODO: Break out the cloudprovider functionality out of core and only support the new flow
@@ -41,7 +44,6 @@ var cloudproviders = []string{
 	"azure",
 	"cloudstack",
 	"gce",
-	"mesos",
 	"openstack",
 	"ovirt",
 	"photon",
@@ -62,6 +64,7 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList
 	allErrs = append(allErrs, ValidateNetworking(&c.Networking, field.NewPath("networking"))...)
 	allErrs = append(allErrs, ValidateAPIServerCertSANs(c.APIServerCertSANs, field.NewPath("cert-altnames"))...)
 	allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificates-dir"))...)
+	allErrs = append(allErrs, ValidateNodeName(c.NodeName, field.NewPath("node-name"))...)
 	allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...)
 	return allErrs
 }
@@ -236,6 +239,14 @@ func ValidateAbsolutePath(path string, fldPath *field.Path) field.ErrorList {
 	return allErrs
 }
 
+func ValidateNodeName(nodename string, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if node.GetHostname(nodename) != nodename {
+		allErrs = append(allErrs, field.Invalid(fldPath, nodename, "nodename is not valid, must be lower case"))
+	}
+	return allErrs
+}
+
 func ValidateCloudProvider(provider string, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 	if len(provider) == 0 {
@@ -249,3 +260,10 @@ func ValidateCloudProvider(provider string, fldPath *field.Path) field.ErrorList
 	allErrs = append(allErrs, field.Invalid(fldPath, provider, "cloudprovider not supported"))
 	return allErrs
 }
+
+func ValidateMixedArguments(flag *pflag.FlagSet) error {
+	if flag.Changed("config") && flag.NFlag() != 1 {
+		return fmt.Errorf("can not mix '--config' with other arguments")
+	}
+	return nil
+}

cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go

@@ -19,6 +19,8 @@ package validation
 import (
 	"testing"
 
+	"github.com/spf13/pflag"
+
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 )
@@ -76,6 +78,29 @@ func TestValidateAuthorizationModes(t *testing.T) {
 	}
 }
 
+func TestValidateNodeName(t *testing.T) {
+	var tests = []struct {
+		s        string
+		f        *field.Path
+		expected bool
+	}{
+		{"", nil, false},                 // ok if not provided
+		{"1234", nil, true},              // supported
+		{"valid-nodename", nil, true},    // supported
+		{"INVALID-NODENAME", nil, false}, // Upper cases is invalid
+	}
+	for _, rt := range tests {
+		actual := ValidateNodeName(rt.s, rt.f)
+		if (len(actual) == 0) != rt.expected {
+			t.Errorf(
+				"failed ValidateNodeName:\n\texpected: %t\n\t  actual: %t",
+				rt.expected,
+				(len(actual) == 0),
+			)
+		}
+	}
+}
+
 func TestValidateCloudProvider(t *testing.T) {
 	var tests = []struct {
 		s        string
@@ -175,6 +200,7 @@ func TestValidateIPNetFromString(t *testing.T) {
 }
 
 func TestValidateMasterConfiguration(t *testing.T) {
+	nodename := "valid-nodename"
 	var tests = []struct {
 		s        *kubeadm.MasterConfiguration
 		expected bool
@@ -187,6 +213,7 @@ func TestValidateMasterConfiguration(t *testing.T) {
 				DNSDomain:     "cluster.local",
 			},
 			CertificatesDir: "/some/cert/dir",
+			NodeName:        nodename,
 		}, false},
 		{&kubeadm.MasterConfiguration{
 			AuthorizationModes: []string{"Node", "RBAC"},
@@ -196,6 +223,7 @@ func TestValidateMasterConfiguration(t *testing.T) {
 			},
 			CertificatesDir: "/some/other/cert/dir",
 			Token:           "abcdef.0123456789abcdef",
+			NodeName:        nodename,
 		}, true},
 		{&kubeadm.MasterConfiguration{
 			AuthorizationModes: []string{"Node", "RBAC"},
@@ -204,6 +232,7 @@ func TestValidateMasterConfiguration(t *testing.T) {
 				DNSDomain:     "cluster.local",
 			},
 			CertificatesDir: "/some/cert/dir",
+			NodeName:        nodename,
 		}, false},
 		{&kubeadm.MasterConfiguration{
 			AuthorizationModes: []string{"Node", "RBAC"},
@@ -213,6 +242,7 @@ func TestValidateMasterConfiguration(t *testing.T) {
 			},
 			CertificatesDir: "/some/other/cert/dir",
 			Token:           "abcdef.0123456789abcdef",
+			NodeName:        nodename,
 		}, true},
 	}
 	for _, rt := range tests {
@@ -250,3 +280,42 @@ func TestValidateNodeConfiguration(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateMixedArguments(t *testing.T) {
+	var tests = []struct {
+		args     []string
+		expected bool
+	}{
+		{[]string{"--foo=bar"}, true},
+		{[]string{"--config=hello"}, true},
+		{[]string{"--foo=bar", "--config=hello"}, false},
+	}
+
+	var cfgPath string
+	var skipPreFlight bool
+
+	for _, rt := range tests {
+		f := pflag.NewFlagSet("test", pflag.ContinueOnError)
+		if f.Parsed() {
+			t.Error("f.Parse() = true before Parse")
+		}
+		f.String("foo", "", "string value")
+		f.StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file")
+		f.BoolVar(
+			&skipPreFlight, "skip-preflight-checks", skipPreFlight,
+			"Skip preflight checks normally run before modifying the system",
+		)
+		if err := f.Parse(rt.args); err != nil {
+			t.Fatal(err)
+		}
+
+		actual := ValidateMixedArguments(f)
+		if (actual == nil) != rt.expected {
+			t.Errorf(
+				"failed ValidateMixedArguments:\n\texpected: %t\n\t  actual: %t",
+				rt.expected,
+				(actual == nil),
+			)
+		}
+	}
+}

cmd/kubeadm/app/apis/kubeadm/well_known_labels.go

-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kubeadm
-
-// Role labels are applied to Nodes to mark their purpose.  In particular, we
-// usually want to distinguish the master, so that we can isolate privileged
-// pods and operations.
-//
-// Originally we relied on not registering the master, on the fact that the
-// master was Unschedulable, and on static manifests for master components.
-// But we now do register masters in many environments, are generally moving
-// away from static manifests (for better manageability), and working towards
-// deprecating the unschedulable field (replacing it with taints & tolerations
-// instead).
-//
-// Even with tainting, a label remains the easiest way of making a positive
-// selection, so that pods can schedule only to master nodes for example, and
-// thus installations will likely define a label for their master nodes.
-//
-// So that we can recognize master nodes in consequent places though (such as
-// kubectl get nodes), we encourage installations to use the well-known labels.
-// We define NodeLabelRole, which is the preferred form, but we will also recognize
-// other forms that are known to be in widespread use (NodeLabelKubeadmAlphaRole).
-
-const (
-	// NodeLabelKubeadmAlphaRole is a label that kubeadm applies to a Node as a hint that it has a particular purpose.
-	// Use of NodeLabelRole is preferred.
-	NodeLabelKubeadmAlphaRole = "kubeadm.alpha.kubernetes.io/role"
-)

cmd/kubeadm/app/cmd/BUILD

@@ -13,7 +13,6 @@ go_library(
     srcs = [
         "cmd.go",
         "completion.go",
-        "defaults.go",
         "init.go",
         "join.go",
         "reset.go",
@@ -28,15 +27,16 @@ go_library(
         "//cmd/kubeadm/app/cmd/phases:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/discovery:go_default_library",
-        "//cmd/kubeadm/app/master:go_default_library",
         "//cmd/kubeadm/app/node:go_default_library",
         "//cmd/kubeadm/app/phases/addons:go_default_library",
         "//cmd/kubeadm/app/phases/apiconfig:go_default_library",
-        "//cmd/kubeadm/app/phases/certs:go_default_library",
+        "//cmd/kubeadm/app/phases/controlplane:go_default_library",
         "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
+        "//cmd/kubeadm/app/phases/selfhosting:go_default_library",
         "//cmd/kubeadm/app/phases/token:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
+        "//cmd/kubeadm/app/util/config:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//cmd/kubeadm/app/util/token:go_default_library",
         "//pkg/api:go_default_library",
@@ -45,6 +45,7 @@ go_library(
         "//pkg/printers:go_default_library",
         "//pkg/util/i18n:go_default_library",
         "//pkg/util/initsystem:go_default_library",
+        "//pkg/util/node:go_default_library",
         "//pkg/util/version:go_default_library",
         "//pkg/version:go_default_library",
         "//vendor/github.com/ghodss/yaml:go_default_library",
@@ -54,7 +55,6 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
@@ -65,7 +65,6 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
-        "defaults_test.go",
         "reset_test.go",
         "token_test.go",
     ],

cmd/kubeadm/app/cmd/init.go

@@ -31,15 +31,17 @@ import (
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
+	cmdphases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	kubemaster "k8s.io/kubernetes/cmd/kubeadm/app/master"
 	addonsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons"
 	apiconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/apiconfig"
-	certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
+	controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
 	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
+	selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
 	tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/util/version"
 )
@@ -84,7 +86,13 @@ func NewCmdInit(out io.Writer) *cobra.Command {
 
 			i, err := NewInit(cfgPath, internalcfg, skipPreFlight, skipTokenPrint)
 			kubeadmutil.CheckErr(err)
-			kubeadmutil.CheckErr(i.Validate())
+			kubeadmutil.CheckErr(i.Validate(cmd))
+
+			// TODO: remove this warning in 1.9
+			if !cmd.Flags().Lookup("token-ttl").Changed {
+				fmt.Println("[kubeadm] WARNING: starting in 1.8, tokens expire after 24 hours by default (if you require a non-expiring token use --token-ttl 0)")
+			}
+
 			kubeadmutil.CheckErr(i.Run(out))
 		},
 	}
@@ -121,6 +129,10 @@ func NewCmdInit(out io.Writer) *cobra.Command {
 		&cfg.APIServerCertSANs, "apiserver-cert-extra-sans", cfg.APIServerCertSANs,
 		`Optional extra altnames to use for the API Server serving cert. Can be both IP addresses and dns names.`,
 	)
+	cmd.PersistentFlags().StringVar(
+		&cfg.NodeName, "node-name", cfg.NodeName,
+		`Specify the node name`,
+	)
 
 	cmd.PersistentFlags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
 
@@ -132,6 +144,10 @@ func NewCmdInit(out io.Writer) *cobra.Command {
 		&skipTokenPrint, "skip-token-print", skipTokenPrint,
 		"Skip printing of the default bootstrap token generated by 'kubeadm init'",
 	)
+	cmd.PersistentFlags().BoolVar(
+		&cfg.SelfHosted, "self-hosted", cfg.SelfHosted,
+		"[experimental] If kubeadm should make this control plane self-hosted",
+	)
 
 	cmd.PersistentFlags().StringVar(
 		&cfg.Token, "token", cfg.Token,
@@ -159,11 +175,20 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight,
 	}
 
 	// Set defaults dynamically that the API group defaulting can't (by fetching information from the internet, looking up network interfaces, etc.)
-	err := setInitDynamicDefaults(cfg)
+	err := configutil.SetInitDynamicDefaults(cfg)
 	if err != nil {
 		return nil, err
 	}
 
+	fmt.Printf("[init] Using Kubernetes version: %s\n", cfg.KubernetesVersion)
+	fmt.Printf("[init] Using Authorization mode: %v\n", cfg.AuthorizationModes)
+
+	// Warn about the limitations with the current cloudprovider solution.
+	if cfg.CloudProvider != "" {
+		fmt.Println("[init] WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.")
+		fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)")
+	}
+
 	if !skipPreFlight {
 		fmt.Println("[preflight] Running pre-flight checks")
 
@@ -186,7 +211,10 @@ type Init struct {
 }
 
 // Validate validates configuration passed to "kubeadm init"
-func (i *Init) Validate() error {
+func (i *Init) Validate(cmd *cobra.Command) error {
+	if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil {
+		return err
+	}
 	return validation.ValidateMasterConfiguration(i.cfg).ToAggregate()
 }
 
@@ -194,7 +222,7 @@ func (i *Init) Validate() error {
 func (i *Init) Run(out io.Writer) error {
 
 	// PHASE 1: Generate certificates
-	err := certphase.CreatePKIAssets(i.cfg)
+	err := cmdphases.CreatePKIAssets(i.cfg)
 	if err != nil {
 		return err
 	}
@@ -202,36 +230,26 @@ func (i *Init) Run(out io.Writer) error {
 	// PHASE 2: Generate kubeconfig files for the admin and the kubelet
 
 	masterEndpoint := fmt.Sprintf("https://%s:%d", i.cfg.API.AdvertiseAddress, i.cfg.API.BindPort)
-	err = kubeconfigphase.CreateInitKubeConfigFiles(masterEndpoint, i.cfg.CertificatesDir, kubeadmapi.GlobalEnvParams.KubernetesDir)
+	err = kubeconfigphase.CreateInitKubeConfigFiles(masterEndpoint, i.cfg.CertificatesDir, kubeadmapi.GlobalEnvParams.KubernetesDir, i.cfg.NodeName)
 	if err != nil {
 		return err
 	}
 
 	// PHASE 3: Bootstrap the control plane
-	if err := kubemaster.WriteStaticPodManifests(i.cfg); err != nil {
+	if err := controlplanephase.WriteStaticPodManifests(i.cfg); err != nil {
 		return err
 	}
 
 	adminKubeConfigPath := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName)
-	client, err := kubemaster.CreateClientAndWaitForAPI(adminKubeConfigPath)
+	client, err := kubeadmutil.CreateClientAndWaitForAPI(adminKubeConfigPath)
 	if err != nil {
 		return err
 	}
 
-	if err := apiconfigphase.UpdateMasterRoleLabelsAndTaints(client); err != nil {
+	if err := apiconfigphase.UpdateMasterRoleLabelsAndTaints(client, i.cfg.NodeName); err != nil {
 		return err
 	}
 
-	// Is deployment type self-hosted?
-	if i.cfg.SelfHosted {
-		// Temporary control plane is up, now we create our self hosted control
-		// plane components and remove the static manifests:
-		fmt.Println("[self-hosted] Creating self-hosted control plane...")
-		if err := kubemaster.CreateSelfHostedControlPlane(i.cfg, client); err != nil {
-			return err
-		}
-	}
-
 	// PHASE 4: Set up the bootstrap tokens
 	if !i.skipTokenPrint {
 		fmt.Printf("[token] Using token: %s\n", i.cfg.Token)
@@ -268,6 +286,16 @@ func (i *Init) Run(out io.Writer) error {
 		return err
 	}
 
+	// Is deployment type self-hosted?
+	if i.cfg.SelfHosted {
+		// Temporary control plane is up, now we create our self hosted control
+		// plane components and remove the static manifests:
+		fmt.Println("[self-hosted] Creating self-hosted control plane...")
+		if err := selfhostingphase.CreateSelfHostedControlPlane(i.cfg, client); err != nil {
+			return err
+		}
+	}
+
 	ctx := map[string]string{
 		"KubeConfigPath": filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName),
 		"KubeConfigName": kubeadmconstants.AdminKubeConfigFileName,

cmd/kubeadm/app/cmd/join.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"os"
 	"path/filepath"
 
 	"github.com/renstrom/dedent"
@@ -33,11 +32,12 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
-	kubenode "k8s.io/kubernetes/cmd/kubeadm/app/node"
+	kubeadmnode "k8s.io/kubernetes/cmd/kubeadm/app/node"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	"k8s.io/kubernetes/pkg/api"
+	nodeutil "k8s.io/kubernetes/pkg/util/node"
 )
 
 var (
@@ -63,26 +63,26 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
 		Use:   "join <flags> [DiscoveryTokenAPIServers]",
 		Short: "Run this on any machine you wish to join an existing cluster",
 		Long: dedent.Dedent(`
-		When joining a kubeadm initialized cluster, we need to establish 
-		bidirectional trust. This is split into discovery (having the Node 
-		trust the Kubernetes Master) and TLS bootstrap (having the Kubernetes 
+		When joining a kubeadm initialized cluster, we need to establish
+		bidirectional trust. This is split into discovery (having the Node
+		trust the Kubernetes Master) and TLS bootstrap (having the Kubernetes
 		Master trust the Node).
 
-		There are 2 main schemes for discovery. The first is to use a shared 
-		token along with the IP address of the API server. The second is to 
-		provide a file (a subset of the standard kubeconfig file). This file 
-		can be a local file or downloaded via an HTTPS URL. The forms are 
-		kubeadm join --discovery-token abcdef.1234567890abcdef 1.2.3.4:6443, 
+		There are 2 main schemes for discovery. The first is to use a shared
+		token along with the IP address of the API server. The second is to
+		provide a file (a subset of the standard kubeconfig file). This file
+		can be a local file or downloaded via an HTTPS URL. The forms are
+		kubeadm join --discovery-token abcdef.1234567890abcdef 1.2.3.4:6443,
 		kubeadm join --discovery-file path/to/file.conf, or kubeadm join
-		--discovery-file https://url/file.conf. Only one form can be used. If 
-		the discovery information is loaded from a URL, HTTPS must be used and 
+		--discovery-file https://url/file.conf. Only one form can be used. If
+		the discovery information is loaded from a URL, HTTPS must be used and
 		the host installed CA bundle is used to verify the connection.
 
-		The TLS bootstrap mechanism is also driven via a shared token. This is 
+		The TLS bootstrap mechanism is also driven via a shared token. This is
 		used to temporarily authenticate with the Kubernetes Master to submit a
-		certificate signing request (CSR) for a locally created key pair. By 
-		default kubeadm will set up the Kubernetes Master to automatically 
-		approve these signing requests. This token is passed in with the 
+		certificate signing request (CSR) for a locally created key pair. By
+		default kubeadm will set up the Kubernetes Master to automatically
+		approve these signing requests. This token is passed in with the
 		--tls-bootstrap-token abcdef.1234567890abcdef flag.
 
 		Often times the same token is used for both parts. In this case, the
@@ -97,7 +97,7 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
 
 			j, err := NewJoin(cfgPath, args, internalcfg, skipPreFlight)
 			kubeadmutil.CheckErr(err)
-			kubeadmutil.CheckErr(j.Validate())
+			kubeadmutil.CheckErr(j.Validate(cmd))
 			kubeadmutil.CheckErr(j.Run(out))
 		},
 	}
@@ -113,6 +113,9 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
 		&cfg.DiscoveryToken, "discovery-token", "",
 		"A token used to validate cluster information fetched from the master")
 	cmd.PersistentFlags().StringVar(
+		&cfg.NodeName, "node-name", "",
+		"Specify the node name")
+	cmd.PersistentFlags().StringVar(
 		&cfg.TLSBootstrapToken, "tls-bootstrap-token", "",
 		"A token used for TLS bootstrapping")
 	cmd.PersistentFlags().StringVar(
@@ -161,7 +164,10 @@ func NewJoin(cfgPath string, args []string, cfg *kubeadmapi.NodeConfiguration, s
 	return &Join{cfg: cfg}, nil
 }
 
-func (j *Join) Validate() error {
+func (j *Join) Validate(cmd *cobra.Command) error {
+	if err := validation.ValidateMixedArguments(cmd.PersistentFlags()); err != nil {
+		return err
+	}
 	return validation.ValidateNodeConfiguration(j.cfg).ToAggregate()
 }
 
@@ -172,18 +178,16 @@ func (j *Join) Run(out io.Writer) error {
 		return err
 	}
 
-	hostname, err := os.Hostname()
-	if err != nil {
-		return err
-	}
+	hostname := nodeutil.GetHostname(j.cfg.NodeName)
+
 	client, err := kubeconfigutil.KubeConfigToClientSet(cfg)
 	if err != nil {
 		return err
 	}
-	if err := kubenode.ValidateAPIServer(client); err != nil {
+	if err := kubeadmnode.ValidateAPIServer(client); err != nil {
 		return err
 	}
-	if err := kubenode.PerformTLSBootstrap(cfg, hostname); err != nil {
+	if err := kubeadmnode.PerformTLSBootstrap(cfg, hostname); err != nil {
 		return err
 	}
 

cmd/kubeadm/app/cmd/phases/BUILD

@@ -5,6 +5,7 @@ licenses(["notice"])
 load(
     "@io_bazel_rules_go//go:def.bzl",
     "go_library",
+    "go_test",
 )
 
 go_library(
@@ -14,20 +15,38 @@ go_library(
         "kubeconfig.go",
         "phase.go",
         "preflight.go",
+        "selfhosting.go",
     ],
     tags = ["automanaged"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/phases/certs:go_default_library",
+        "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
         "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
+        "//cmd/kubeadm/app/phases/selfhosting:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
+        "//cmd/kubeadm/app/util/config:go_default_library",
+        "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//pkg/api:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["certs_test.go"],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm/install:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
+        "//vendor/github.com/renstrom/dedent:go_default_library",
+        "//vendor/github.com/spf13/cobra:go_default_library",
     ],
 )