Create, distribute, and use certificates for TLS and identity

contrib/ansible/roles/kubernetes/defaults/main.yml

+# This directory is where all the additional scripts go
+# that Kubernetes normally puts in /srv/kubernetes.
+# This puts them in a sane location
+kube_script_dir: /usr/libexec/kubernetes
+
+# This directory is where all the additional config stuff goes
+# the kubernetes normally puts in /srv/kubernets.
+# This puts them in a sane location.
+# Editting this value will almost surely break something. Don't
+# change it. Things like the systemd scripts are hard coded to
+# look in here. Don't do it.
+kube_config_dir: /etc/kubernetes
+
+# This is where all the cert scripts and certs will be located
+kube_cert_dir: "{{ kube_config_dir }}/certs"
+
+
+# This is the group that the cert creation scripts chgrp the
+# cert files to. Not really changable...
+kube_cert_group: kube-cert

contrib/ansible/roles/kubernetes/files/make-ca-cert.sh

+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+cert_ip=$1
+cert_dir=${CERT_DIR:-/srv/kubernetes}
+cert_group=${CERT_GROUP:-kube-cert}
+
+mkdir -p "$cert_dir"
+
+use_cn=false
+
+# TODO: Add support for discovery on other providers?
+if [ "$cert_ip" == "_use_gce_external_ip_" ]; then
+  cert_ip=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
+fi
+
+if [ "$cert_ip" == "_use_aws_external_ip_" ]; then
+  cert_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
+fi
+
+if [ "$cert_ip" == "_use_azure_dns_name_" ]; then
+  cert_ip=$(uname -n | awk -F. '{ print $2 }').cloudapp.net
+  use_cn=true
+fi
+
+tmpdir=$(mktemp -d -t kubernetes_cacert.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+# TODO: For now, this is a patched tool that makes subject-alt-name work, when
+# the fix is upstream  move back to the upstream easyrsa.  This is cached in GCS
+# but is originally taken from:
+#   https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
+#
+# To update, do the following:
+# curl -o easy-rsa.tar.gz https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
+# gsutil cp easy-rsa.tar.gz gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
+# gsutil acl ch -R -g all:R gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
+#
+# Due to GCS caching of public objects, it may take time for this to be widely
+# distributed.
+curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1
+tar xzf easy-rsa.tar.gz > /dev/null 2>&1
+
+cd easy-rsa-master/easyrsa3
+./easyrsa init-pki > /dev/null 2>&1
+./easyrsa --batch "--req-cn=$cert_ip@`date +%s`" build-ca nopass > /dev/null 2>&1
+if [ $use_cn = "true" ]; then
+    ./easyrsa build-server-full $cert_ip nopass > /dev/null 2>&1
+    cp -p pki/issued/$cert_ip.crt "${cert_dir}/server.cert" > /dev/null 2>&1
+    cp -p pki/private/$cert_ip.key "${cert_dir}/server.key" > /dev/null 2>&1
+else
+    ./easyrsa --subject-alt-name=IP:$cert_ip build-server-full kubernetes-master nopass > /dev/null 2>&1
+    cp -p pki/issued/kubernetes-master.crt "${cert_dir}/server.cert" > /dev/null 2>&1
+    cp -p pki/private/kubernetes-master.key "${cert_dir}/server.key" > /dev/null 2>&1
+fi
+./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1
+cp -p pki/ca.crt "${cert_dir}/ca.crt"
+cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt"
+cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key"
+# Make server certs accessible to apiserver.
+chgrp $cert_group "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt"
+chmod 660 "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt"

contrib/ansible/roles/kubernetes/files/make-cert.sh

+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cert_dir=${CERT_DIR:-/srv/kubernetes}
+cert_group=${CERT_GROUP:-kube-cert}
+
+mkdir -p "$cert_dir"
+
+openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
+  -subj "/CN=kubernetes.invalid/O=Kubernetes" \
+  -keyout "${cert_dir}/server.key" -out "${cert_dir}/server.cert"
+chgrp $cert_group "${cert_dir}/server.key" "${cert_dir}/server.cert"
+chmod 660 "${cert_dir}/server.key" "${cert_dir}/server.cert"

contrib/ansible/roles/kubernetes/tasks/certs.yml

+---
+- name: Create system kube-cert groups
+  group: name={{ kube_cert_group }} state=present system=yes
+
+- name: Create system kube user
+  user:
+    name=kube
+    comment="Kubernetes user"
+    shell=/sbin/nologin
+    state=present
+    system=yes
+    groups={{ kube_cert_group }}
+
+- name: make sure the certificate directory exits
+  file:
+    path={{ kube_cert_dir }}
+    state=directory
+    mode=o-rwx
+    group={{ kube_cert_group }}
+
+- name: Install rsync to push certs around
+  action: "{{ ansible_pkg_mgr }}"
+  args:
+    name: rsync
+    state: latest
+  when: not is_atomic
+
+- name: Generating RSA key for cert node to push to others
+  user: name=root generate_ssh_key=yes
+  run_once: true
+  delegate_to: "{{ groups['masters'][0] }}"
+
+- name: Downloading pub key
+  fetch:
+    src=/root/.ssh/id_rsa.pub
+    dest=/tmp/id_rsa.pub
+    flat=yes
+    fail_on_missing=true
+  run_once: true
+  delegate_to: "{{ groups['masters'][0] }}"
+  changed_when: false
+
+- include: gen_certs.yml
+  when: inventory_hostname == groups['masters'][0]
+
+- include: place_certs.yml
+
+- name: Delete the downloaded pub key
+  local_action: file path=/tmp/id_rsa.pub state=absent
+  sudo: false
+  run_once: true
+  changed_when: false

contrib/ansible/roles/kubernetes/tasks/gen_certs.yml

+---
+#- name: Get create cert script from Kubernetes
+#  get_url:
+#    url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/saltbase/salt/generate-cert/make-cert.sh
+#    dest={{ kube_script_dir }}/make-cert.sh mode=0500
+#    force=yes
+
+#- name: Get create ca cert script from Kubernetes
+#  get_url:
+#    url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/saltbase/salt/generate-cert/make-ca-cert.sh
+#    dest={{ kube_script_dir }}/make-ca-cert.sh mode=0500
+#    force=yes
+
+- name: HACK | overwrite make-cert.sh from local copy
+  copy:
+    src=make-cert.sh
+    dest={{ kube_script_dir }}
+    mode=0500
+  changed_when: false
+
+- name: HACK | overwrite make-ca-cert.sh from local copy
+  copy:
+    src=make-ca-cert.sh
+    dest={{ kube_script_dir }}
+    mode=0500
+  changed_when: false
+
+# FIXME This only generates a cert for one master...
+- name: Run create cert script on master
+  command:
+    "{{ kube_script_dir }}/make-ca-cert.sh {{ inventory_hostname }}"
+  args:
+    creates: "{{ kube_cert_dir }}/server.cert"
+  environment:
+    CERT_DIR: "{{ kube_cert_dir }}"
+    CERT_GROUP: "{{ kube_cert_group }}"
+
+- name: Verify certificate permissions
+  file:
+    path={{ item }}
+    group={{ kube_cert_group }}
+    owner=kube
+    mode=0440
+  with_items:
+    - "{{ kube_cert_dir }}/ca.crt"
+    - "{{ kube_cert_dir }}/server.cert"
+    - "{{ kube_cert_dir }}/server.key"
+    - "{{ kube_cert_dir }}/kubecfg.crt"
+    - "{{ kube_cert_dir }}/kubecfg.key"

contrib/ansible/roles/kubernetes/tasks/main.yml

@@ -2,7 +2,22 @@
 - include: fedora.yml
   when: ansible_distribution == "Fedora"
 
+- name: Update {{ kube_script_dir }} if this is atomic
+  set_fact:
+    kube_script_dir: "/usr/local/libexec/kubernretes"
+  when: is_atomic and kube_script_dir == "/usr/libexec/kubernetes"
+
+- name: Create kubernetes config directory
+  file: path={{ kube_config_dir }} state=directory
+
+- name: Create kubernetes script directory
+  file: path={{ kube_script_dir }} state=directory
+
 - name: write the global config file
-  template: src=config.j2 dest=/etc/kubernetes/config
+  template: src=config.j2 dest={{ kube_config_dir }}/config
   notify:
     - restart daemons
+
+- include: certs.yml
+  tags:
+    certs

contrib/ansible/roles/kubernetes/tasks/place_certs.yml

+---
+- name: place ssh public key so apiserver can push certs
+  authorized_key: user=root key="{{ item }}" state=present
+  with_file:
+    - '/tmp/id_rsa.pub'
+  changed_when: false
+
+- name: Copy certificates directly from the apiserver to nodes
+  synchronize: src={{ kube_cert_dir }}/{{ item }} dest={{ kube_cert_dir }}/{{ item }}
+  delegate_to: "{{ groups['masters'][0] }}"
+  with_items:
+    - "ca.crt"
+    - "kubecfg.crt"
+    - "kubecfg.key"
+  notify:
+    - restart daemons
+
+- name: remove ssh public key so apiserver can not push stuff
+  authorized_key: user=root key="{{ item }}" state=absent
+  with_file:
+    - '/tmp/id_rsa.pub'
+  changed_when: false

contrib/ansible/roles/kubernetes/templates/config.j2

@@ -20,4 +20,4 @@ KUBE_LOG_LEVEL="--v=0"
 KUBE_ALLOW_PRIV="--allow_privileged=true"
 
 # How the replication controller, scheduler, and proxy
-KUBE_MASTER="--master=http://{{ groups['masters'][0] }}:8080"
+KUBE_MASTER="--master=https://{{ groups['masters'][0] }}:443"

contrib/ansible/roles/master/tasks/firewalld.yml

 ---
 - name: Open firewalld port for apiserver
-  firewalld: port=8080/tcp permanent=false state=enabled
+  firewalld: port=443/tcp permanent=false state=enabled
   # in case this is also a node with firewalld turned off
   ignore_errors: yes
 
 - name: Save firewalld port for apiserver
-  firewalld: port=8080/tcp permanent=true state=enabled
+  firewalld: port=443/tcp permanent=true state=enabled
   # in case this is also a node with firewalld turned off
   ignore_errors: yes

contrib/ansible/roles/master/tasks/iptables.yml

@@ -5,7 +5,7 @@
   always_run: yes
 
 - name: Open apiserver port with iptables
-  command: /sbin/iptables -I INPUT 1 -p tcp --dport 8080 -j ACCEPT -m comment --comment "kube-apiserver"
+  command: /sbin/iptables -I INPUT 1 -p tcp --dport 443 -j ACCEPT -m comment --comment "kube-apiserver"
   when: kube-apiserver not in iptablesrules.stdout
   notify:
     - restart iptables

contrib/ansible/roles/master/tasks/main.yml

@@ -6,17 +6,17 @@
   when: not is_atomic and ansible_distribution == "CentOS"
 
 - name: write the config file for the api server
-  template: src=apiserver.j2 dest=/etc/kubernetes/apiserver
+  template: src=apiserver.j2 dest={{ kube_config_dir }}/apiserver
   notify:
     - restart apiserver
 
 - name: write the config file for the controller-manager
-  copy: src=controller-manager dest=/etc/kubernetes/controller-manager
+  template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager
   notify:
     - restart controller-manager
 
 - name: write the config file for the scheduler
-  copy: src=scheduler dest=/etc/kubernetes/scheduler
+  template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler
   notify:
     - restart scheduler
 
@@ -24,6 +24,16 @@
   capabilities: path=/usr/bin/kube-apiserver capability=cap_net_bind_service=ep state=present
   when: not is_atomic
 
+- name: write the kubecfg (auth) file for controller-manager
+  template: src=controller-manager.kubeconfig.j2 dest={{ kube_config_dir }}/controller-manager.kubeconfig
+  notify:
+    - restart controller-manager
+
+- name: write the kubecfg (auth) file for scheduler
+  template: src=scheduler.kubeconfig.j2 dest={{ kube_config_dir }}/scheduler.kubeconfig
+  notify:
+    - restart scheduler
+
 - name: Enable apiserver
   service: name=kube-apiserver enabled=yes state=started
 

contrib/ansible/roles/master/templates/apiserver.j2

@@ -5,10 +5,10 @@
 #
 
 # The address on the local server to listen to.
-KUBE_API_ADDRESS="--address=0.0.0.0"
+KUBE_API_ADDRESS="--insecure-bind-address=127.0.0.1"
 
 # The port on the local server to listen on.
-# KUBE_API_PORT="--port=8080"
+KUBE_API_PORT="--secure-port=443"
 
 # Port nodes listen on
 # KUBELET_PORT="--kubelet_port=10250"
@@ -23,4 +23,4 @@ KUBE_ETCD_SERVERS="--etcd_servers={% for node in groups['etcd'] %}http://{{ node
 KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
 
 # Add your own!
-KUBE_API_ARGS=""
+KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.cert --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt "

contrib/ansible/roles/master/files/controller-manager → contrib/ansible/roles/master/templates/controller-manager.j2

@@ -4,4 +4,4 @@
 # defaults from config and apiserver should be adequate
 
 # Add your own!
-KUBE_CONTROLLER_MANAGER_ARGS=""
+KUBE_CONTROLLER_MANAGER_ARGS="--kubeconfig={{ kube_config_dir }}/controller-manager.kubeconfig"

contrib/ansible/roles/master/templates/controller-manager.kubeconfig.j2

+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.crt
+    server: http://{{ groups['masters'][0] }}:443
+  name: {{ cluster_name }}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: kubelet
+  name: kubelet-to-{{ cluster_name }}
+current-context: kubelet-to-{{ cluster_name }}
+kind: Config
+preferences: {}
+users:
+- name: kubelet
+  user:
+    client-certificate: {{ kube_cert_dir }}/kubecfg.crt
+    client-key: {{ kube_cert_dir }}/kubecfg.key

contrib/ansible/roles/master/files/scheduler → contrib/ansible/roles/master/templates/scheduler.j2

@@ -4,4 +4,4 @@
 # default config should be adequate
 
 # Add your own!
-KUBE_SCHEDULER_ARGS=""
+KUBE_SCHEDULER_ARGS="--kubeconfig={{ kube_config_dir }}/scheduler.kubeconfig"

contrib/ansible/roles/master/templates/scheduler.kubeconfig.j2

+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.crt
+    server: http://{{ groups['masters'][0] }}:443
+  name: {{ cluster_name }}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: kubelet
+  name: kubelet-to-{{ cluster_name }}
+current-context: kubelet-to-{{ cluster_name }}
+kind: Config
+preferences: {}
+users:
+- name: kubelet
+  user:
+    client-certificate: {{ kube_cert_dir }}/kubecfg.crt
+    client-key: {{ kube_cert_dir }}/kubecfg.key

contrib/ansible/roles/node/tasks/main.yml

@@ -6,12 +6,22 @@
   when: not is_atomic and ansible_distribution == "CentOS"
 
 - name: write the config files for kubelet
-  template: src=kubelet.j2 dest=/etc/kubernetes/kubelet
+  template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet
   notify:
     - restart kubelet
 
 - name: write the config files for proxy
-  copy: src=proxy dest=/etc/kubernetes/proxy
+  template: src=proxy.j2 dest={{ kube_config_dir }}/proxy
+  notify:
+    - restart proxy
+
+- name: write the kubecfg (auth) file for kubelet
+  template: src=kubelet.kubeconfig.j2 dest={{ kube_config_dir }}/kubelet.kubeconfig
+  notify:
+    - restart kubelet
+
+- name: write the kubecfg (auth) file for kube-proxy
+  template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig
   notify:
     - restart proxy
 

contrib/ansible/roles/node/templates/kubelet.j2

@@ -11,7 +11,7 @@ KUBELET_ADDRESS="--address=0.0.0.0"
 KUBELET_HOSTNAME="--hostname_override={{ inventory_hostname }}"
 
 # location of the api-server
-KUBELET_API_SERVER="--api_servers=http://{{ groups['masters'][0]}}:8080"
+KUBELET_API_SERVER="--api_servers=https://{{ groups['masters'][0]}}:443"
 
 # Add your own!
-KUBELET_ARGS=""
+KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig"

contrib/ansible/roles/node/templates/kubelet.kubeconfig.j2

+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.crt
+    server: http://{{ groups['masters'][0] }}:443
+  name: {{ cluster_name }}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: kubelet
+  name: kubelet-to-{{ cluster_name }}
+current-context: kubelet-to-{{ cluster_name }}
+kind: Config
+preferences: {}
+users:
+- name: kubelet
+  user:
+    client-certificate: {{ kube_cert_dir }}/kubecfg.crt
+    client-key: {{ kube_cert_dir }}/kubecfg.key

contrib/ansible/roles/node/files/proxy → contrib/ansible/roles/node/templates/proxy.j2

@@ -4,4 +4,4 @@
 # default config should be adequate
 
 # Add your own!
-KUBE_PROXY_ARGS=""
+KUBE_PROXY_ARGS="--kubeconfig={{ kube_config_dir }}/proxy.kubeconfig"

contrib/ansible/roles/node/templates/proxy.kubeconfig.j2

+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.crt
+    server: http://{{ groups['masters'][0] }}:443
+  name: {{ cluster_name }}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: kubelet
+  name: kubelet-to-{{ cluster_name }}
+current-context: kubelet-to-{{ cluster_name }}
+kind: Config
+preferences: {}
+users:
+- name: kubelet
+  user:
+    client-certificate: {{ kube_cert_dir }}/kubecfg.crt
+    client-key: {{ kube_cert_dir }}/kubecfg.key

contrib/ansible/setup.sh

@@ -15,4 +15,5 @@
 # limitations under the License.
 
 inventory=${INVENTORY:-inventory}
+
 ansible-playbook -i ${inventory} cluster.yml $@