Merge pull request #17121 from mikedanese/kube-proxy-static

build/common.sh

@@ -99,6 +99,7 @@ readonly KUBE_DOCKER_WRAPPED_BINARIES=(
   kube-apiserver,busybox
   kube-controller-manager,busybox
   kube-scheduler,busybox
+  kube-proxy,gcr.io/google_containers/debian-iptables:v1
 )
 
 # The set of addons images that should be prepopulated

cluster/saltbase/install.sh

@@ -25,9 +25,10 @@ SALT_ROOT=$(dirname "${BASH_SOURCE}")
 readonly SALT_ROOT
 
 readonly KUBE_DOCKER_WRAPPED_BINARIES=(
-      kube-apiserver
-      kube-controller-manager
-      kube-scheduler
+  kube-apiserver
+  kube-controller-manager
+  kube-scheduler
+  kube-proxy
 )
     
 readonly SERVER_BIN_TAR=${1-}

cluster/saltbase/pillar/docker-images.sls

@@ -2,3 +2,4 @@
 kube-apiserver_docker_tag: #kube-apiserver_docker_tag_value#
 kube-controller-manager_docker_tag: #kube-controller-manager_docker_tag_value#
 kube-scheduler_docker_tag: #kube-scheduler_docker_tag_value#
+kube-proxy_docker_tag: #kube-proxy_docker_tag_value#

cluster/saltbase/salt/kube-node-unpacker/init.sls

+/etc/kubernetes/kube-node-unpacker.sh:
+  file.managed:
+    - source: salt://kube-node-unpacker/kube-node-unpacker.sh
+    - user: root
+    - group: root
+    - mode: 755
+
+node-docker-image-tags:
+  file.touch:
+    - name: /srv/pillar/docker-images.sls
+
+{% if pillar.get('is_systemd') %}
+
+{{ pillar.get('systemd_system_path') }}/kube-node-unpacker.service:
+  file.managed:
+    - source: salt://kube-node-unpacker/kube-node-unpacker.service
+    - user: root
+    - group: root
+  cmd.wait:
+    - name: /opt/kubernetes/helpers/services bounce kube-node-unpacker
+    - watch:
+      - file: node-docker-image-tags
+      - file: /etc/kubernetes/kube-node-unpacker.sh
+      - file: {{ pillar.get('systemd_system_path') }}/kube-node-unpacker.service
+
+{% else %}
+
+/etc/init.d/kube-node-unpacker:
+  file.managed:
+    - source: salt://kube-node-unpacker/initd
+    - user: root
+    - group: root
+    - mode: 755
+
+kube-node-unpacker:
+  service.running:
+    - enable: True
+    - restart: True
+    - watch:
+      - file: node-docker-image-tags
+      - file: /etc/kubernetes/kube-node-unpacker.sh
+
+{% endif %}

cluster/saltbase/salt/kube-proxy/initd → cluster/saltbase/salt/kube-node-unpacker/initd

 #!/bin/bash
 #
 ### BEGIN INIT INFO
-# Provides:    kube-proxy
-# Required-Start:    $local_fs $network $syslog
+# Provides:   kube-node-unpacker
+# Required-Start:    $local_fs $network $syslog docker
 # Required-Stop:
 # Default-Start:     2 3 4 5
 # Default-Stop:      0 1 6
-# Short-Description: The Kubernetes network proxy
+# Short-Description: Kubernetes Node Unpacker
 # Description:
-#   The Kubernetes network proxy enables network redirection and
-#   loadbalancing for dynamically placed containers.
+#   Unpacks docker images on Kubernetes nodes
 ### END INIT INFO
 
 
 # PATH should only include /usr/* if it runs after the mountnfs.sh script
 PATH=/sbin:/usr/sbin:/bin:/usr/bin
-DESC="The Kubernetes network proxy"
-NAME=kube-proxy
-DAEMON=/usr/local/bin/kube-proxy
-DAEMON_ARGS=""
+DESC="Kubernetes Node Unpacker"
+NAME=kube-node-unpacker
 DAEMON_LOG_FILE=/var/log/$NAME.log
 PIDFILE=/var/run/$NAME.pid
 SCRIPTNAME=/etc/init.d/$NAME
-DAEMON_USER=root
-
-# Exit if the package is not installed
-[ -x "$DAEMON" ] || exit 0
-
-# Read configuration variable file if it is present
-[ -r /etc/default/$NAME ] && . /etc/default/$NAME
+KUBE_MASTER_ADDONS_SH=/etc/kubernetes/kube-node-unpacker.sh
 
 # Define LSB log_* functions.
 # Depend on lsb-base (>= 3.2-14) to ensure that this file is present
 # and status_of_proc is working.
 . /lib/lsb/init-functions
 
+
+
+
 #
 # Function that starts the daemon/service
 #
 do_start()
 {
-        # Avoid a potential race at boot time when both monit and init.d start
-        # the same service
-        PIDS=$(pidof $DAEMON)
-        for PID in ${PIDS}; do
-            kill -9 $PID
-	done
-
-        # Raise the file descriptor limit - we expect to open a lot of sockets!
-        ulimit -n 65536
-
-        # Return
-        #   0 if daemon has been started
-        #   1 if daemon was already running
-        #   2 if daemon could not be started
-        start-stop-daemon --start --quiet --background --no-close \
-                --make-pidfile --pidfile $PIDFILE \
-                --exec $DAEMON -c $DAEMON_USER --test > /dev/null \
-                || return 1
-        start-stop-daemon --start --quiet --background --no-close \
-                --make-pidfile --pidfile $PIDFILE \
-                --exec $DAEMON -c $DAEMON_USER -- \
-                $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \
-                || return 2
+    ${KUBE_MASTER_ADDONS_SH} </dev/null >>${DAEMON_LOG_FILE} 2>&1 &
+    echo $! > ${PIDFILE}
+    disown
 }
 
 #
@@ -70,20 +44,11 @@ do_start()
 #
 do_stop()
 {
-        # Return
-        #   0 if daemon has been stopped
-        #   1 if daemon was already stopped
-        #   2 if daemon could not be stopped
-        #   other if a failure occurred
-        start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME
-        RETVAL="$?"
-        [ "$RETVAL" = 2 ] && return 2
-        # Many daemons don't delete their pidfiles when they exit.
-        rm -f $PIDFILE
-        return "$RETVAL"
+    kill $(cat ${PIDFILE})
+    rm ${PIDFILE}
+    return
 }
 
-
 case "$1" in
   start)
         log_daemon_msg "Starting $DESC" "$NAME"
@@ -102,7 +67,7 @@ case "$1" in
         esac
         ;;
   status)
-        status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $?
+        status_of_proc -p $PIDFILE $KUBE_MASTER_ADDONS_SH $NAME
         ;;
 
   restart|force-reload)

cluster/saltbase/salt/kube-proxy/kube-proxy.service → cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service

 [Unit]
-Description=Kubernetes Kube-Proxy Server
+Description=Kubernetes Node Unpacker
 Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 
 [Service]
-EnvironmentFile=/etc/sysconfig/kube-proxy
-ExecStart=/usr/local/bin/kube-proxy "$DAEMON_ARGS"
-Restart=on-failure
-LimitNOFILE=65536
+ExecStart=/etc/kubernetes/kube-node-unpacker.sh
 
 [Install]
 WantedBy=multi-user.target

cluster/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh

+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# loadedImageFlags is a bit-flag to track which docker images loaded successfully.
+let loadedImageFlags=0
+
+while true; do
+  restart_docker=false
+
+  if which docker 1>/dev/null 2>&1; then
+
+    timeout 30 docker load -i /srv/salt/kube-bins/kube-proxy.tar 1>/dev/null 2>&1
+    rc=$?
+    if [[ "${rc}" == 0 ]]; then
+      let loadedImageFlags="${loadedImageFlags}|1"
+    elif [[ "${rc}" == 124 ]]; then
+      restart_docker=true
+    fi
+  fi
+
+  # required docker images got installed. exit while loop.
+  if [[ "${loadedImageFlags}" == 1 ]]; then break; fi
+
+  # Sometimes docker load hang, restart docker daemon resolve the issue
+  if [[ "${restart_docker}" ]]; then service docker restart; fi
+
+  # sleep for 15 seconds before attempting to load docker images again
+  sleep 15
+
+done
+
+# Now exit. After kube-push, salt will notice that the service is down and it
+# will start it and new docker images will be loaded.

cluster/saltbase/salt/kube-proxy/default

-{% set daemon_args = "$DAEMON_ARGS" -%}
-{% if grains['os_family'] == 'RedHat' -%}
-	{% set daemon_args = "" -%}
-{% endif -%}
-
-{% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%}
-{% if grains.api_servers is defined -%}
-  {% set api_servers = "--master=https://" + grains.api_servers -%}
-{% else -%}
-  {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
-  {% set api_servers = "--master=https://" + ips[0][0] -%}
-{% endif -%}
-
-# TODO: remove nginx for other cloud providers.
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ]  %}
-   {% set api_servers_with_port = api_servers -%}
-{% else -%}
-  {% set api_servers_with_port = api_servers + ":6443" -%}
-{% endif -%}
-
-{% set test_args = "" -%}
-{% if pillar['kubeproxy_test_args'] is defined -%}
-  {% set test_args=pillar['kubeproxy_test_args'] %}
-{% endif -%}
-
-# test_args has to be kept at the end, so they'll overwrite any prior configuration
-DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{kubeconfig}} {{pillar['log_level']}} {{test_args}}"

cluster/saltbase/salt/kube-proxy/init.sls

-{% if pillar.get('is_systemd') %}
-{% set environment_file = '/etc/sysconfig/kube-proxy' %}
-{% else %}
-{% set environment_file = '/etc/default/kube-proxy' %}
-{% endif %}
-
-/usr/local/bin/kube-proxy:
+/var/lib/kube-proxy/kubeconfig:
   file.managed:
-    - source: salt://kube-bins/kube-proxy
+    - source: salt://kube-proxy/kubeconfig
     - user: root
     - group: root
-    - mode: 755
+    - mode: 400
+    - makedirs: true
 
-{{ environment_file }}:
+# kube-proxy in a static pod
+/etc/kubernetes/manifests/kube-proxy.manifest:
   file.managed:
-    - source: salt://kube-proxy/default
+    - source: salt://kube-proxy/kube-proxy.manifest
     - template: jinja
     - user: root
     - group: root
     - mode: 644
-
-kube-proxy:
-  group.present:
-    - system: True
-  user.present:
-    - system: True
-    - gid_from_name: True
-    - shell: /sbin/nologin
-    - home: /var/kube-proxy
+    - makedirs: true
+    - dir_mode: 755
     - require:
-      - group: kube-proxy
-
-{% if pillar.get('is_systemd') %}
+      - service: docker
+      - service: kubelet
 
-{{ pillar.get('systemd_system_path') }}/kube-proxy.service:
+/var/log/kube-proxy.log:
   file.managed:
-    - source: salt://kube-proxy/kube-proxy.service
     - user: root
     - group: root
-  cmd.wait:
-    - name: /opt/kubernetes/helpers/services bounce kube-proxy
-    - watch:
-      - file: {{ environment_file }}
-      - file: {{ pillar.get('systemd_system_path') }}/kube-proxy.service
-      - file: /var/lib/kube-proxy/kubeconfig
-
-{% else %}
-
-/etc/init.d/kube-proxy:
-  file.managed:
-    - source: salt://kube-proxy/initd
-    - user: root
-    - group: root
-    - mode: 755
-
-{% endif %}
+    - mode: 644
 
-kube-proxy-service:
-  service.running:
+#stop legacy kube-proxy service 
+stop_kube-proxy:
+  service.dead:
     - name: kube-proxy
-    - enable: True
-    - watch:
-      - file: {{ environment_file }}
-{% if pillar.get('is_systemd') %}
-      - file: {{ pillar.get('systemd_system_path') }}/kube-proxy.service
-{% else %}
-      - file: /etc/init.d/kube-proxy
-{% endif %}
-      - file: /var/lib/kube-proxy/kubeconfig
-
-/var/lib/kube-proxy/kubeconfig:
-  file.managed:
-    - source: salt://kube-proxy/kubeconfig
-    - user: root
-    - group: root
-    - mode: 400
-    - makedirs: true
+    - enable: None

cluster/saltbase/salt/kube-proxy/kube-proxy.manifest

+{% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%}
+{% if grains.api_servers is defined -%}
+  {% set api_servers = "--master=https://" + grains.api_servers -%}
+{% else -%}
+  {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
+  {% set api_servers = "--master=https://" + ips[0][0] -%}
+{% endif -%}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ]  %}
+  {% set api_servers_with_port = api_servers -%}
+{% else -%}
+  {% set api_servers_with_port = api_servers + ":6443" -%}
+{% endif -%}
+{% set test_args = "" -%}
+{% if pillar['kubeproxy_test_args'] is defined -%}
+  {% set test_args=pillar['kubeproxy_test_args'] %}
+{% endif -%}
+
+# kube-proxy podspec
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-proxy
+    image: gcr.io/google_containers/kube-proxy:{{pillar['kube-proxy_docker_tag']}}
+    command:
+    - /bin/sh
+    - -c
+    - kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{pillar['log_level']}} {{test_args}} 1>>/var/log/kube-proxy.log 2>&1
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: /var/log
+      name: varlog
+      readOnly: false
+    - mountPath: /var/lib/kube-proxy/kubeconfig
+      name: kubeconfig
+      readOnly: false
+  volumes:
+  - hostPath:
+      path: /usr/share/ca-certificates
+    name: ssl-certs-host
+  - hostPath:
+      path: /var/lib/kube-proxy/kubeconfig
+    name: kubeconfig
+  - hostPath:
+      path: /var/log
+    name: varlog

cluster/saltbase/salt/supervisor/init.sls

@@ -52,30 +52,6 @@ monit:
     - mode: 755
     - makedirs: True
 
-{% if "kubernetes-pool" in grains.get('roles', []) %}
-/etc/supervisor/conf.d/kube-proxy.conf:
-  file:
-    - managed
-    - source: salt://supervisor/kube-proxy.conf
-    - user: root
-    - group: root
-    - mode: 644
-    - makedirs: True
-    - require_in: 
-      - pkg: supervisor
-    - require: 
-      - file: /usr/sbin/kube-proxy-checker.sh
-
-/usr/sbin/kube-proxy-checker.sh:
-  file:
-    - managed
-    - source: salt://supervisor/kube-proxy-checker.sh
-    - user: root
-    - group: root
-    - mode: 755
-    - makedirs: True
-{% endif %}
-
 {% if grains['roles'][0] == 'kubernetes-master' -%}
 /etc/supervisor/conf.d/kube-addons.conf:
   file:

cluster/saltbase/salt/top.sls

@@ -16,6 +16,7 @@ base:
     - helpers
     - cadvisor
     - kube-client-tools
+    - kube-node-unpacker
     - kubelet
 {% if pillar.get('network_provider', '').lower() == 'opencontrail' %}
     - opencontrail-networking-minion

hack/lib/golang.sh

@@ -129,6 +129,7 @@ readonly KUBE_STATIC_LIBRARIES=(
   kube-apiserver
   kube-controller-manager
   kube-scheduler
+  kube-proxy
 )
 
 kube::golang::is_statically_linked_library() {