Splitting master/node services into separate charm layers

cluster/juju/layers/kubeapi-load-balancer/README.md

+# kubeapi-load-balancer
+
+Simple NGINX reverse proxy to lend a hand in HA kubernetes-master deployments.
+
+

cluster/juju/layers/kubeapi-load-balancer/config.yaml

+options:
+  port:
+    type: int
+    default: 443
+    description: The port to run the loadbalancer

cluster/juju/layers/kubeapi-load-balancer/copyright

+Copyright 2016 The Kubernetes Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.

cluster/juju/layers/kubeapi-load-balancer/layer.yaml

+repo: https://github.com/kubernetes/kubernetes.git
+includes:
+  - 'layer:nginx'
+  - 'layer:tls-client'
+  - 'interface:public-address'
+options:
+  tls-client:
+    ca_certificate_path: '/srv/kubernetes/ca.crt'
+    server_certificate_path: '/srv/kubernetes/server.crt'
+    server_key_path: '/srv/kubernetes/server.key'
+    client_certificate_path: '/srv/kubernetes/client.crt'
+    client_key_path: '/srv/kubernetes/client.key'

cluster/juju/layers/kubeapi-load-balancer/metadata.yaml

+name: kubeapi-load-balancer
+summary: Nginx Load Balancer
+maintainers:
+  - Charles Butler <charles.butler@canonical.com>
+description: |
+  A round robin Nginx load balancer to distribute traffic for kubernetes apiservers.
+tags:
+  - misc
+subordinate: false
+series:
+  - xenial
+requires:
+  apiserver:
+    interface: http
+provides:
+  loadbalancer:
+      interface: public-address

cluster/juju/layers/kubeapi-load-balancer/reactive/load_balancer.py

+#!/usr/bin/env python
+
+# Copyright 2015 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import socket
+import subprocess
+
+from charms import layer
+from charms.reactive import when
+from charmhelpers.core import hookenv
+
+from charms.layer import nginx
+
+from subprocess import Popen
+from subprocess import PIPE
+from subprocess import STDOUT
+
+
+@when('certificates.available')
+def request_server_certificates(tls):
+    '''Send the data that is required to create a server certificate for
+    this server.'''
+    # Use the public ip of this unit as the Common Name for the certificate.
+    common_name = hookenv.unit_public_ip()
+    # Create SANs that the tls layer will add to the server cert.
+    sans = [
+        hookenv.unit_public_ip(),
+        hookenv.unit_private_ip(),
+        socket.gethostname(),
+    ]
+    # Create a path safe name by removing path characters from the unit name.
+    certificate_name = hookenv.local_unit().replace('/', '_')
+    # Request a server cert with this information.
+    tls.request_server_cert(common_name, sans, certificate_name)
+
+
+@when('nginx.available', 'apiserver.available',
+      'certificates.server.cert.available')
+def install_load_balancer(apiserver, tls):
+    ''' Create the default vhost template for load balancing '''
+    # Get the tls paths from the layer data.
+    layer_options = layer.options('tls-client')
+    server_cert_path = layer_options.get('server_certificate_path')
+    cert_exists = server_cert_path and os.path.isfile(server_cert_path)
+    server_key_path = layer_options.get('server_key_path')
+    key_exists = server_key_path and os.path.isfile(server_key_path)
+    # Do both the the key and certificate exist?
+    if cert_exists and key_exists:
+        # At this point the cert and key exist, and they are owned by root.
+        chown = ['chown', 'www-data:www-data', server_cert_path]
+        # Change the owner to www-data so the nginx process can read the cert.
+        subprocess.call(chown)
+        chown = ['chown', 'www-data:www-data', server_key_path]
+        # Change the owner to www-data so the nginx process can read the key.
+        subprocess.call(chown)
+
+        hookenv.open_port(hookenv.config('port'))
+        services = apiserver.services()
+        nginx.configure_site(
+                'apilb',
+                'apilb.conf',
+                server_name='_',
+                services=services,
+                port=hookenv.config('port'),
+                server_certificate=server_cert_path,
+                server_key=server_key_path,
+        )
+        hookenv.status_set('active', 'Loadbalancer ready.')
+
+
+@when('nginx.available')
+def set_nginx_version():
+    ''' Surface the currently deployed version of nginx to Juju '''
+    cmd = 'nginx -v'
+    p = Popen(cmd, shell=True,
+              stdin=PIPE,
+              stdout=PIPE,
+              stderr=STDOUT,
+              close_fds=True)
+    raw = p.stdout.read()
+    # The version comes back as:
+    # nginx version: nginx/1.10.0 (Ubuntu)
+    version = raw.split(b'/')[-1].split(b' ')[0]
+    hookenv.application_version_set(version.rstrip())
+
+
+@when('website.available')
+def provide_application_details(website):
+    ''' re-use the nginx layer website relation to relay the hostname/port
+    to any consuming kubernetes-workers, or other units that require the
+    kubernetes API '''
+    website.configure(port=hookenv.config('port'))
+
+
+@when('loadbalancer.available')
+def provide_loadbalancing(loadbalancer):
+    '''Send the public address and port to the public-address interface, so
+    the subordinates can get the public address of this loadbalancer.'''
+    loadbalancer.set_address_port(hookenv.unit_get('public-address'),
+                                  hookenv.config('port'))

cluster/juju/layers/kubeapi-load-balancer/templates/apilb.conf

+{% for app in services -%}
+upstream target_service {
+  {% for host in app['hosts'] -%}
+  server {{ host['hostname'] }}:{{ host['port'] }};
+  {% endfor %}
+}
+{% endfor %}
+
+
+server {
+    listen 443;
+    server_name {{ server_name }};
+
+    access_log /var/log/nginx.access.log;
+    error_log /var/log/nginx.error.log;
+
+    ssl on;
+    ssl_session_cache builtin:1000 shared:SSL:10m;
+    ssl_certificate {{ server_certificate }};
+    ssl_certificate_key {{ server_key }};
+    ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;
+    ssl_prefer_server_ciphers on;
+
+
+    location / {
+      proxy_buffering         off;
+      proxy_set_header        Host $host;
+      proxy_set_header        X-Real-IP $remote_addr;
+      proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header        X-Forwarded-Proto $scheme;
+      proxy_ssl_certificate {{ server_certificate }};
+      proxy_ssl_certificate_key {{ server_key }};
+      proxy_pass https://target_service;
+      proxy_read_timeout  90;
+    }
+}

cluster/juju/layers/kubernetes-e2e/README.md

+# Kubernetes end to end
+
+End-to-end (e2e) tests for Kubernetes provide a mechanism to test end-to-end
+behavior of the system, and is the last signal to ensure end user operations
+match developer specifications. Although unit and integration tests provide a
+good signal, in a distributed system like Kubernetes it is not uncommon that a
+minor change may pass all unit and integration tests, but cause unforeseen
+changes at the system level.
+
+The primary objectives of the e2e tests are to ensure a consistent and reliable
+behavior of the kubernetes code base, and to catch hard-to-test bugs before
+users do, when unit and integration tests are insufficient.
+
+
+## Usage
+
+To deploy the end-to-end test suite, it is best to deploy the
+[kubernetes-core bundle](https://github.com/juju-solutions/bundle-kubernetes-core)
+and then relate the `kubernetes-e2e` charm.
+
+```shell
+juju deploy kubernetes-core
+juju deploy kubernetes-e2e
+juju add-relation kubernetes-e2e kubernetes-master
+juju add-relation kubernetes-e2e easyrsa
+```
+
+
+Once the relations have settled, and the `kubernetes-e2e` charm reports
+ `Ready to test.` - you may kick off an end to end validation test.
+
+### Running the e2e test
+
+The e2e test is encapsulated as an action to ensure consistent runs of the
+end to end test. The defaults are sensible for most deployments.
+
+```shell
+juju run-action kubernetes-e2e/0 test
+```
+
+### Tuning the e2e test
+
+The e2e test is configurable. By default it will focus on or skip the declared
+conformance tests in a cloud agnostic way. Default behaviors are configurable.
+This allows the operator to test only a subset of the conformance tests, or to
+test more behaviors not enabled by default. You can see all tunable options on
+the charm by inspecting the schema output of the actions:
+
+```shell
+$ juju actions kubernetes-e2e --format=yaml --schema
+test:
+  description: Run end-to-end validation test suite
+  properties:
+    focus:
+      default: \[Conformance\]
+      description: Regex focus for executing the test
+      type: string
+    skip:
+      default: \[Flaky\]
+      description: Regex of tests to skip
+      type: string
+    timeout:
+      default: 30000
+      description: Timeout in nanoseconds
+      type: integer
+  title: test
+  type: object
+```
+
+
+As an example, you can run a more limited set of tests for rapid validation of
+a deployed cluster. The following example will skip the `Flaky`, `Slow`, and
+`Feature` labeled tests:
+
+```shell
+juju run-action kubernetes-e2e/0 skip='\[(Flaky|Slow|Feature:.*)\]'
+```
+
+> Note: the escaping of the regex due to how bash handles brackets.
+
+To see the different types of tests the Kubernetes end-to-end charm has access
+to, we encourage you to see the upstream documentation on the different types
+of tests, and to strongly understand what subsets of the tests you are running.
+
+[Kinds of tests](https://github.com/kubernetes/kubernetes/blob/master/docs/devel/e2e-tests.md#kinds-of-tests)
+
+### More information on end-to-end testing
+
+Along with the above descriptions, end-to-end testing is a much larger subject
+than this readme can encapsulate. There is far more information in the
+[end-to-end testing guide](https://github.com/kubernetes/kubernetes/blob/master/docs/devel/e2e-tests.md).
+
+### Evaluating end-to-end results
+
+It is not enough to just simply run the test. Result output is stored in two
+places. The raw output of the e2e run is available in the `juju show-action-output`
+command, as well as a flat file on disk on the `kubernetes-e2e` unit that
+executed the test.
+
+> Note: The results will only be available once the action has
+completed the test run. End-to-end testing can be quite time intensive. Often
+times taking **greater than 1 hour**, depending on configuration.
+
+##### Flat file
+
+```shell
+$ juju run-action kubernetes-e2e/0 test
+Action queued with id: 4ceed33a-d96d-465a-8f31-20d63442e51b
+
+$ juju scp kubernetes-e2e/0:4ceed33a-d96d-465a-8f31-20d63442e51b.log .
+```
+
+##### Action result output
+
+```shell
+$ juju run-action kubernetes-e2e/0 test
+Action queued with id: 4ceed33a-d96d-465a-8f31-20d63442e51b
+
+$ juju show-action-output 4ceed33a-d96d-465a-8f31-20d63442e51b
+```
+
+## Known issues
+
+The e2e test suite assumes egress network access. It will pull container
+images from `gcr.io`. You will need to have this registry unblocked in your
+firewall to successfully run e2e test results. Or you may use the exposed
+proxy settings [properly configured](https://github.com/juju-solutions/bundle-canonical-kubernetes#proxy-configuration)
+on the kubernetes-worker units.
+
+## Contact information
+
+Primary Authors: The ~containers team at Canonical
+
+- [Matt Bruzek <matthew.bruzek@canonical.com>](mailto:matthew.bruzek@canonical.com)
+- [Charles Butler <charles.butler@canonical.com>](mailto:charles.butler@canonical.com)
+
+More resources for help:
+
+- [Bug Tracker](https://github.com/juju-solutions/bundle-canonical-kubernetes/issues)
+- [Github Repository](https://github.com/kubernetes/kubernetes/)
+- [Mailing List](mailto:juju@lists.ubuntu.com)

cluster/juju/layers/kubernetes-e2e/actions.yaml

+test:
+  description: "Execute an end to end test."
+  params:
+    focus:
+      default: "\\[Conformance\\]"
+      description: Run tests matching the focus regex pattern.
+      type: string
+    parallelism:
+      default: 25
+      description: The number of test nodes to run in parallel.
+      type: integer
+    skip:
+      default: "\\[Flaky\\]|\\[Serial\\]"
+      description: Skip tests matching the skip regex pattern.
+      type: string
+    timeout:
+      default: 30000
+      description: Timeout in nanoseconds
+      type: integer

cluster/juju/layers/kubernetes-e2e/actions/test

+#!/bin/bash
+
+set -ex
+
+# Grab the action parameter values
+FOCUS=$(action-get focus)
+SKIP=$(action-get skip)
+PARALLELISM=$(action-get parallelism)
+
+if [ ! -f /home/ubuntu/.kube/config ]
+then
+  action-fail "Missing Kubernetes configuration."
+  action-set suggestion="Relate to the certificate authority, and kubernetes-master"
+  exit 0
+fi
+
+# get the host from the config file
+SERVER=$(cat /home/ubuntu/.kube/config | grep server | sed 's/    server: //')
+
+ACTION_HOME=/home/ubuntu
+ACTION_LOG=$ACTION_HOME/${JUJU_ACTION_UUID}.log
+ACTION_LOG_TGZ=$ACTION_LOG.tar.gz
+ACTION_JUNIT=$ACTION_HOME/${JUJU_ACTION_UUID}-junit
+ACTION_JUNIT_TGZ=$ACTION_JUNIT.tar.gz
+
+# This initializes an e2e build log with the START TIMESTAMP.
+echo "JUJU_E2E_START=$(date -u +%s)" | tee $ACTION_LOG
+echo "JUJU_E2E_VERSION=$(kubectl version | grep Server | cut -d " " -f 5 | cut -d ":" -f 2 | sed s/\"// | sed s/\",//)" | tee -a $ACTION_LOG
+ginkgo -nodes=$PARALLELISM $(which e2e.test) -- \
+  -kubeconfig /home/ubuntu/.kube/config \
+  -host $SERVER \
+  -ginkgo.focus $FOCUS \
+  -ginkgo.skip "$SKIP" \
+  -report-dir $ACTION_JUNIT 2>&1 | tee -a $ACTION_LOG
+
+# This appends the END TIMESTAMP to the e2e build log
+echo "JUJU_E2E_END=$(date -u +%s)" | tee -a $ACTION_LOG
+
+# set cwd to /home/ubuntu and tar the artifacts using a minimal directory
+# path. Extracing "home/ubuntu/1412341234/foobar.log is cumbersome in ci
+cd $ACTION_HOME/${JUJU_ACTION_UUID}-junit
+tar -czf $ACTION_JUNIT_TGZ *
+cd ..
+tar -czf $ACTION_LOG_TGZ ${JUJU_ACTION_UUID}.log
+
+action-set log="$ACTION_LOG_TGZ"
+action-set junit="$ACTION_JUNIT_TGZ"

cluster/juju/layers/kubernetes-e2e/layer.yaml

+repo: https://github.com/juju-solutions/layer-kubernetes-e2e
+includes:
+  - layer:basic
+  - layer:tls-client
+  - interface:http
+options:
+  tls-client:
+    ca_certificate_path: '/srv/kubernetes/ca.crt'
+    client_certificate_path: '/srv/kubernetes/client.crt'
+    client_key_path: '/srv/kubernetes/client.key'

cluster/juju/layers/kubernetes-e2e/metadata.yaml

+name: kubernetes-e2e
+summary: Run end-2-end validation of a clusters conformance
+maintainers:
+    - Matthew Bruzek <matthew.bruzek@canonical.com>
+    - Charles Butler <charles.butler@canonical.com>
+description: |
+  Deploy the Kubernetes e2e framework and validate the conformance of a
+  deployed kubernetes cluster
+tags:
+  - validation
+  - conformance
+series:
+  - xenial
+requires:
+  kubernetes-master:
+    interface: http
+resources:
+  e2e_amd64:
+    type: file
+    filename: e2e_amd64.tar.gz
+    description: Tarball of the e2e binary, and kubectl binary for amd64
+  e2e_ppc64el:
+    type: file
+    filename: e2e_ppc64le.tar.gz
+    description: Tarball of the e2e binary, and kubectl binary for ppc64le
+  e2e_s390x:
+    type: file
+    filename: e2e_s390x.tar.gz
+    description: Tarball of the e2e binary, and kubectl binary for s390x

cluster/juju/layers/kubernetes-e2e/reactive/kubernetes_e2e.py

+#!/usr/bin/env python
+
+# Copyright 2015 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from charms import layer
+
+from charms.reactive import hook
+from charms.reactive import is_state
+from charms.reactive import remove_state
+from charms.reactive import set_state
+from charms.reactive import when
+from charms.reactive import when_not
+
+from charmhelpers.core import hookenv
+
+from shlex import split
+
+from subprocess import call
+from subprocess import check_call
+from subprocess import check_output
+
+
+@hook('upgrade-charm')
+def reset_delivery_states():
+    ''' Remove the state set when resources are unpacked. '''
+    remove_state('kubernetes-e2e.installed')
+
+
+@when('kubernetes-e2e.installed')
+def messaging():
+    ''' Probe our relations to determine the propper messaging to the
+    end user '''
+
+    missing_services = []
+    if not is_state('kubernetes-master.available'):
+        missing_services.append('kubernetes-master')
+    if not is_state('certificates.available'):
+        missing_services.append('certificates')
+
+    if missing_services:
+        if len(missing_services) > 1:
+            subject = 'relations'
+        else:
+            subject = 'relation'
+
+        services = ','.join(missing_services)
+        message = 'Missing {0}: {1}'.format(subject, services)
+        hookenv.status_set('blocked', message)
+        return
+
+    hookenv.status_set('active', 'Ready to test.')
+
+
+@when_not('kubernetes-e2e.installed')
+def install_kubernetes_e2e():
+    ''' Deliver the e2e and kubectl components from the binary resource stream
+    packages declared in the charm '''
+    charm_dir = os.getenv('CHARM_DIR')
+    arch = determine_arch()
+
+    # Get the resource via resource_get
+    resource = 'e2e_{}'.format(arch)
+    try:
+        archive = hookenv.resource_get(resource)
+    except Exception:
+        message = 'Error fetching the {} resource.'.format(resource)
+        hookenv.log(message)
+        hookenv.status_set('blocked', message)
+        return
+
+    if not archive:
+        hookenv.log('Missing {} resource.'.format(resource))
+        hookenv.status_set('blocked', 'Missing {} resource.'.format(resource))
+        return
+
+    # Handle null resource publication, we check if filesize < 1mb
+    filesize = os.stat(archive).st_size
+    if filesize < 1000000:
+        hookenv.status_set('blocked',
+                           'Incomplete {} resource.'.format(resource))
+        return
+
+    hookenv.status_set('maintenance',
+                       'Unpacking {} resource.'.format(resource))
+
+    unpack_path = '{}/files/kubernetes'.format(charm_dir)
+    os.makedirs(unpack_path, exist_ok=True)
+    cmd = ['tar', 'xfvz', archive, '-C', unpack_path]
+    hookenv.log(cmd)
+    check_call(cmd)
+
+    services = ['e2e.test', 'ginkgo', 'kubectl']
+
+    for service in services:
+        unpacked = '{}/{}'.format(unpack_path, service)
+        app_path = '/usr/local/bin/{}'.format(service)
+        install = ['install', '-v', unpacked, app_path]
+        call(install)
+
+    set_state('kubernetes-e2e.installed')
+
+
+@when('tls_client.ca.saved', 'tls_client.client.certificate.saved',
+      'tls_client.client.key.saved', 'kubernetes-master.available',
+      'kubernetes-e2e.installed')
+@when_not('kubeconfig.ready')
+def prepare_kubeconfig_certificates(master):
+    ''' Prepare the data to feed to create the kubeconfig file. '''
+
+    layer_options = layer.options('tls-client')
+    # Get all the paths to the tls information required for kubeconfig.
+    ca = layer_options.get('ca_certificate_path')
+    key = layer_options.get('client_key_path')
+    cert = layer_options.get('client_certificate_path')
+
+    servers = get_kube_api_servers(master)
+
+    # pedantry
+    kubeconfig_path = '/home/ubuntu/.kube/config'
+
+    # Create kubernetes configuration in the default location for ubuntu.
+    create_kubeconfig('/root/.kube/config', servers[0], ca, key, cert,
+                      user='root')
+    create_kubeconfig(kubeconfig_path, servers[0], ca, key, cert,
+                      user='ubuntu')
+    # Set permissions on the ubuntu users kubeconfig to ensure a consistent UX
+    cmd = ['chown', 'ubuntu:ubuntu', kubeconfig_path]
+    check_call(cmd)
+
+    set_state('kubeconfig.ready')
+
+
+@when('kubernetes-e2e.installed', 'kubeconfig.ready')
+def set_app_version():
+    ''' Declare the application version to juju '''
+    cmd = ['kubectl', 'version', '--client']
+    from subprocess import CalledProcessError
+    try:
+        version = check_output(cmd).decode('utf-8')
+    except CalledProcessError:
+        message = "Missing kubeconfig causes errors. Skipping version set."
+        hookenv.log(message)
+        return
+    git_version = version.split('GitVersion:"v')[-1]
+    version_from = git_version.split('",')[0]
+    hookenv.application_version_set(version_from.rstrip())
+
+
+def create_kubeconfig(kubeconfig, server, ca, key, certificate, user='ubuntu',
+                      context='juju-context', cluster='juju-cluster'):
+    '''Create a configuration for Kubernetes based on path using the supplied
+    arguments for values of the Kubernetes server, CA, key, certificate, user
+    context and cluster.'''
+    # Create the config file with the address of the master server.
+    cmd = 'kubectl config --kubeconfig={0} set-cluster {1} ' \
+          '--server={2} --certificate-authority={3} --embed-certs=true'
+    check_call(split(cmd.format(kubeconfig, cluster, server, ca)))
+    # Create the credentials using the client flags.
+    cmd = 'kubectl config --kubeconfig={0} set-credentials {1} ' \
+          '--client-key={2} --client-certificate={3} --embed-certs=true'
+    check_call(split(cmd.format(kubeconfig, user, key, certificate)))
+    # Create a default context with the cluster.
+    cmd = 'kubectl config --kubeconfig={0} set-context {1} ' \
+          '--cluster={2} --user={3}'
+    check_call(split(cmd.format(kubeconfig, context, cluster, user)))
+    # Make the config use this new context.
+    cmd = 'kubectl config --kubeconfig={0} use-context {1}'
+    check_call(split(cmd.format(kubeconfig, context)))
+
+
+def get_kube_api_servers(master):
+    '''Return the kubernetes api server address and port for this
+    relationship.'''
+    hosts = []
+    # Iterate over every service from the relation object.
+    for service in master.services():
+        for unit in service['hosts']:
+            hosts.append('https://{0}:{1}'.format(unit['hostname'],
+                                                  unit['port']))
+    return hosts
+
+
+def determine_arch():
+    ''' dpkg wrapper to surface the architecture we are tied to'''
+    cmd = ['dpkg', '--print-architecture']
+    output = check_output(cmd).decode('utf-8')
+
+    return output.rstrip()

cluster/juju/layers/kubernetes-master/README.md

+# Kubernetes-master
+
+[Kubernetes](http://kubernetes.io/) is an open source system for managing 
+application containers across a cluster of hosts. The Kubernetes project was
+started by Google in 2014, combining the experience of running production 
+workloads combined with best practices from the community.
+
+The Kubernetes project defines some new terms that may be unfamiliar to users
+or operators. For more information please refer to the concept guide in the 
+[getting started guide](http://kubernetes.io/docs/user-guide/#concept-guide).
+
+This charm is an encapsulation of the Kubernetes master processes and the 
+operations to run on any cloud for the entire lifecycle of the cluster.
+
+This charm is built from other charm layers using the Juju reactive framework.
+The other layers focus on specific subset of operations making this layer 
+specific to operations of Kubernetes master processes.
+
+# Deployment
+
+This charm is not fully functional when deployed by itself. It requires other
+charms to model a complete Kubernetes cluster. A Kubernetes cluster needs a
+distributed key value store such as [Etcd](https://coreos.com/etcd/) and the
+kubernetes-worker charm which delivers the Kubernetes node services. A cluster
+requires a Software Defined Network (SDN) and Transport Layer Security (TLS) so
+the components in a cluster communicate securely. 
+
+Please take a look at the [Canonical Distribution of Kubernetes](https://jujucharms.com/canonical-kubernetes/) 
+or the [Kubernetes core](https://jujucharms.com/kubernetes-core/) bundles for 
+examples of complete models of Kubernetes clusters.
+
+# Resources
+
+The kubernetes-master charm takes advantage of the [Juju Resources](https://jujucharms.com/docs/2.0/developer-resources) 
+feature to deliver the Kubernetes software.
+
+In deployments on public clouds the Charm Store provides the resource to the
+charm automatically with no user intervention. Some environments with strict
+firewall rules may not be able to contact the Charm Store. In these network
+restricted  environments the resource can be uploaded to the model by the Juju
+operator.
+
+# Configuration
+
+This charm supports some configuration options to set up a Kubernetes cluster 
+that works in your environment:  
+
+#### dns_domain
+
+The domain name to use for the Kubernetes cluster for DNS.
+
+#### enable-dashboard-addons
+
+Enables the installation of Kubernetes dashboard, Heapster, Grafana, and
+InfluxDB.
+
+# DNS for the cluster
+
+The DNS add-on allows the pods to have a DNS names in addition to IP addresses.
+The Kubernetes cluster DNS server (based off the SkyDNS library) supports 
+forward lookups (A records), service lookups (SRV records) and reverse IP 
+address lookups (PTR records). More information about the DNS can be obtained
+from the [Kubernetes DNS admin guide](http://kubernetes.io/docs/admin/dns/).
+
+# Actions
+
+The kubernetes-master charm models a few one time operations called 
+[Juju actions](https://jujucharms.com/docs/stable/actions) that can be run by
+Juju users.
+
+#### create-rbd-pv
+
+This action creates RADOS Block Device (RBD) in Ceph and defines a Persistent
+Volume in Kubernetes so the containers can use durable storage. This action
+requires a relation to the ceph-mon charm before it can create the volume.
+
+#### restart
+
+This action restarts the master processes `kube-apiserver`, 
+`kube-controller-manager`, and `kube-scheduler` when the user needs a restart.
+
+# More information
+
+ - [Kubernetes github project](https://github.com/kubernetes/kubernetes)
+ - [Kubernetes issue tracker](https://github.com/kubernetes/kubernetes/issues)
+ - [Kubernetes documentation](http://kubernetes.io/docs/)
+ - [Kubernetes releases](https://github.com/kubernetes/kubernetes/releases)
+
+# Contact
+
+The kubernetes-master charm is free and open source operations created
+by the containers team at Canonical. 
+
+Canonical also offers enterprise support and customization services. Please
+refer to the [Kubernetes product page](https://www.ubuntu.com/cloud/kubernetes)
+for more details.

cluster/juju/layers/kubernetes-master/actions.yaml

+restart:
+  description: Restart the Kubernetes master services on demand.
+create-rbd-pv:
+  description: Create RADOS Block Device (RDB) volume in Ceph and creates PersistentVolume.
+  params:
+    name:
+      type: string
+      description: Name the persistent volume.
+      minLength: 1
+    size:
+      type: integer
+      description: Size in MB of the RBD volume.
+      minimum: 1
+    mode:
+      type: string
+      default: ReadWriteOnce
+      description: Access mode for the persistent volume.
+    filesystem:
+      type: string
+      default: xfs
+      description: File system type to format the volume.
+    skip-size-check:
+      type: boolean
+      default: false
+      description: Allow creation of overprovisioned RBD.
+  required:
+    - name
+    - size

cluster/juju/layers/kubernetes-master/actions/create-rbd-pv

+#!/usr/bin/env python
+
+# Copyright 2015 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from charms.templating.jinja2 import render
+from charms.reactive import is_state
+from charmhelpers.core.hookenv import action_get
+from charmhelpers.core.hookenv import action_set
+from charmhelpers.core.hookenv import action_fail
+from subprocess import check_call
+from subprocess import check_output
+from subprocess import CalledProcessError
+from tempfile import TemporaryDirectory
+import re
+import os
+import sys
+
+
+def main():
+    ''' Control logic to enlist Ceph RBD volumes as PersistentVolumes in
+    Kubernetes. This will invoke the validation steps, and only execute if
+    this script thinks the environment is 'sane' enough to provision volumes.
+    '''
+
+    # validate relationship pre-reqs before additional steps can be taken
+    if not validate_relation():
+        print('Failed ceph relationship check')
+        action_fail('Failed ceph relationship check')
+        return
+
+    if not is_ceph_healthy():
+        print('Ceph was not healthy.')
+        action_fail('Ceph was not healthy.')
+        return
+
+    context = {}
+
+    context['RBD_NAME'] = action_get_or_default('name').strip()
+    context['RBD_SIZE'] = action_get_or_default('size')
+    context['RBD_FS'] = action_get_or_default('filesystem').strip()
+    context['PV_MODE'] = action_get_or_default('mode').strip()
+
+    # Ensure we're not exceeding available space in the pool
+    if not validate_space(context['RBD_SIZE']):
+        return
+
+    # Ensure our paramters match
+    param_validation = validate_parameters(context['RBD_NAME'],
+                                           context['RBD_FS'],
+                                           context['PV_MODE'])
+    if not param_validation == 0:
+        return
+
+    if not validate_unique_volume_name(context['RBD_NAME']):
+        action_fail('Volume name collision detected. Volume creation aborted.')
+        return
+
+    context['monitors'] = get_monitors()
+
+    # Invoke creation and format the mount device
+    create_rbd_volume(context['RBD_NAME'],
+                      context['RBD_SIZE'],
+                      context['RBD_FS'])
+
+    # Create a temporary workspace to render our persistentVolume template, and
+    # enlist the RDB based PV we've just created
+    with TemporaryDirectory() as active_working_path:
+        temp_template = '{}/pv.yaml'.format(active_working_path)
+        render('rbd-persistent-volume.yaml', temp_template, context)
+
+        cmd = ['kubectl', 'create', '-f', temp_template]
+        debug_command(cmd)
+        check_call(cmd)
+
+
+def action_get_or_default(key):
+    ''' Convenience method to manage defaults since actions dont appear to
+    properly support defaults '''
+
+    value = action_get(key)
+    if value:
+        return value
+    elif key == 'filesystem':
+        return 'xfs'
+    elif key == 'size':
+        return 0
+    elif key == 'mode':
+        return "ReadWriteOnce"
+    elif key == 'skip-size-check':
+        return False
+    else:
+        return ''
+
+
+def create_rbd_volume(name, size, filesystem):
+    ''' Create the RBD volume in Ceph. Then mount it locally to format it for
+    the requested filesystem.
+
+    :param name - The name of the RBD volume
+    :param size - The size in MB of the volume
+    :param filesystem - The type of filesystem to format the block device
+    '''
+
+    # Create the rbd volume
+    # $ rbd create foo --size 50 --image-feature layering
+    command = ['rbd', 'create', '--size', '{}'.format(size), '--image-feature',
+               'layering', name]
+    debug_command(command)
+    check_call(command)
+
+    # Lift the validation sequence to determine if we actually created the
+    # rbd volume
+    if validate_unique_volume_name(name):
+        # we failed to create the RBD volume. whoops
+        action_fail('RBD Volume not listed after creation.')
+        print('Ceph RBD volume {} not found in rbd list'.format(name))
+        # hack, needs love if we're killing the process thread this deep in
+        # the call stack.
+        sys.exit(0)
+
+    mount = ['rbd', 'map', name]
+    debug_command(mount)
+    device_path = check_output(mount).strip()
+
+    try:
+        format_command = ['mkfs.{}'.format(filesystem), device_path]
+        debug_command(format_command)
+        check_call(format_command)
+        unmount = ['rbd', 'unmap', name]
+        debug_command(unmount)
+        check_call(unmount)
+    except CalledProcessError:
+        print('Failed to format filesystem and unmount. RBD created but not'
+              ' enlisted.')
+        action_fail('Failed to format filesystem and unmount.'
+                    ' RDB created but not enlisted.')
+
+
+def is_ceph_healthy():
+    ''' Probe the remote ceph cluster for health status '''
+    command = ['ceph', 'health']
+    debug_command(command)
+    health_output = check_output(command)
+    if b'HEALTH_OK' in health_output:
+        return True
+    else:
+        return False
+
+
+def get_monitors():
+    ''' Parse the monitors out of /etc/ceph/ceph.conf '''
+    found_hosts = []
+    # This is kind of hacky. We should be piping this in from juju relations
+    with open('/etc/ceph/ceph.conf', 'r') as ceph_conf:
+        for line in ceph_conf.readlines():
+            if 'mon host' in line:
+                # strip out the key definition
+                hosts = line.lstrip('mon host = ').split(' ')
+                for host in hosts:
+                    found_hosts.append(host)
+    return found_hosts
+
+
+def get_available_space():
+    ''' Determine the space available in the RBD pool. Throw an exception if
+    the RBD pool ('rbd') isn't found. '''
+    command = ['ceph', 'df']
+    debug_command(command)
+    out = check_output(command).decode('utf-8')
+    for line in out.splitlines():
+        stripped = line.strip()
+        if stripped.startswith('rbd'):
+            M = stripped.split()[-2].replace('M', '')
+            return int(M)
+    raise UnknownAvailableSpaceException('Unable to determine available space.')  # noqa
+
+
+def validate_unique_volume_name(name):
+    ''' Poll the CEPH-MON services to determine if we have a unique rbd volume
+    name to use. If there is naming collisions, block the request for volume
+    provisioning.
+
+    :param name - The name of the RBD volume
+    '''
+
+    command = ['rbd', 'list']
+    debug_command(command)
+    raw_out = check_output(command)
+
+    # Split the output on newlines
+    # output spec:
+    # $ rbd list
+    # foo
+    # foobar
+    volume_list = raw_out.decode('utf-8').splitlines()
+
+    for volume in volume_list:
+        if volume.strip() == name:
+            return False
+
+    return True
+
+
+def validate_relation():
+    ''' Determine if we are related to ceph. If we are not, we should
+    note this in the action output and fail this action run. We are relying
+    on specific files in specific paths to be placed in order for this function
+    to work. This method verifies those files are placed. '''
+
+    # TODO: Validate that the ceph-common package is installed
+    if not is_state('ceph-storage.available'):
+        message = 'Failed to detect connected ceph-mon'
+        print(message)
+        action_set({'pre-req.ceph-relation': message})
+        return False
+
+    if not os.path.isfile('/etc/ceph/ceph.conf'):
+        message = 'No Ceph configuration found in /etc/ceph/ceph.conf'
+        print(message)
+        action_set({'pre-req.ceph-configuration': message})
+        return False
+
+    # TODO: Validate ceph key
+
+    return True
+
+
+def validate_space(size):
+    if action_get_or_default('skip-size-check'):
+        return True
+    available_space = get_available_space()
+    if available_space < size:
+        msg = 'Unable to allocate RBD of size {}MB, only {}MB are available.'
+        action_fail(msg.format(size, available_space))
+        return False
+    return True
+
+
+def validate_parameters(name, fs, mode):
+    ''' Validate the user inputs to ensure they conform to what the
+    action expects. This method will check the naming characters used
+    for the rbd volume, ensure they have selected a fstype we are expecting
+    and the mode against our whitelist '''
+    name_regex = '^[a-zA-z0-9][a-zA-Z0-9|-]'
+
+    fs_whitelist = ['xfs', 'ext4']
+
+    # see http://kubernetes.io/docs/user-guide/persistent-volumes/#access-modes
+    # for supported operations on RBD volumes.
+    mode_whitelist = ['ReadWriteOnce', 'ReadOnlyMany']
+
+    fails = 0
+
+    if not re.match(name_regex, name):
+        message = 'Validation failed for RBD volume-name'
+        action_fail(message)
+        fails = fails + 1
+        action_set({'validation.name': message})
+
+    if fs not in fs_whitelist:
+        message = 'Validation failed for file system'
+        action_fail(message)
+        fails = fails + 1
+        action_set({'validation.filesystem': message})
+
+    if mode not in mode_whitelist:
+        message = "Validation failed for mode"
+        action_fail(message)
+        fails = fails + 1
+        action_set({'validation.mode': message})
+
+    return fails
+
+
+def debug_command(cmd):
+    ''' Print a debug statement of the command invoked '''
+    print("Invoking {}".format(cmd))
+
+
+class UnknownAvailableSpaceException(Exception):
+    pass
+
+
+if __name__ == '__main__':
+    main()

cluster/juju/layers/kubernetes-master/actions/restart

+#!/bin/bash
+
+set +ex
+
+# Restart the apiserver, controller-manager, and scheduler
+
+systemctl restart kube-apiserver
+
+action-set 'apiserver.status' 'restarted'
+
+systemctl restart kube-controller-manager
+
+action-set 'controller-manager.status' 'restarted'
+
+systemctl restart kube-scheduler
+
+action-set 'kube-scheduler.status' 'restarted'

cluster/juju/layers/kubernetes-master/config.yaml

+options:
+  enable-dashboard-addons:
+    type: boolean
+    default: True
+    description: Deploy the Kubernetes Dashboard and Heapster addons
+  dns_domain:
+    type: string
+    default: cluster.local
+    description: The local domain for cluster dns
+  service-cidr:
+    type: string
+    default: 10.152.183.0/24
+    description: CIDR to user for Kubernetes services. Cannot be changed after deployment.

cluster/juju/layers/kubernetes-master/copyright

+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

cluster/juju/layers/kubernetes-master/debug-scripts/kubectl

+#!/bin/sh
+set -ux
+
+alias kubectl="kubectl --kubeconfig=/home/ubuntu/config"
+
+kubectl cluster-info > $DEBUG_SCRIPT_DIR/cluster-info
+kubectl cluster-info dump > $DEBUG_SCRIPT_DIR/cluster-info-dump
+for obj in pods svc ingress secrets pv pvc rc; do
+  kubectl describe $obj --all-namespaces > $DEBUG_SCRIPT_DIR/describe-$obj
+done
+for obj in nodes; do
+  kubectl describe $obj > $DEBUG_SCRIPT_DIR/describe-$obj
+done

cluster/juju/layers/kubernetes-master/debug-scripts/kubernetes-master-services

+#!/bin/sh
+set -ux
+
+for service in kube-apiserver kube-controller-manager kube-scheduler; do
+  systemctl status $service > $DEBUG_SCRIPT_DIR/$service-systemctl-status
+  journalctl -u $service > $DEBUG_SCRIPT_DIR/$service-journal
+done
+
+mkdir -p $DEBUG_SCRIPT_DIR/etc-default
+cp -v /etc/default/kube* $DEBUG_SCRIPT_DIR/etc-default
+
+mkdir -p $DEBUG_SCRIPT_DIR/lib-systemd-system
+cp -v /lib/systemd/system/kube* $DEBUG_SCRIPT_DIR/lib-systemd-system

cluster/juju/layers/kubernetes-master/layer.yaml

+repo: https://github.com/kubernetes/kubernetes.git
+includes:
+  - 'layer:basic'
+  - 'layer:tls-client'
+  - 'layer:debug'
+  - 'interface:etcd'
+  - 'interface:http'
+  - 'interface:kubernetes-cni'
+  - 'interface:kube-dns'
+  - 'interface:ceph-admin'
+  - 'interface:public-address'
+options:
+  basic:
+    packages:
+      - socat
+  tls-client:
+    ca_certificate_path: '/srv/kubernetes/ca.crt'
+    server_certificate_path: '/srv/kubernetes/server.crt'
+    server_key_path: '/srv/kubernetes/server.key'
+    client_certificate_path: '/srv/kubernetes/client.crt'
+    client_key_path: '/srv/kubernetes/client.key'
+tactics:
+  - 'tactics.update_addons.UpdateAddonsTactic'

cluster/juju/layers/kubernetes-master/lib/charms/kubernetes/flagmanager.py

+#!/usr/bin/env python
+
+# Copyright 2015 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from charmhelpers.core import unitdata
+
+
+class FlagManager:
+    '''
+    FlagManager - A Python class for managing the flags to pass to an
+    application without remembering what's been set previously.
+
+    This is a blind class assuming the operator knows what they are doing.
+    Each instance of this class should be initialized with the intended
+    application to manage flags. Flags are then appended to a data-structure
+    and cached in unitdata for later recall.
+
+    THe underlying data-provider is backed by a SQLITE database on each unit,
+    tracking the dictionary, provided from the 'charmhelpers' python package.
+    Summary:
+    opts = FlagManager('docker')
+    opts.add('bip', '192.168.22.2')
+    opts.to_s()
+    '''
+
+    def __init__(self, daemon, opts_path=None):
+        self.db = unitdata.kv()
+        self.daemon = daemon
+        if not self.db.get(daemon):
+            self.data = {}
+        else:
+            self.data = self.db.get(daemon)
+
+    def __save(self):
+        self.db.set(self.daemon, self.data)
+
+    def add(self, key, value, strict=False):
+        '''
+        Adds data to the map of values for the DockerOpts file.
+        Supports single values, or "multiopt variables". If you
+        have a flag only option, like --tlsverify, set the value
+        to None. To preserve the exact value, pass strict
+        eg:
+        opts.add('label', 'foo')
+        opts.add('label', 'foo, bar, baz')
+        opts.add('flagonly', None)
+        opts.add('cluster-store', 'consul://a:4001,b:4001,c:4001/swarm',
+                 strict=True)
+        '''
+        if strict:
+            self.data['{}-strict'.format(key)] = value
+            self.__save()
+            return
+
+        if value:
+            values = [x.strip() for x in value.split(',')]
+            # handle updates
+            if key in self.data and self.data[key] is not None:
+                item_data = self.data[key]
+                for c in values:
+                    c = c.strip()
+                    if c not in item_data:
+                        item_data.append(c)
+                self.data[key] = item_data
+            else:
+                # handle new
+                self.data[key] = values
+        else:
+            # handle flagonly
+            self.data[key] = None
+        self.__save()
+
+    def remove(self, key, value):
+        '''
+        Remove a flag value from the DockerOpts manager
+        Assuming the data is currently {'foo': ['bar', 'baz']}
+        d.remove('foo', 'bar')
+        > {'foo': ['baz']}
+        :params key:
+        :params value:
+        '''
+        self.data[key].remove(value)
+        self.__save()
+
+    def destroy(self, key, strict=False):
+        '''
+        Destructively remove all values and key from the FlagManager
+        Assuming the data is currently {'foo': ['bar', 'baz']}
+        d.wipe('foo')
+        >{}
+        :params key:
+        :params strict:
+        '''
+        try:
+            if strict:
+                self.data.pop('{}-strict'.format(key))
+            else:
+                self.data.pop('key')
+        except KeyError:
+            pass
+
+    def to_s(self):
+        '''
+        Render the flags to a single string, prepared for the Docker
+        Defaults file. Typically in /etc/default/docker
+        d.to_s()
+        > "--foo=bar --foo=baz"
+        '''
+        flags = []
+        for key in self.data:
+            if self.data[key] is None:
+                # handle flagonly
+                flags.append("{}".format(key))
+            elif '-strict' in key:
+                # handle strict values, and do it in 2 steps.
+                # If we rstrip -strict it strips a tailing s
+                proper_key = key.rstrip('strict').rstrip('-')
+                flags.append("{}={}".format(proper_key, self.data[key]))
+            else:
+                # handle multiopt and typical flags
+                for item in self.data[key]:
+                    flags.append("{}={}".format(key, item))
+        return ' '.join(flags)

cluster/juju/layers/kubernetes/metadata.yaml → cluster/juju/layers/kubernetes-master/metadata.yaml

-name: kubernetes
-summary: Kubernetes is an application container orchestration platform.
+name: kubernetes-master
+summary: The Kubernetes control plane.
 maintainers:
     - Matthew Bruzek <matthew.bruzek@canonical.com>
     - Charles Butler <charles.butler@canonical.com>
@@ -11,9 +11,28 @@ description: |
   restart and place containers on healthy nodes if a node ever goes away.
 tags:
   - infrastructure
+  - kubernetes
+  - master
 subordinate: false
-requires:
-   etcd:
-     interface: etcd
-series: 
+series:
   - xenial
+provides:
+  kube-api-endpoint:
+    interface: http
+  cluster-dns:
+    interface: kube-dns
+  cni:
+    interface: kubernetes-cni
+    scope: container
+requires:
+  etcd:
+    interface: etcd
+  loadbalancer:
+    interface: public-address
+  ceph-storage:
+    interface: ceph-admin
+resources:
+  kubernetes:
+    type: file
+    filename: kubernetes.tar.gz
+    description: "A tarball packaged release of the kubernetes bins."

cluster/juju/layers/kubernetes/actions/guestbook-example → cluster/juju/layers/kubernetes-master/tactics/__init__.py

-#!/bin/bash
+#!/usr/bin/env python
 
-# Copyright 2016 The Kubernetes Authors.
+# Copyright 2015 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,22 +14,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-# Launch the Guestbook example in Kubernetes. This will use the pod and service
-# definitions from `files/guestbook-example/*.yaml` to launch a leader/follower
-# redis cluster, with a web-front end to collect user data and store in redis.
-# This example app can easily scale across multiple nodes, and exercises the
-# networking, pod creation/scale, service definition, and replica controller of
-# kubernetes.
-#
-# Lifted from github.com/kubernetes/kubernetes/examples/guestbook-example
-
-set -e
-
-if [ ! -d files/guestbook-example ]; then
-  mkdir -p files/guestbook-example
-  curl -o $CHARM_DIR/files/guestbook-example/guestbook-all-in-one.yaml https://raw.githubusercontent.com/kubernetes/kubernetes/master/examples/guestbook/all-in-one/guestbook-all-in-one.yaml
-fi
-
-kubectl create -f files/guestbook-example/guestbook-all-in-one.yaml
-

cluster/juju/layers/kubernetes-master/tactics/update_addons.py

+#!/usr/bin/env python
+
+# Copyright 2015 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import shutil
+import subprocess
+import tempfile
+import logging
+from contextlib import contextmanager
+
+import charmtools.utils
+from charmtools.build.tactics import Tactic
+
+
+description = """
+Update addon manifests for the charm.
+
+This will clone the kubernetes repo and place the addons in
+<charm>/templates/addons.
+
+Can be run with no arguments and from any folder.
+"""
+
+log = logging.getLogger(__name__)
+
+
+def clean_addon_dir(addon_dir):
+    """ Remove and recreate the addons folder """
+    log.debug("Cleaning " + addon_dir)
+    shutil.rmtree(addon_dir, ignore_errors=True)
+    os.makedirs(addon_dir)
+
+
+@contextmanager
+def kubernetes_repo():
+    """ Shallow clone kubernetes repo and clean up when we are done """
+    repo = "https://github.com/kubernetes/kubernetes.git"
+    path = tempfile.mkdtemp(prefix="kubernetes")
+    try:
+        log.info("Cloning " + repo)
+        cmd = ["git", "clone", "--depth", "1", repo, path]
+        process = subprocess.Popen(cmd, stderr=subprocess.PIPE)
+        stderr = process.communicate()[1].rstrip()
+        process.wait()
+        if process.returncode != 0:
+            log.error(stderr)
+            raise Exception("clone failed: exit code %d" % process.returncode)
+        log.debug(stderr)
+        yield path
+    finally:
+        shutil.rmtree(path)
+
+
+def add_addon(source, dest):
+    """ Add an addon manifest from the given source.
+
+    Any occurrences of 'amd64' are replaced with '{{ arch }}' so the charm can
+    fill it in during deployment. """
+    if os.path.isdir(dest):
+        dest = os.path.join(dest, os.path.basename(source))
+    log.debug("Copying: %s -> %s" % (source, dest))
+    with open(source, "r") as f:
+        content = f.read()
+    content = content.replace("amd64", "{{ arch }}")
+    with open(dest, "w") as f:
+        f.write(content)
+
+
+def update_addons(dest):
+    """ Update addons. This will clean the addons folder and add new manifests
+    from upstream. """
+    with kubernetes_repo() as repo:
+        log.info("Copying addons to charm")
+        clean_addon_dir(dest)
+        add_addon(repo + "/cluster/addons/dashboard/dashboard-controller.yaml",
+                  dest)
+        add_addon(repo + "/cluster/addons/dashboard/dashboard-service.yaml",
+                  dest)
+        add_addon(repo + "/cluster/addons/dns/kubedns-controller.yaml.in",
+                  dest + "/kubedns-controller.yaml")
+        add_addon(repo + "/cluster/addons/dns/kubedns-svc.yaml.in",
+                  dest + "/kubedns-svc.yaml")
+        influxdb = "/cluster/addons/cluster-monitoring/influxdb"
+        add_addon(repo + influxdb + "/grafana-service.yaml", dest)
+        add_addon(repo + influxdb + "/heapster-controller.yaml", dest)
+        add_addon(repo + influxdb + "/heapster-service.yaml", dest)
+        add_addon(repo + influxdb + "/influxdb-grafana-controller.yaml", dest)
+        add_addon(repo + influxdb + "/influxdb-service.yaml", dest)
+
+# Entry points
+
+
+class UpdateAddonsTactic(Tactic):
+    """ This tactic is used by charm-tools to dynamically populate the
+    template/addons folder at `charm build` time. """
+
+    @classmethod
+    def trigger(cls, entity, target=None, layer=None, next_config=None):
+        """ Determines which files the tactic should apply to. We only want
+        this tactic to trigger once, so let's use the templates/ folder
+        """
+        relpath = entity.relpath(layer.directory) if layer else entity
+        return relpath == "templates"
+
+    @property
+    def dest(self):
+        """ The destination we are writing to. This isn't a Tactic thing,
+        it's just a helper for UpdateAddonsTactic """
+        return self.target / "templates" / "addons"
+
+    def __call__(self):
+        """ When the tactic is called, update addons and put them directly in
+        our build destination """
+        update_addons(self.dest)
+
+    def sign(self):
+        """ Return signatures for the charm build manifest. We need to do this
+        because the addon template files were added dynamically """
+        sigs = {}
+        for file in os.listdir(self.dest):
+            path = self.dest / file
+            relpath = path.relpath(self.target.directory)
+            sigs[relpath] = (
+                self.current.url,
+                "dynamic",
+                charmtools.utils.sign(path)
+            )
+        return sigs
+
+
+def parse_args():
+    """ Parse args. This is solely done for the usage output with -h """
+    parser = argparse.ArgumentParser(description=description)
+    parser.parse_args()
+
+
+def main():
+    """ Update addons into the layer's templates/addons folder """
+    parse_args()
+    dest = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                        "../templates/addons"))
+    update_addons(dest)
+
+
+if __name__ == "__main__":
+    main()

cluster/juju/layers/kubernetes-master/templates/ceph-secret.yaml

+apiVersion: v1
+kind: Secret
+metadata:
+  name: ceph-secret
+type: Opaque
+data:
+  key: {{ secret }}

cluster/juju/layers/kubernetes-master/templates/ceph.conf

+[global]
+auth cluster required = {{ auth_supported }}
+auth service required = {{ auth_supported }}
+auth client required = {{ auth_supported }}
+keyring = /etc/ceph/$cluster.$name.keyring
+mon host = {{ mon_hosts }}
+fsid = {{ fsid }}
+
+log to syslog = {{ use_syslog }}
+err to syslog = {{ use_syslog }}
+clog to syslog = {{ use_syslog }}
+mon cluster log to syslog = {{ use_syslog }}
+debug mon = {{ loglevel }}/5
+debug osd = {{ loglevel }}/5
+
+[client]
+log file = /var/log/ceph.log
+

cluster/juju/layers/kubernetes-master/templates/kube-apiserver.defaults

+###
+# kubernetes system config
+#
+# The following values are used to configure the kube-apiserver
+#
+
+# The address on the local server to listen to.
+KUBE_API_ADDRESS="--insecure-bind-address=127.0.0.1"
+
+# The port on the local server to listen on.
+KUBE_API_PORT="--insecure-port=8080"
+
+# default admission control policies
+KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota"
+
+# Add your own!
+KUBE_API_ARGS="{{ kube_apiserver_flags }}"

cluster/juju/layers/kubernetes-master/templates/kube-apiserver.service

+[Unit]
+Description=Kubernetes API Server
+Documentation=http://kubernetes.io/docs/admin/kube-apiserver/
+After=network.target
+
+[Service]
+EnvironmentFile=-/etc/default/kube-defaults
+EnvironmentFile=-/etc/default/kube-apiserver
+ExecStart=/usr/local/bin/kube-apiserver \
+	    $KUBE_LOGTOSTDERR \
+	    $KUBE_LOG_LEVEL \
+	    $KUBE_API_ADDRESS \
+	    $KUBE_API_PORT \
+	    $KUBE_ALLOW_PRIV \
+	    $KUBE_ADMISSION_CONTROL \
+	    $KUBE_API_ARGS
+Restart=on-failure
+Type=notify
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

cluster/juju/layers/kubernetes-master/templates/kube-controller-manager.defaults

+
+###
+# The following values are used to configure the kubernetes controller-manager
+
+# defaults from config and apiserver should be adequate
+
+# Add your own!
+KUBE_CONTROLLER_MANAGER_ARGS="{{ kube_controller_manager_flags }}"

cluster/juju/layers/kubernetes-master/templates/kube-controller-manager.service

+
+[Unit]
+Description=Kubernetes Controller Manager
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+
+[Service]
+EnvironmentFile=-/etc/default/kube-defaults
+EnvironmentFile=-/etc/default/kube-controller-manager
+ExecStart=/usr/local/bin/kube-controller-manager \
+	    $KUBE_LOGTOSTDERR \
+	    $KUBE_LOG_LEVEL \
+	    $KUBE_MASTER \
+	    $KUBE_CONTROLLER_MANAGER_ARGS
+Restart=on-failure
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

cluster/juju/layers/kubernetes-master/templates/kube-defaults.defaults

+###
+# kubernetes system config
+#
+# The following values are used to configure various aspects of all
+# kubernetes services, including
+#
+#   kube-apiserver.service
+#   kube-controller-manager.service
+#   kube-scheduler.service
+#   kubelet.service
+#   kube-proxy.service
+# logging to stderr means we get it in the systemd journal
+KUBE_LOGTOSTDERR="--logtostderr=true"
+
+# journal message level, 0 is debug
+KUBE_LOG_LEVEL="--v=0"
+
+# Should this cluster be allowed to run privileged docker containers
+KUBE_ALLOW_PRIV="--allow-privileged=false"
+
+# How the controller-manager, scheduler, and proxy find the apiserver
+KUBE_MASTER="--master=http://127.0.0.1:8080"

cluster/juju/layers/kubernetes-master/templates/kube-scheduler.defaults

+###
+# kubernetes scheduler config
+
+# default config should be adequate
+
+# Add your own!
+KUBE_SCHEDULER_ARGS="{{ kube_scheduler_flags }}"

cluster/juju/layers/kubernetes-master/templates/kube-scheduler.service

+[Unit]
+Description=Kubernetes Scheduler Plugin
+Documentation=http://kubernetes.io/docs/admin/multiple-schedulers/
+
+[Service]
+EnvironmentFile=-/etc/default/kube-defaults
+EnvironmentFile=-/etc/default/kube-scheduler
+ExecStart=/usr/local/bin/kube-scheduler \
+	    $KUBE_LOGTOSTDERR \
+	    $KUBE_LOG_LEVEL \
+	    $KUBE_MASTER \
+	    $KUBE_SCHEDULER_ARGS
+Restart=on-failure
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

cluster/juju/layers/kubernetes-master/templates/rbd-persistent-volume.yaml

+# JUJU Internal Template used to enlist RBD volumes from the
+# `create-rbd-pv` action. This is a temporary file on disk to enlist resources.
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ RBD_NAME }}
+  annotations:
+    volume.beta.kubernetes.io/storage-class: "rbd"
+spec:
+  capacity:
+    storage: {{ RBD_SIZE }}M
+  accessModes:
+    - {{ PV_MODE }}
+  rbd:
+    monitors:
+      {% for host in monitors %}
+      - {{ host }}
+      {% endfor %}
+    pool: rbd
+    image: {{ RBD_NAME }}
+    user: admin
+    secretRef:
+      name: ceph-secret
+    fsType: {{ RBD_FS }}
+    readOnly: false
+    # persistentVolumeReclaimPolicy: Recycle

cluster/juju/layers/kubernetes-worker/HACKING.md

+ # Kubernetes Worker
+
+### Building from the layer
+
+You can clone the kubenetes-worker layer with git and build locally if you
+have the charm package/snap installed.
+
+```shell
+# Instal the snap
+sudo snap install charm --channel=edge
+
+# Set the build environment
+export JUJU_REPOSITORY=$HOME
+
+# Clone the layer and build it to our JUJU_REPOSITORY
+git clone https://github.com/juju-solutions/kubernetes
+cd kubernetes/cluster/juju/layers/kubernetes-worker
+charm build -r
+```
+
+### Contributing
+
+TBD
+
+

cluster/juju/layers/kubernetes-worker/README.md

+# Kubernetes Worker
+
+## Usage
+
+This charm deploys a container runtime, and additionally stands up the Kubernetes
+worker applications: kubelet, and kube-proxy.
+
+In order for this charm to be useful, it should be deployed with its companion
+charm [kubernetes-master](https://jujucharms.com/u/containers/kubernetes-master)
+and linked with an SDN-Plugin.
+
+This charm has also been bundled up for your convenience so you can skip the
+above steps, and deploy it with a single command:
+
+```shell
+juju deploy canonical-kubernetes
+```
+
+For more information about [Canonical Kubernetes](https://jujucharms.com/canonical-kubernetes)
+consult the bundle `README.md` file.
+
+
+## Scale out
+
+To add additional compute capacity to your Kubernetes workers, you may
+`juju add-unit` scale the cluster of applications. They will automatically
+join any related kubernetes-master, and enlist themselves as ready once the
+deployment is complete.
+
+## Operational actions
+
+The kubernetes-worker charm supports the following Operational Actions:
+
+#### Pause
+
+Pausing the workload enables administrators to both [drain](http://kubernetes.io/docs/user-guide/kubectl/kubectl_drain/) and [cordon](http://kubernetes.io/docs/user-guide/kubectl/kubectl_cordon/)
+a unit for maintenance.
+
+
+#### Resume
+
+Resuming the workload will [uncordon](http://kubernetes.io/docs/user-guide/kubectl/kubectl_uncordon/) a paused unit. Workloads will automatically migrate unless otherwise directed via their application declaration.
+
+## Known Limitations
+
+Kubernetes workers currently only support 'phaux' HA scenarios. Even when configured with an HA cluster string, they will only ever contact the first unit in the cluster map. To enalbe a proper HA story, kubernetes-worker units are encouraged to proxy through a [kubeapi-load-balancer](https://jujucharms.com/kubeapi-load-balancer)
+application. This enables a HA deployment without the need to
+re-render configuration and disrupt the worker services.
+
+External access to pods must be performed through a [Kubernetes
+Ingress Resource](http://kubernetes.io/docs/user-guide/ingress/). More
+information

cluster/juju/layers/kubernetes-worker/actions.yaml

+pause:
+    description: |
+      Cordon the unit, draining all active workloads.
+resume:
+    description: |
+      UnCordon the unit, enabling workload scheduling.
+microbot:
+    description: Launch microbot containers
+    params:
+      replicas:
+        type: integer
+        default: 3
+        description: Number of microbots to launch in Kubernetes.
+      delete:
+        type: boolean
+        default: False
+        description: Removes a microbots deployment, service, and ingress if True.

cluster/juju/layers/kubernetes-worker/actions/microbot

+#!/usr/bin/env python
+
+# Copyright 2015 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from charmhelpers.core.hookenv import action_get
+from charmhelpers.core.hookenv import action_set
+from charmhelpers.core.hookenv import unit_public_ip
+from charms.templating.jinja2 import render
+from subprocess import call
+
+
+context = {}
+context['replicas'] = action_get('replicas')
+context['delete'] = action_get('delete')
+context['public_address'] = unit_public_ip()
+
+if not context['replicas']:
+    context['replicas'] = 3
+
+# Declare a kubectl template when invoking kubectl
+kubectl = ['kubectl', '--kubeconfig=/srv/kubernetes/config']
+
+# Remove deployment if requested
+if context['delete']:
+    service_del = kubectl + ['delete', 'svc', 'microbot']
+    service_response = call(service_del)
+    deploy_del = kubectl + ['delete', 'deployment', 'microbot']
+    deploy_response = call(deploy_del)
+    ingress_del = kubectl + ['delete', 'ing', 'microbot-ingress']
+    ingress_response = call(ingress_del)
+
+    if ingress_response != 0:
+        action_set({'microbot-ing':
+                   'Failed removal of microbot ingress resource.'})
+    if deploy_response != 0:
+        action_set({'microbot-deployment':
+                   'Failed removal of microbot deployment resource.'})
+    if service_response != 0:
+        action_set({'microbot-service':
+                   'Failed removal of microbot service resource.'})
+    sys.exit(0)
+
+# Creation request
+
+render('microbot-example.yaml', '/etc/kubernetes/addons/microbot.yaml',
+       context)
+
+create_command = kubectl + ['create', '-f',
+                            '/etc/kubernetes/addons/microbot.yaml']
+
+create_response = call(create_command)
+
+if create_response == 0:
+    action_set({'address':
+               'microbot.{}.xip.io'.format(context['public_address'])})
+else:
+    action_set({'microbot-create': 'Failed microbot creation.'})

cluster/juju/layers/kubernetes-worker/actions/pause

+#!/bin/bash
+
+set -ex
+
+kubectl --kubeconfig=/srv/kubernetes/config cordon $(hostname)
+kubectl --kubeconfig=/srv/kubernetes/config drain $(hostname) --force
+status-set 'waiting' 'Kubernetes unit paused'

cluster/juju/layers/kubernetes-worker/actions/resume

+#!/bin/bash
+
+set -ex
+
+kubectl --kubeconfig=/srv/kubernetes/config uncordon $(hostname)
+status-set 'active' 'Kubernetes unit resumed'

cluster/juju/layers/kubernetes-worker/config.yaml

+options:
+  ingress:
+    type: boolean
+    default: true
+    description: |
+      Deploy the default http backend and ingress controller to handle
+      ingress requests.
+  labels:
+    type: string
+    default: ""
+    description: |
+      Labels can be used to organize and to select subsets of nodes in the
+      cluster. Declare node labels in key=value format, separated by spaces.

cluster/juju/layers/kubernetes-worker/copyright

+Copyright 2016 The Kubernetes Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.

cluster/juju/layers/kubernetes-worker/debug-scripts/inotify

+#!/bin/sh
+set -ux
+
+# We had to bump inotify limits once in the past, hence why this oddly specific
+# script lives here in kubernetes-worker.
+
+sysctl fs.inotify > $DEBUG_SCRIPT_DIR/sysctl-limits
+ls -l /proc/*/fd/* | grep inotify > $DEBUG_SCRIPT_DIR/inotify-instances

cluster/juju/layers/kubernetes-worker/debug-scripts/kubectl

+#!/bin/sh
+set -ux
+
+alias kubectl="kubectl --kubeconfig=/srv/kubernetes/config"
+
+kubectl cluster-info > $DEBUG_SCRIPT_DIR/cluster-info
+kubectl cluster-info dump > $DEBUG_SCRIPT_DIR/cluster-info-dump
+for obj in pods svc ingress secrets pv pvc rc; do
+  kubectl describe $obj --all-namespaces > $DEBUG_SCRIPT_DIR/describe-$obj
+done
+for obj in nodes; do
+  kubectl describe $obj > $DEBUG_SCRIPT_DIR/describe-$obj
+done

cluster/juju/layers/kubernetes-worker/debug-scripts/kubernetes-worker-services

+#!/bin/sh
+set -ux
+
+for service in kubelet kube-proxy; do
+  systemctl status $service > $DEBUG_SCRIPT_DIR/$service-systemctl-status
+  journalctl -u $service > $DEBUG_SCRIPT_DIR/$service-journal
+done
+
+mkdir -p $DEBUG_SCRIPT_DIR/etc-default
+cp -v /etc/default/kube* $DEBUG_SCRIPT_DIR/etc-default
+
+mkdir -p $DEBUG_SCRIPT_DIR/lib-systemd-system
+cp -v /lib/systemd/system/kube* $DEBUG_SCRIPT_DIR/lib-systemd-system

cluster/juju/layers/kubernetes-worker/exec.d/docker-compose/charm-pre-install

+# This stubs out charm-pre-install coming from layer-docker as a workaround for 
+# offline installs until https://github.com/juju/charm-tools/issues/301 is fixed.

cluster/juju/layers/kubernetes-worker/layer.yaml

+repo: https://github.com/kubernetes/kubernetes.git
+includes:
+  - 'layer:basic'
+  - 'layer:docker'
+  - 'layer:tls-client'
+  - 'layer:debug'
+  - 'interface:http'
+  - 'interface:kubernetes-cni'
+  - 'interface:kube-dns'
+options:
+  basic:
+    packages:
+      - 'nfs-common'
+      - 'ceph-common'
+      - 'socat'
+  tls-client:
+    ca_certificate_path: '/srv/kubernetes/ca.crt'
+    server_certificate_path: '/srv/kubernetes/server.crt'
+    server_key_path: '/srv/kubernetes/server.key'
+    client_certificate_path: '/srv/kubernetes/client.crt'
+    client_key_path: '/srv/kubernetes/client.key'

cluster/juju/layers/kubernetes-worker/lib/charms/kubernetes/flagmanager.py

+#!/usr/bin/env python
+
+# Copyright 2015 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from charmhelpers.core import unitdata
+
+
+class FlagManager:
+    '''
+    FlagManager - A Python class for managing the flags to pass to an
+    application without remembering what's been set previously.
+
+    This is a blind class assuming the operator knows what they are doing.
+    Each instance of this class should be initialized with the intended
+    application to manage flags. Flags are then appended to a data-structure
+    and cached in unitdata for later recall.
+
+    THe underlying data-provider is backed by a SQLITE database on each unit,
+    tracking the dictionary, provided from the 'charmhelpers' python package.
+    Summary:
+    opts = FlagManager('docker')
+    opts.add('bip', '192.168.22.2')
+    opts.to_s()
+    '''
+
+    def __init__(self, daemon, opts_path=None):
+        self.db = unitdata.kv()
+        self.daemon = daemon
+        if not self.db.get(daemon):
+            self.data = {}
+        else:
+            self.data = self.db.get(daemon)
+
+    def __save(self):
+        self.db.set(self.daemon, self.data)
+
+    def add(self, key, value, strict=False):
+        '''
+        Adds data to the map of values for the DockerOpts file.
+        Supports single values, or "multiopt variables". If you
+        have a flag only option, like --tlsverify, set the value
+        to None. To preserve the exact value, pass strict
+        eg:
+        opts.add('label', 'foo')
+        opts.add('label', 'foo, bar, baz')
+        opts.add('flagonly', None)
+        opts.add('cluster-store', 'consul://a:4001,b:4001,c:4001/swarm',
+                 strict=True)
+        '''
+        if strict:
+            self.data['{}-strict'.format(key)] = value
+            self.__save()
+            return
+
+        if value:
+            values = [x.strip() for x in value.split(',')]
+            # handle updates
+            if key in self.data and self.data[key] is not None:
+                item_data = self.data[key]
+                for c in values:
+                    c = c.strip()
+                    if c not in item_data:
+                        item_data.append(c)
+                self.data[key] = item_data
+            else:
+                # handle new
+                self.data[key] = values
+        else:
+            # handle flagonly
+            self.data[key] = None
+        self.__save()
+
+    def remove(self, key, value):
+        '''
+        Remove a flag value from the DockerOpts manager
+        Assuming the data is currently {'foo': ['bar', 'baz']}
+        d.remove('foo', 'bar')
+        > {'foo': ['baz']}
+        :params key:
+        :params value:
+        '''
+        self.data[key].remove(value)
+        self.__save()
+
+    def destroy(self, key, strict=False):
+        '''
+        Destructively remove all values and key from the FlagManager
+        Assuming the data is currently {'foo': ['bar', 'baz']}
+        d.wipe('foo')
+        >{}
+        :params key:
+        :params strict:
+        '''
+        try:
+            if strict:
+                self.data.pop('{}-strict'.format(key))
+            else:
+                self.data.pop('key')
+        except KeyError:
+            pass
+
+    def to_s(self):
+        '''
+        Render the flags to a single string, prepared for the Docker
+        Defaults file. Typically in /etc/default/docker
+        d.to_s()
+        > "--foo=bar --foo=baz"
+        '''
+        flags = []
+        for key in self.data:
+            if self.data[key] is None:
+                # handle flagonly
+                flags.append("{}".format(key))
+            elif '-strict' in key:
+                # handle strict values, and do it in 2 steps.
+                # If we rstrip -strict it strips a tailing s
+                proper_key = key.rstrip('strict').rstrip('-')
+                flags.append("{}={}".format(proper_key, self.data[key]))
+            else:
+                # handle multiopt and typical flags
+                for item in self.data[key]:
+                    flags.append("{}={}".format(key, item))
+        return ' '.join(flags)

cluster/juju/layers/kubernetes-worker/metadata.yaml

+name: kubernetes-worker
+summary: The workload bearing units of a kubernetes cluster
+maintainers:
+  - Charles Butler <charles.butler@canonical.com>
+  - Matthew Bruzek <matthew.bruzek@canonical.com>
+description: |
+  Kubernetes is an open-source platform for deploying, scaling, and operations
+  of application containers across a cluster of hosts. Kubernetes is portable
+  in that it works with public, private, and hybrid clouds. Extensible through
+  a pluggable infrastructure. Self healing in that it will automatically
+  restart and place containers on healthy nodes if a node ever goes away.
+tags:
+  - misc
+series:
+  - xenial
+subordinate: false
+requires:
+  kube-api-endpoint:
+    interface: http
+  kube-dns:
+    interface: kube-dns
+provides:
+  cni:
+    interface: kubernetes-cni
+    scope: container
+resources:
+  kubernetes:
+    type: file
+    filename: kubernetes.tar.gz
+    description: "An archive of kubernetes binaries for the worker."

cluster/juju/layers/kubernetes-worker/templates/default-http-backend.yaml

+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: default-http-backend
+spec:
+  replicas: 1
+  selector:
+    app: default-http-backend
+  template:
+    metadata:
+      labels:
+        app: default-http-backend
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+      - name: default-http-backend
+        # Any image is permissable as long as:
+        # 1. It serves a 404 page at /
+        # 2. It serves 200 on a /healthz endpoint
+        image: gcr.io/google_containers/defaultbackend:1.0
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        ports:
+        - containerPort: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: default-http-backend
+  labels:
+    app: default-http-backend
+spec:
+  ports:
+    - port: 80
+      protocol: TCP
+      targetPort: 80
+  selector:
+    app: default-http-backend

cluster/juju/layers/kubernetes-worker/templates/ingress-replication-controller.yaml

+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: nginx-ingress-controller
+  labels:
+    k8s-app: nginx-ingress-lb
+spec:
+  replicas: 1
+  selector:
+    k8s-app: nginx-ingress-lb
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-ingress-lb
+        name: nginx-ingress-lb
+    spec:
+      terminationGracePeriodSeconds: 60
+      # hostPort doesn't work with CNI, so we have to use hostNetwork instead
+      # see https://github.com/kubernetes/kubernetes/issues/23920
+      hostNetwork: true
+      containers:
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
+        name: nginx-ingress-lb
+        imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        # use downward API
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        ports:
+        - containerPort: 80
+        - containerPort: 443
+        args:
+        - /nginx-ingress-controller
+        - --default-backend-service=$(POD_NAMESPACE)/default-http-backend

cluster/juju/layers/kubernetes-worker/templates/kube-default

+###
+# kubernetes system config
+#
+# The following values are used to configure various aspects of all
+# kubernetes services, including
+#
+#   kube-apiserver.service
+#   kube-controller-manager.service
+#   kube-scheduler.service
+#   kubelet.service
+#   kube-proxy.service
+# logging to stderr means we get it in the systemd journal
+KUBE_LOGTOSTDERR="--logtostderr=true"
+
+# journal message level, 0 is debug
+KUBE_LOG_LEVEL="--v=0"
+
+# Should this cluster be allowed to run privileged docker containers
+KUBE_ALLOW_PRIV="--allow-privileged=false"
+
+# How the controller-manager, scheduler, and proxy find the apiserver
+KUBE_MASTER="--master={{ kube_api_endpoint }}"

cluster/juju/layers/kubernetes-worker/templates/kube-proxy.defaults

+KUBE_PROXY_ARGS="{{ kube_proxy_opts }}"

cluster/juju/layers/kubernetes-worker/templates/kube-proxy.service

+
+[Unit]
+Description=Kubernetes Kube-Proxy Server
+Documentation=http://kubernetes.io/docs/admin/kube-proxy/
+After=network.target
+
+[Service]
+EnvironmentFile=-/etc/default/kube-default
+EnvironmentFile=-/etc/default/kube-proxy
+ExecStart=/usr/local/bin/kube-proxy \
+	    $KUBE_LOGTOSTDERR \
+	    $KUBE_LOG_LEVEL \
+	    $KUBE_MASTER \
+	    $KUBE_PROXY_ARGS
+Restart=on-failure
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

cluster/juju/layers/kubernetes-worker/templates/kubelet.defaults

+# kubernetes kubelet (node) config
+
+# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
+KUBELET_ADDRESS="--address=0.0.0.0"
+
+# The port for the info server to serve on
+KUBELET_PORT="--port=10250"
+
+# You may leave this blank to use the actual hostname. If you override this
+# reachability problems become your own issue.
+# KUBELET_HOSTNAME="--hostname-override={{ JUJU_UNIT_NAME }}"
+
+# Add your own!
+KUBELET_ARGS="{{ kubelet_opts }}"

cluster/juju/layers/kubernetes-worker/templates/kubelet.service

+[Unit]
+Description=Kubernetes Kubelet Server
+Documentation=http://kubernetes.io/docs/admin/kubelet/
+After=docker.service
+Requires=docker.service
+
+[Service]
+WorkingDirectory=/var/lib/kubelet
+EnvironmentFile=-/etc/default/kube-default
+EnvironmentFile=-/etc/default/kubelet
+ExecStart=/usr/local/bin/kubelet \
+	    $KUBE_LOGTOSTDERR \
+	    $KUBE_LOG_LEVEL \
+	    $KUBELET_ADDRESS \
+	    $KUBELET_PORT \
+	    $KUBELET_HOSTNAME \
+	    $KUBE_ALLOW_PRIV \
+	    $KUBELET_ARGS
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

cluster/juju/layers/kubernetes-worker/templates/microbot-example.yaml

+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    app: microbot
+  name: microbot
+spec:
+  replicas: {{ replicas }}
+  selector:
+    matchLabels:
+      app: microbot
+  strategy: {}
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: microbot
+    spec:
+      containers:
+      - image: dontrebootme/microbot:v1
+        imagePullPolicy: ""
+        name: microbot
+        ports:
+        - containerPort: 80
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 80
+          initialDelaySeconds: 5
+          timeoutSeconds: 30
+        resources: {}
+      restartPolicy: Always
+      serviceAccountName: ""
+status: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: microbot
+  labels:
+    app: microbot
+spec:
+  ports:
+    - port: 80
+      protocol: TCP
+      targetPort: 80
+  selector:
+    app: microbot
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ name: microbot-ingress
+spec:
+ rules:
+   - host: microbot.{{ public_address }}.xip.io
+     http:
+       paths:
+         - path: /
+           backend:
+             serviceName: microbot
+             servicePort: 80

cluster/juju/layers/kubernetes-worker/wheelhouse.txt

+charms.templating.jinja2>=0.0.1,<2.0.0

cluster/juju/layers/kubernetes/README.md

-# kubernetes
-
-[Kubernetes](https://github.com/kubernetes/kubernetes) is an open
-source system for managing application containers across multiple hosts.
-This version of Kubernetes uses [Docker](http://www.docker.io/) to package,
-instantiate and run containerized applications.
-
-This charm is an encapsulation of the
-[Running Kubernetes locally via
-Docker](http://kubernetes.io/docs/getting-started-guides/docker)
-document. The released hyperkube image (`gcr.io/google_containers/hyperkube`)
-is currently pulled from a [Google owned container repository
-repository](https://cloud.google.com/container-registry/).  For this charm to
-work it will need access to the repository to `docker pull` the images.
-
-This charm was built from other charm layers using the reactive framework. The
-`layer:docker` is the base layer. For more information please read [Getting
-Started Developing charms](https://jujucharms.com/docs/devel/developer-getting-started)
-
-# Deployment
-The kubernetes charms require a relation to a distributed key value store
-(ETCD) which Kubernetes uses for persistent storage of all of its REST API
-objects.
-
-```
-juju deploy etcd
-juju deploy kubernetes
-juju add-relation kubernetes etcd
-```
-
-# Configuration
-For your convenience this charm supports some configuration options to set up
-a Kubernetes cluster that works in your environment:  
-
-**version**: Set the version of the Kubernetes containers to deploy. The
-version string must be in the following format "v#.#.#" where the numbers
-match with the
-[kubernetes release labels](https://github.com/kubernetes/kubernetes/releases)
-of the [kubernetes github project](https://github.com/kubernetes/kubernetes).
-Changing the version causes the all the Kubernetes containers to be restarted.
-
-**cidr**: Set the IP range for the Kubernetes cluster. eg: 10.1.0.0/16
-
-**dns_domain**: Set the DNS domain for the Kubernetes cluster.
-
-# Storage
-The kubernetes charm is built to handle multiple storage devices if the cloud
-provider works with
-[Juju storage](https://jujucharms.com/docs/devel/charms-storage).
-
-The 16.04 (xenial) release introduced [ZFS](https://en.wikipedia.org/wiki/ZFS)
-to Ubuntu. The xenial charm can use ZFS witha raidz pool. A raidz pool
-distributes parity along with the data (similar to a raid5 pool) and can suffer
-the loss of one drive while still retaining data. The raidz pool requires a
-minimum of 3 disks, but will accept more if they are provided.
-
-You can add storage to the kubernetes charm in increments of 3 or greater:
-
-```
-juju add-storage kubernetes/0 disk-pool=ebs,3,1G
-```
-
-**Note**: Due to a limitation of raidz you can not add individual disks to an
-existing pool. Should you need to expand the storage of the raidz pool, the
-additional add-storage commands must be the same number of disks as the original
-command. At this point the charm will have two raidz pools added together, both
-of which could handle the  loss of one disk each.
-
-The storage code handles the addition of devices to the charm and when it
-receives three disks creates a raidz pool that is mounted at the /srv/kubernetes
-directory by default. If you need the storage in another location you must
-change the `mount-point` value in layer.yaml before the charms is deployed.
-
-To avoid data loss you must attach the storage before making the connection to
-the etcd cluster.
-
-## State Events
-While this charm is meant to be a top layer, it can be used to build other
-solutions.  This charm sets or removes states from the reactive framework that
-other layers could react appropriately. The states that other layers would be
-interested in are as follows:
-
-**kubelet.available** - The hyperkube container has been run with the kubelet
-service and configuration that started the apiserver, controller-manager and
-scheduler containers.
-
-**proxy.available** - The hyperkube container has been run with the proxy
-service and configuration that handles Kubernetes networking.
-
-**kubectl.package.created** - Indicates the availability of the `kubectl`
-application along with the configuration needed to contact the cluster
-securely. You will need to download the `/home/ubuntu/kubectl_package.tar.gz`
-from the kubernetes leader unit to your machine so you can control the cluster.
-
-**kubedns.available** - Indicates when the Domain Name System (DNS) for the
-cluster is operational.
-
-
-# Kubernetes information
-
- - [Kubernetes github project](https://github.com/kubernetes/kubernetes)
- - [Kubernetes issue tracker](https://github.com/kubernetes/kubernetes/issues)
- - [Kubernetes Documenation](http://kubernetes.io/docs/)
- - [Kubernetes releases](https://github.com/kubernetes/kubernetes/releases)
-
-# Contact
-
- * Charm Author: Matthew Bruzek <Matthew.Bruzek@canonical.com>
- * Charm Contributor: Charles Butler <Charles.Butler@canonical.com>
-
-
-[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/juju/layers/kubernetes/README.md?pixel)]()

cluster/juju/layers/kubernetes/actions.yaml

-guestbook-example:
-    description: Launch the guestbook example in your k8s cluster

cluster/juju/layers/kubernetes/config.yaml

-options:
-  version:
-    type: string
-    default: "v1.2.3"
-    description: |
-      The version of Kubernetes to use in this charm. The version is inserted
-      in the configuration files that specify the hyperkube container to use
-      when starting a Kubernetes cluster. Changing this value will restart the
-      Kubernetes cluster.
-  cidr:
-    type: string
-    default: 10.1.0.0/16
-    description: |
-      Network CIDR to assign to Kubernetes service groups. This must not
-      overlap with any IP ranges assigned to nodes for pods.
-  dns_domain:
-    type: string
-    default: cluster.local
-    description: |
-      The domain name to use for the Kubernetes cluster by the
-      kubedns service.

cluster/juju/layers/kubernetes/layer.yaml

-includes: ['layer:leadership', 'layer:docker', 'layer:flannel', 'layer:storage', 'layer:tls', 'interface:etcd']
-repo: https://github.com/mbruzek/layer-k8s.git
-options:
-  storage:
-    storage-driver: zfs
-    mount-point: '/srv/kubernetes'

cluster/juju/layers/kubernetes/templates/docker-compose.yml

-# http://kubernetes.io/docs/getting-started-guides/docker/
-
-# # Start kubelet and then start master components as pods
-# docker run \
-#     --net=host \
-#     --pid=host \
-#     --privileged \
-#     --restart=on-failure \
-#     -d \
-#     -v /sys:/sys:ro \
-#     -v /var/run:/var/run:rw \
-#     -v /:/rootfs:ro \
-#     -v /var/lib/docker/:/var/lib/docker:rw \
-#     -v /var/lib/kubelet/:/var/lib/kubelet:rw \
-#     gcr.io/google_containers/hyperkube-${ARCH}:v${K8S_VERSION} \
-#     /hyperkube kubelet \
-#         --address=0.0.0.0 \
-#         --allow-privileged=true \
-#         --enable-server \
-#         --api-servers=http://localhost:8080 \
-#         --config=/etc/kubernetes/manifests-multi \
-#         --cluster-dns=10.0.0.10 \
-#         --cluster-domain=cluster.local \
-#         --containerized \
-#         --v=2
-
-master:
-  image: gcr.io/google_containers/hyperkube-{{ arch }}:{{ version }}
-  net: host
-  pid: host
-  privileged: true
-  restart: always
-  volumes:
-    - /:/rootfs:ro
-    - /sys:/sys:ro
-    - /var/lib/docker/:/var/lib/docker:rw
-    - /var/lib/kubelet/:/var/lib/kubelet:rw
-    - /var/run:/var/run:rw
-    - {{ manifest_directory }}:/etc/kubernetes/manifests:rw
-    - /srv/kubernetes:/srv/kubernetes
-  command: |
-    /hyperkube kubelet
-    --address="0.0.0.0"
-    --allow-privileged=true
-    --api-servers=http://localhost:8080
-    --cluster-dns={{ pillar['dns_server'] }}
-    --cluster-domain={{ pillar['dns_domain'] }}
-    --config=/etc/kubernetes/manifests
-    --containerized
-    --hostname-override="{{ private_address }}"
-    --tls-cert-file="/srv/kubernetes/server.crt"
-    --tls-private-key-file="/srv/kubernetes/server.key"
-    --v=2
-
-# Start kubelet without the config option and only kubelet starts.
-# kubelet gets the tls credentials from /var/lib/kubelet/kubeconfig
-# docker run \
-#     --net=host \
-#     --pid=host \
-#     --privileged \
-#     --restart=on-failure \
-#     -d \
-#     -v /sys:/sys:ro \
-#     -v /var/run:/var/run:rw  \
-#     -v /:/rootfs:ro \
-#     -v /var/lib/docker/:/var/lib/docker:rw \
-#     -v /var/lib/kubelet/:/var/lib/kubelet:rw \
-#     gcr.io/google_containers/hyperkube-${ARCH}:v${K8S_VERSION} \
-#     /hyperkube kubelet \
-#         --allow-privileged=true \
-#         --api-servers=http://${MASTER_IP}:8080 \
-#         --address=0.0.0.0 \
-#         --enable-server \
-#         --cluster-dns=10.0.0.10 \
-#         --cluster-domain=cluster.local \
-#         --containerized \
-#         --v=2
-
-
-kubelet:
-  image: gcr.io/google_containers/hyperkube-{{ arch }}:{{ version }}
-  net: host
-  pid: host
-  privileged: true
-  restart: always
-  volumes:
-    - /:/rootfs:ro
-    - /sys:/sys:ro
-    - /var/lib/docker/:/var/lib/docker:rw
-    - /var/lib/kubelet/:/var/lib/kubelet:rw
-    - /var/run:/var/run:rw
-    - /srv/kubernetes:/srv/kubernetes
-  command: |
-    /hyperkube kubelet
-    --address="0.0.0.0"
-    --allow-privileged=true
-    --api-servers=https://{{ master_address }}:6443
-    --cluster-dns={{ pillar['dns_server'] }}
-    --cluster-domain={{ pillar['dns_domain'] }}
-    --containerized
-    --hostname-override="{{ private_address }}"
-    --v=2
-
-# docker run \
-#     -d \
-#     --net=host \
-#     --privileged \
-#     --restart=on-failure \
-#     gcr.io/google_containers/hyperkube-${ARCH}:v${K8S_VERSION} \
-#     /hyperkube proxy \
-#         --master=http://${MASTER_IP}:8080 \
-#         --v=2
-proxy:
-  net: host
-  privileged: true
-  restart: always
-  image: gcr.io/google_containers/hyperkube-{{ arch }}:{{ version }}
-  command: |
-    /hyperkube proxy
-    --master=http://{{ master_address }}:8080
-    --v=2
-
-# cAdvisor (Container Advisor) provides container users an understanding of
-# the resource usage and performance characteristics of their running containers.
-cadvisor:
-  image: google/cadvisor:latest
-  volumes:
-    - /:/rootfs:ro
-    - /var/run:/var/run:rw
-    - /sys:/sys:ro
-    - /var/lib/docker:/var/lib/docker:ro
-  ports:
-    - 8088:8080
-  restart: always

cluster/juju/layers/kubernetes/templates/kubedns-controller.yaml

-# Copyright 2016 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This file should be kept in sync with cluster/addons/dns/kubedns-controller.yaml.base
-
-# Warning: This is a file generated from the base underscore template file: kubedns-controller.yaml.base
-
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: kube-dns
-  namespace: kube-system
-  labels:
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-spec:
-  # replicas: not specified here:
-  # 1. In order to make Addon Manager do not reconcile this replicas parameter.
-  # 2. Default is 1.
-  # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
-  strategy:
-    rollingUpdate:
-      maxSurge: 10%
-      maxUnavailable: 0
-  selector:
-    matchLabels:
-      k8s-app: kube-dns
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-dns
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
-        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
-    spec:
-      containers:
-      - name: kubedns
-        image: gcr.io/google_containers/k8s-dns-kube-dns-{{ arch }}:1.11.0
-        resources:
-          # TODO: Set memory limits when we've profiled the container for large
-          # clusters, then set request = limit to keep this container in
-          # guaranteed class. Currently, this container falls into the
-          # "burstable" category so the kubelet doesn't backoff from restarting it.
-          limits:
-            memory: 200Mi
-          requests:
-            cpu: 100m
-            memory: 100Mi
-        livenessProbe:
-          httpGet:
-            path: /healthcheck/kubedns
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        readinessProbe:
-          httpGet:
-            path: /readiness
-            port: 8081
-            scheme: HTTP
-          # we poll on pod startup for the Kubernetes master service and
-          # only setup the /readiness HTTP server once that's available.
-          initialDelaySeconds: 3
-          timeoutSeconds: 5
-        args:
-        # command = "/kube-dns"
-        - --domain={{ pillar['dns_domain'] }}.
-        - --dns-port=10053
-        - --config-map=kube-dns
-        - --v=2
-        - --kube_master_url=http://{{ private_address }}:8080
-        {{ pillar['federations_domain_map'] }}
-        env:
-          - name: PROMETHEUS_PORT
-            value: "10055"
-        ports:
-        - containerPort: 10053
-          name: dns-local
-          protocol: UDP
-        - containerPort: 10053
-          name: dns-tcp-local
-          protocol: TCP
-        - containerPort: 10055
-          name: metrics
-          protocol: TCP
-      - name: dnsmasq
-        image: gcr.io/google_containers/kube-dnsmasq-{{ arch }}:1.4
-        livenessProbe:
-          httpGet:
-            path: /healthcheck/dnsmasq
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        args:
-        - --cache-size=1000
-        - --no-resolv
-        - --server=127.0.0.1#10053
-        - --log-facility=-
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-      - name: sidecar
-        image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.11.0
-        livenessProbe:
-          httpGet:
-            path: /metrics
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        args:
-        - --v=2
-        - --logtostderr
-        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.{{ pillar['dns_domain'] }},5,A
-        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.{{ pillar['dns_domain'] }},5,A
-        ports:
-        - containerPort: 10054
-          name: metrics
-          protocol: TCP
-        resources:
-          requests:
-            memory: 20Mi
-            cpu: 10m
-      dnsPolicy: Default  # Don't use cluster DNS.

cluster/juju/layers/kubernetes/templates/kubedns-svc.yaml

-# Copyright 2016 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This file should be kept in sync with cluster/addons/dns/kubedns-svc.yaml.base
-
-# Warning: This is a file generated from the base underscore template file: kubedns-svc.yaml.base
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: kube-dns
-  namespace: kube-system
-  labels:
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "KubeDNS"
-spec:
-  selector:
-    k8s-app: kube-dns
-  clusterIP: {{ pillar['dns_server'] }}
-  ports:
-  - name: dns
-    port: 53
-    protocol: UDP
-  - name: dns-tcp
-    port: 53
-    protocol: TCP

cluster/juju/layers/kubernetes/templates/master.json

-{
-"apiVersion": "v1",
-"kind": "Pod",
-"metadata": {"name":"k8s-master"},
-"spec":{
-  "hostNetwork": true,
-  "containers":[
-    {
-      "name": "controller-manager",
-      "image": "gcr.io/google_containers/hyperkube-{{ arch }}:{{ version }}",
-      "command": [
-              "/hyperkube",
-              "controller-manager",
-              "--master=127.0.0.1:8080",
-              "--service-account-private-key-file=/srv/kubernetes/server.key",
-              "--root-ca-file=/srv/kubernetes/ca.crt",
-              "--min-resync-period=3m",
-              "--v=2"
-      ],
-      "volumeMounts": [
-        {
-          "name": "data",
-          "mountPath": "/srv/kubernetes"
-        }
-      ]
-    },
-    {
-      "name": "apiserver",
-      "image": "gcr.io/google_containers/hyperkube-{{ arch }}:{{ version }}",
-      "command": [
-              "/hyperkube",
-              "apiserver",
-              "--service-cluster-ip-range={{ cidr }}",
-              "--insecure-bind-address=0.0.0.0",
-              {% if etcd_dir -%}
-              "--etcd-cafile={{ etcd_ca }}",
-              "--etcd-keyfile={{ etcd_key }}",
-              "--etcd-certfile={{ etcd_cert }}",
-              {%- endif %}
-              "--etcd-servers={{ connection_string }}",
-              "--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota",
-              "--client-ca-file=/srv/kubernetes/ca.crt",
-              "--basic-auth-file=/srv/kubernetes/basic_auth.csv",
-              "--min-request-timeout=300",
-              "--tls-cert-file=/srv/kubernetes/server.crt",
-              "--tls-private-key-file=/srv/kubernetes/server.key",
-              "--token-auth-file=/srv/kubernetes/known_tokens.csv",
-              "--allow-privileged=true",
-              "--v=4"
-      ],
-      "volumeMounts": [
-        {
-          "name": "data",
-          "mountPath": "/srv/kubernetes"
-        },
-        {% if etcd_dir -%}
-        {
-          "name": "etcd-tls",
-          "mountPath": "{{ etcd_dir }}"
-        }
-        {%- endif %}
-      ]
-    },
-    {
-      "name": "scheduler",
-      "image": "gcr.io/google_containers/hyperkube-{{ arch }}:{{ version }}",
-      "command": [
-              "/hyperkube",
-              "scheduler",
-              "--master=127.0.0.1:8080",
-              "--v=2"
-        ]
-    },
-    {
-      "name": "setup",
-      "image": "gcr.io/google_containers/hyperkube-{{ arch }}:{{ version }}",
-      "command": [
-              "/setup-files.sh",
-              "IP:{{ private_address }},IP:{{ public_address }},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local"
-      ],
-      "volumeMounts": [
-        {
-          "name": "data",
-          "mountPath": "/data"
-        }
-      ]
-    }
-  ],
-  "volumes": [
-    {
-      "hostPath": {
-          "path": "/srv/kubernetes"
-      },
-      "name": "data"
-    },
-    {% if etcd_dir -%}
-    {
-      "hostPath": {
-        "path": "{{ etcd_dir }}"
-      },
-      "name": "etcd-tls"
-    }
-    {%- endif %}
-  ]
- }
-}

cluster/juju/layers/kubernetes/tests/tests.yaml

-tests: "*kubernetes*"
-bootstrap: false
-reset: false
-python_packages:
-  - tox

hack/verify-flags/exceptions.txt

@@ -27,11 +27,14 @@ cluster/gce/gci/configure-helper.sh:  sed -i -e "s@{{pillar\['allow_privileged'\
 cluster/gce/trusty/configure-helper.sh:  sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}"
 cluster/gce/trusty/configure-helper.sh:  sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
 cluster/gce/util.sh:    local node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
-cluster/juju/layers/kubernetes/reactive/k8s.py:    check_call(split(cmd.format(kubeconfig, cluster_name, server, ca)))
-cluster/juju/layers/kubernetes/reactive/k8s.py:    check_call(split(cmd.format(kubeconfig, context, cluster_name, user)))
-cluster/juju/layers/kubernetes/reactive/k8s.py:    client_key = '/srv/kubernetes/client.key'
-cluster/juju/layers/kubernetes/reactive/k8s.py:    cluster_name = 'kubernetes'
-cluster/juju/layers/kubernetes/reactive/k8s.py:    tlslib.client_key(None, client_key, user='ubuntu', group='ubuntu')
+cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py:            context['pillar'] = {'num_nodes': get_node_count()}
+cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py:    cluster_dns.set_dns_info(53, hookenv.config('dns_domain'), dns_ip)
+cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py:    ip = service_cidr().split('/')[0]
+cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py:    ip = service_cidr().split('/')[0]
+cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py:def send_cluster_dns_detail(cluster_dns):
+cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py:def service_cidr():
+cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py:    context.update({'kube_api_endpoint': ','.join(api_servers),
+cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py:def render_init_scripts(api_servers):
 cluster/lib/logging.sh:      local source_file=${BASH_SOURCE[$frame_no]}
 cluster/lib/logging.sh:    local source_file=${BASH_SOURCE[$stack_skip]}
 cluster/log-dump.sh:    local -r node_name="${1}"