Adding cloudprovider support for Openstack kube-up provider.

Making the assumption that the person running kube-up has their Openstack environment setup, those same variables are being passed into heat, and then into openstack.conf. The salt codebase was modified to add openstack as well.

Adding cloudprovider support for Openstack kube-up provider.
5ae161e0 · Elson O Rodriguez · Lukasz Zajaczkowski · 21627f5b · 5ae161e0 · 5ae161e0
Commit 5ae161e0 authored Apr 13, 2016 by Elson O Rodriguez Committed by Lukasz Zajaczkowski Apr 28, 2016
11 changed files
--- a/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml
+++ b/cluster/openstack/kubernetes-heat/fragments/configure-salt.yaml
@@ -16,13 +16,22 @@ write_files:
        network_mode: openvswitch
        networkInterfaceName: eth0
        api_servers: $MASTER_IP
-        cloud: vagrant # It's not vagrant, but required to install Docker
+        cloud: openstack
+        cloud_config: /srv/kubernetes/openstack.conf
        roles:
          - $role
        runtime_config: ""
        docker_opts: ""
        master_extra_sans: "DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local,DNS:kubernetes-master"
        keep_host_etcd: true
+  - path: /srv/kubernetes/openstack.conf
+    content: |
+      [Global]
+      auth-url=$OS_AUTH_URL
+      username=$OS_USERNAME
+      password=$OS_PASSWORD
+      region=$OS_REGION_NAME
+      tenant-id=$OS_TENANT_ID
  - path: /srv/salt-overlay/pillar/cluster-params.sls
    content: |
      service_cluster_ip_range: 10.246.0.0/16

--- a/cluster/openstack/kubernetes-heat/fragments/run-salt.sh
+++ b/cluster/openstack/kubernetes-heat/fragments/run-salt.sh
@@ -43,9 +43,6 @@ fi
 # Salt server runs at locahost
 echo "127.0.0.1 salt" >> /etc/hosts

-# Currently heat template tells a lie that the target is Vagrant. If Vagrant cloud provider is enabled, "Unable to construct api.Node object for kubelet" error will occur.
-sed -e 's/{{cloud_provider}}//' -i /srv/salt/kubelet/default
-
 # Run salt-call
 # salt-call wants to start docker daemon but is unable to.
 # See <https://github.com/projectatomic/docker-storage-setup/issues/77>.

--- a/cluster/openstack/kubernetes-heat/kubecluster.yaml
+++ b/cluster/openstack/kubernetes-heat/kubecluster.yaml
@@ -90,6 +90,31 @@ parameters:
      timeout for the Wait Conditions
    default: 6000

+  os_auth_url:
+    type: string
+    description: OpenStack Auth URL
+    default: false
+
+  os_username:
+    type: string
+    description: OpenStack Username
+    default: false
+
+  os_password:
+    type: string
+    description: OpenStack Password
+    default: false
+
+  os_region_name:
+    type: string
+    description: OpenStack Region Name
+    default: false
+
+  os_tenant_id:
+    type: string
+    description: OpenStack Tenant ID
+    default: false
+
  enable_proxy:
    type: string
    description: Whether or not to enable proxy settings
@@ -259,6 +284,11 @@ resources:
          template: {get_file: fragments/configure-salt.yaml}
          params:
            "$MASTER_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
+            "$OS_AUTH_URL": {get_param: os_auth_url}
+            "$OS_USERNAME": {get_param: os_username}
+            "$OS_PASSWORD": {get_param: os_password}
+            "$OS_REGION_NAME": {get_param: os_region_name}
+            "$OS_TENANT_ID": {get_param: os_tenant_id}
            "$role": "kubernetes-master"

  run_salt:
@@ -354,6 +384,11 @@ resources:
          cluster_name: {get_param: "OS::stack_name"}
          secgroup_base: {get_resource: secgroup_base}
          secgroup_node: {get_resource: secgroup_node}
+          os_auth_url: {get_param: os_auth_url}
+          os_username: {get_param: os_username}
+          os_password: {get_param: os_password}
+          os_region_name: {get_param: os_region_name}
+          os_tenant_id: {get_param: os_tenant_id}
          enable_proxy: {get_param: enable_proxy }
          ftp_proxy: {get_param: ftp_proxy }
          http_proxy: {get_param: http_proxy }

--- a/cluster/openstack/kubernetes-heat/kubeminion.yaml
+++ b/cluster/openstack/kubernetes-heat/kubeminion.yaml
@@ -41,6 +41,31 @@ parameters:
    type: string
    description: Token used by kube-proxy

+  os_auth_url:
+    type: string
+    description: OpenStack Auth URL
+    default: false
+
+  os_username:
+    type: string
+    description: OpenStack Username
+    default: false
+
+  os_password:
+    type: string
+    description: OpenStack Password
+    default: false
+
+  os_region_name:
+    type: string
+    description: OpenStack Region Name
+    default: false
+
+  os_tenant_id:
+    type: string
+    description: OpenStack Tenant ID
+    default: false
+
  enable_proxy:
    type: string
    description: Whether or not to enable proxy settings
@@ -172,6 +197,11 @@ resources:
          template: {get_file: fragments/configure-salt.yaml}
          params:
            "$MASTER_IP": {get_param: kube_master_ip}
+            "$OS_AUTH_URL": {get_param: os_auth_url}
+            "$OS_USERNAME": {get_param: os_username}
+            "$OS_PASSWORD": {get_param: os_password}
+            "$OS_REGION_NAME": {get_param: os_region_name}
+            "$OS_TENANT_ID": {get_param: os_tenant_id}
            "$role": "kubernetes-pool"

  run_salt:

--- a/cluster/openstack/util.sh
+++ b/cluster/openstack/util.sh
@@ -200,6 +200,11 @@ function run-heat-script() {
      -P dns_nameserver=${DNS_SERVER} \
      -P kubernetes_salt_url=${swift_repo_url}/kubernetes-salt.tar.gz \
      -P kubernetes_server_url=${swift_repo_url}/kubernetes-server.tar.gz \
+      -P os_auth_url=${OS_AUTH_URL} \
+      -P os_username=${OS_USERNAME} \
+      -P os_password=${OS_PASSWORD} \
+      -P os_region_name=${OS_REGION_NAME} \
+      -P os_tenant_id=${OS_TENANT_ID} \
      -P enable_proxy=${ENABLE_PROXY} \
      -P ftp_proxy="${FTP_PROXY}" \
      -P http_proxy="${HTTP_PROXY}" \

--- a/cluster/saltbase/salt/kube-apiserver/init.sls
+++ b/cluster/saltbase/salt/kube-apiserver/init.sls
-{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere'] %}
+{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'openstack] %}
 # TODO: generate and distribute tokens on other cloud providers.
 /srv/kubernetes/known_tokens.csv:
  file.managed:

--- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
+++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
@@ -18,6 +18,10 @@
    {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
  {% endif -%}

+  {% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%}
+    {% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
+  {% endif -%}
+
  {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%}
    {% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
    {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%}
@@ -58,7 +62,7 @@
 {% set client_ca_file = "" -%}

 {% set secure_port = "6443" -%}
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ]    %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack']    %}
  {% set secure_port = "443" -%}
  {% set client_ca_file = "--client-ca-file=/srv/kubernetes/ca.crt" -%}
 {% endif -%}
@@ -72,7 +76,7 @@
 {% set basic_auth_file = "" -%}
 {% set authz_mode = "" -%}
 {% set abac_policy_file = "" -%}
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere']  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'openstack']  %}
  {% set token_auth_file = " --token-auth-file=/srv/kubernetes/known_tokens.csv" -%}
  {% set basic_auth_file = " --basic-auth-file=/srv/kubernetes/basic_auth.csv" -%}
  {% set authz_mode = " --authorization-mode=ABAC" -%}

--- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest
+++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest
@@ -37,6 +37,10 @@
  {% endif -%}
  {% set service_account_key = "--service-account-private-key-file=/srv/kubernetes/server.key" -%}

+  {% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%}
+    {% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
+  {% endif -%}
+
  {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%}
    {% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
    {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%}
@@ -46,7 +50,7 @@

 {% set root_ca_file = "" -%}

-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ]    %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack']    %}
   {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%}
 {% endif -%}


--- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
+++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
@@ -5,7 +5,7 @@
  {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
  {% set api_servers = "--master=https://" + ips[0][0] -%}
 {% endif -%}
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ]  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack' ]  %}
  {% set api_servers_with_port = api_servers -%}
 {% else -%}
  {% set api_servers_with_port = api_servers + ":6443" -%}

--- a/cluster/saltbase/salt/kubelet/default
+++ b/cluster/saltbase/salt/kubelet/default
@@ -16,7 +16,7 @@
 {% endif -%}

 # TODO: remove nginx for other cloud providers.
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ]  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack']  %}
  {% set api_servers_with_port = api_servers -%}
 {% else -%}
  {% set api_servers_with_port = api_servers + ":6443" -%}
@@ -28,7 +28,7 @@

 {% set reconcile_cidr_args = "" -%}
 {% if grains['roles'][0] == 'kubernetes-master' -%}
-  {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller'] -%}
+  {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] -%}

    # Unless given a specific directive, disable registration for the kubelet
    # running on the master.
@@ -52,6 +52,11 @@
  {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
 {% endif -%}

+{% set cloud_config = "" -%}
+{% if grains.cloud in [ 'openstack' ] and grains.cloud_config is defined -%}
+  {% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
+{% endif -%}
+
 {% set config = "--config=/etc/kubernetes/manifests" -%}

 {% set manifest_url = "" -%}
@@ -192,4 +197,4 @@
 {% endif -%}

 # test_args has to be kept at the end, so they'll overwrite any prior configuration
-DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{test_args}}"
+DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{test_args}}"
--- a/cluster/saltbase/salt/top.sls
+++ b/cluster/saltbase/salt/top.sls
@@ -58,6 +58,10 @@ base:
    - kube-controller-manager
    - kube-scheduler
    - supervisor
+{% if grains['cloud'] is defined and not grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'openstack' ] %}
+    - nginx
+{% endif %}
+    - cadvisor
    - kube-client-tools
    - kube-master-addons
    - kube-admission-controls
@@ -72,7 +76,7 @@ base:
    - logrotate
 {% endif %}
    - kube-addons
-{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere', 'photon-controller' ] %}
+{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere', 'photon-controller', 'openstack'] %}
    - docker
    - kubelet
 {% endif %}