Add unit test for configure-helper.

build/root/BUILD.root

@@ -28,13 +28,13 @@ gcs_upload(
     data = [
         ":_binary-artifacts-and-hashes",
         "//build/release-tars:release-tars-and-hashes",
-        "//cluster/gce:gcs-release-artifacts-and-hashes",
+        "//cluster/gce/gci:gcs-release-artifacts-and-hashes",
     ],
     tags = ["manual"],
     upload_paths = {
         "//:_binary-artifacts-and-hashes": "bin/linux/amd64",
         "//build/release-tars:release-tars-and-hashes": "",
-        "//cluster/gce:gcs-release-artifacts-and-hashes": "extra/gce",
+        "//cluster/gce/gci:gcs-release-artifacts-and-hashes": "extra/gce",
     },
 )
 

cluster/BUILD

@@ -30,9 +30,9 @@ pkg_tar(
     package_dir = "kubernetes/gci-trusty",
     deps = [
         "//cluster/addons",
-        "//cluster/gce:gce-master-manifests",
-        "//cluster/gce:gci-trusty-manifests",
         "//cluster/gce/addons",
+        "//cluster/gce/gci:gci-trusty-manifests",
+        "//cluster/gce/manifests:gce-master-manifests",
     ],
 )
 

cluster/gce/BUILD

@@ -3,17 +3,6 @@ package(default_visibility = ["//visibility:public"])
 load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
 load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
 
-pkg_tar(
-    name = "gci-trusty-manifests",
-    files = {
-        "//cluster/gce/gci/mounter": "gci-mounter",
-        "gci/configure-helper.sh": "gci-configure-helper.sh",
-        "gci/health-monitor.sh": "health-monitor.sh",
-    },
-    mode = "0755",
-    strip_prefix = ".",
-)
-
 filegroup(
     name = "package-srcs",
     srcs = glob(["**"]),
@@ -26,37 +15,8 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//cluster/gce/addons:all-srcs",
-        "//cluster/gce/gci/mounter:all-srcs",
+        "//cluster/gce/gci:all-srcs",
+        "//cluster/gce/manifests:all-srcs",
     ],
     tags = ["automanaged"],
 )
-
-# Having the COS code from the GCE cluster deploy hosted with the release is
-# useful for GKE.  This list should match the list in
-# kubernetes/release/lib/releaselib.sh.
-release_filegroup(
-    name = "gcs-release-artifacts",
-    srcs = [
-        "gci/configure.sh",
-        "gci/master.yaml",
-        "gci/node.yaml",
-    ],
-)
-
-pkg_tar(
-    name = "gce-master-manifests",
-    srcs = [
-        "manifests/abac-authz-policy.jsonl",
-        "manifests/cluster-autoscaler.manifest",
-        "manifests/e2e-image-puller.manifest",
-        "manifests/etcd.manifest",
-        "manifests/glbc.manifest",
-        "manifests/kube-addon-manager.yaml",
-        "manifests/kube-apiserver.manifest",
-        "manifests/kube-controller-manager.manifest",
-        "manifests/kube-proxy.manifest",
-        "manifests/kube-scheduler.manifest",
-        "manifests/rescheduler.manifest",
-    ],
-    mode = "0644",
-)

cluster/gce/gci/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
+load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "apiserver_manifest_test.go",
+        "configure_helper_test.go",
+    ],
+    data = [
+        ":scripts-test-data",
+        "//cluster/gce/manifests",
+    ],
+    deps = [
+        "//pkg/api/legacyscheme:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+# Having the COS code from the GCE cluster deploy hosted with the release is
+# useful for GKE.  This list should match the list in
+# kubernetes/release/lib/releaselib.sh.
+release_filegroup(
+    name = "gcs-release-artifacts",
+    srcs = [
+        "configure.sh",
+        "master.yaml",
+        "node.yaml",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+pkg_tar(
+    name = "gci-trusty-manifests",
+    files = {
+        "//cluster/gce/gci/mounter": "gci-mounter",
+        "configure-helper.sh": "gci-configure-helper.sh",
+        "health-monitor.sh": "health-monitor.sh",
+    },
+    mode = "0755",
+    strip_prefix = ".",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//cluster/gce/gci/mounter:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "scripts-test-data",
+    srcs = [
+        "configure-helper.sh",
+    ],
+)

cluster/gce/gci/apiserver_manifest_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gci
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"k8s.io/api/core/v1"
+)
+
+const (
+	/*
+	   Template for defining the environment state of configure-helper.sh
+	   The environment of configure-helper.sh is initially configured via kube-env file. However, as deploy-helper
+	   executes new variables are created. ManifestTestCase does not care where a variable came from. However, future
+	   test scenarios, may require such a distinction.
+	   The list of variables is, by no means, complete - this is what is required to run currently defined tests.
+	*/
+	deployHelperEnv = `
+readonly KUBE_HOME={{.KubeHome}}
+readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log
+readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log
+readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf
+readonly CA_CERT_BUNDLE_PATH=/foo/bar
+readonly APISERVER_SERVER_CERT_PATH=/foo/bar
+readonly APISERVER_SERVER_KEY_PATH=/foo/bar
+readonly APISERVER_CLIENT_CERT_PATH=/foo/bar
+readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
+readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
+readonly DOCKER_REGISTRY="k8s.gcr.io"
+readonly ENABLE_LEGACY_ABAC=false
+readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
+readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
+readonly LOG_OWNER_USER=$(whoami)
+readonly LOG_OWNER_GROUP=$(id -gn)
+ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
+ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
+readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}}
+`
+	kubeAPIServerManifestFileName = "kube-apiserver.manifest"
+	kmsPluginManifestFileName     = "kms-plugin-container.manifest"
+	kubeAPIServerStartFuncName    = "start-kube-apiserver"
+
+	// Position of containers within a pod manifest
+	kmsPluginContainerIndex        = 0
+	apiServerContainerIndexNoKMS   = 0
+	apiServerContainerIndexWithKMS = 1
+
+	//	command": [
+	//   "/bin/sh", - Index 0
+	//   "-c",      - Index 1
+	//   "exec /usr/local/bin/kube-apiserver " - Index 2
+	execArgsIndex = 2
+
+	socketVolumeMountIndexKMSPlugin = 1
+	socketVolumeMountIndexAPIServer = 0
+)
+
+type kubeAPIServerEnv struct {
+	KubeHome                     string
+	EncryptionProviderConfig     string
+	EncryptionProviderConfigPath string
+	ETCDKMSKeyID                 string
+}
+
+type kubeAPIServerManifestTestCase struct {
+	*ManifestTestCase
+	apiServerContainer v1.Container
+	kmsPluginContainer v1.Container
+}
+
+func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
+	return &kubeAPIServerManifestTestCase{
+		ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}),
+	}
+}
+
+func (c *kubeAPIServerManifestTestCase) mustLoadContainers() {
+	c.mustLoadPodFromManifest()
+
+	switch len(c.pod.Spec.Containers) {
+	case 1:
+		c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS]
+	case 2:
+		c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS]
+		c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex]
+	default:
+		c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers))
+	}
+}
+
+func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) {
+	c.mustInvokeFunc(deployHelperEnv, e)
+	c.mustLoadContainers()
+}
+
+func getEncryptionProviderConfigFlag(path string) string {
+	return fmt.Sprintf("--experimental-encryption-provider-config=%s", path)
+}
+
+func TestEncryptionProviderFlag(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	e := kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("FOO")),
+		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+	}
+
+	c.invokeTest(e)
+
+	expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
+	execArgs := c.apiServerContainer.Command[execArgsIndex]
+	if !strings.Contains(execArgs, expectedFlag) {
+		c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
+	}
+}
+
+func TestEncryptionProviderConfig(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
+	e := kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("FOO")),
+		EncryptionProviderConfigPath: p,
+	}
+
+	c.mustInvokeFunc(deployHelperEnv, e)
+
+	if _, err := os.Stat(p); err != nil {
+		c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
+	}
+}
+
+// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce
+// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag.
+func TestKMSEncryptionProviderConfig(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	e := kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+		ETCDKMSKeyID:                 "FOO",
+	}
+
+	c.invokeTest(e)
+
+	expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
+	execArgs := c.apiServerContainer.Command[execArgsIndex]
+	if !strings.Contains(execArgs, expectedFlag) {
+		c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
+	}
+
+	p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
+	if _, err := os.Stat(p); err != nil {
+		c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
+	}
+
+	d, err := ioutil.ReadFile(p)
+	if err != nil {
+		c.t.Fatalf("Failed to read encryption provider config %s", p)
+	}
+
+	if !strings.Contains(string(d), "name: grpc-kms-provider") {
+		c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d))
+	}
+}
+
+func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	var e = kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+		ETCDKMSKeyID:                 "FOO",
+	}
+
+	c.invokeTest(e)
+
+	k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath
+	a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath
+
+	if k != a {
+		t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s",
+			k, a, k, a)
+	}
+}

cluster/gce/gci/configure_helper_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gci
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"text/template"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+)
+
+const (
+	envScriptFileName         = "kube-env"
+	configureHelperScriptName = "configure-helper.sh"
+)
+
+type ManifestTestCase struct {
+	pod                 v1.Pod
+	envScriptPath       string
+	manifest            string
+	auxManifests        []string
+	kubeHome            string
+	manifestSources     string
+	manifestDestination string
+	manifestTemplateDir string
+	manifestTemplate    string
+	manifestFuncName    string
+	t                   *testing.T
+}
+
+func newManifestTestCase(t *testing.T, manifest, funcName string, auxManifests []string) *ManifestTestCase {
+	c := &ManifestTestCase{
+		t:                t,
+		manifest:         manifest,
+		auxManifests:     auxManifests,
+		manifestFuncName: funcName,
+	}
+
+	d, err := ioutil.TempDir("", "configure-helper-test")
+	if err != nil {
+		c.t.Fatalf("Failed to create temp directory: %v", err)
+	}
+
+	c.kubeHome = d
+	c.envScriptPath = filepath.Join(c.kubeHome, envScriptFileName)
+	c.manifestSources = filepath.Join(c.kubeHome, "kube-manifests", "kubernetes", "gci-trusty")
+
+	currentPath, err := os.Getwd()
+	if err != nil {
+		c.t.Fatalf("Failed to get current directory: %v", err)
+	}
+	gceDir := filepath.Dir(currentPath)
+	c.manifestTemplateDir = filepath.Join(gceDir, "manifests")
+	c.manifestTemplate = filepath.Join(c.manifestTemplateDir, c.manifest)
+	c.manifestDestination = filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests", c.manifest)
+
+	c.mustCopyFromTemplate()
+	c.mustCopyAuxFromTemplate()
+	c.mustCreateManifestDstDir()
+
+	return c
+}
+
+func (c *ManifestTestCase) mustCopyFromTemplate() {
+	if err := os.MkdirAll(c.manifestSources, os.ModePerm); err != nil {
+		c.t.Fatalf("Failed to create source directory: %v", err)
+	}
+
+	if err := copyFile(c.manifestTemplate, filepath.Join(c.manifestSources, c.manifest)); err != nil {
+		c.t.Fatalf("Failed to copy source manifest to KUBE_HOME: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) mustCopyAuxFromTemplate() {
+	for _, m := range c.auxManifests {
+		err := copyFile(filepath.Join(c.manifestTemplateDir, m), filepath.Join(c.manifestSources, m))
+		if err != nil {
+			c.t.Fatalf("Failed to copy source manifest %s to KUBE_HOME: %v", m, err)
+		}
+	}
+}
+
+func (c *ManifestTestCase) mustCreateManifestDstDir() {
+	p := filepath.Join(filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests"))
+	if err := os.MkdirAll(p, os.ModePerm); err != nil {
+		c.t.Fatalf("Failed to create designation folder for kube-apiserver.manifest: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) mustCreateEnv(envTemplate string, env interface{}) {
+	f, err := os.Create(filepath.Join(c.kubeHome, envScriptFileName))
+	if err != nil {
+		c.t.Fatalf("Failed to create envScript: %v", err)
+	}
+	defer f.Close()
+
+	t := template.Must(template.New("env").Parse(envTemplate))
+
+	if err = t.Execute(f, env); err != nil {
+		c.t.Fatalf("Failed to execute template: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) mustInvokeFunc(envTemplate string, env interface{}) {
+	c.mustCreateEnv(envTemplate, env)
+	args := fmt.Sprintf("source %s ; source %s --source-only ; %s", c.envScriptPath, configureHelperScriptName, c.manifestFuncName)
+	cmd := exec.Command("bash", "-c", args)
+
+	bs, err := cmd.CombinedOutput()
+	if err != nil {
+		c.t.Logf("%s", bs)
+		c.t.Fatalf("Failed to run configure-helper.sh: %v", err)
+	}
+	c.t.Logf("%s", string(bs))
+}
+
+func (c *ManifestTestCase) mustLoadPodFromManifest() {
+	json, err := ioutil.ReadFile(c.manifestDestination)
+	if err != nil {
+		c.t.Fatalf("Failed to read manifest: %s, %v", c.manifestDestination, err)
+	}
+
+	if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil {
+		c.t.Fatalf("Failed to decode manifest: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) tearDown() {
+	os.RemoveAll(c.kubeHome)
+}
+
+func copyFile(src, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		cerr := out.Close()
+		if cerr == nil {
+			err = cerr
+		}
+	}()
+	_, err = io.Copy(out, in)
+	return err
+}

cluster/gce/manifests/BUILD

+package(default_visibility = ["//visibility:public"])
+
+load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
+load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
+
+pkg_tar(
+    name = "gce-master-manifests",
+    srcs = [":manifests"],
+    mode = "0644",
+)
+
+filegroup(
+    name = "manifests",
+    srcs = [
+        "abac-authz-policy.jsonl",
+        "cluster-autoscaler.manifest",
+        "e2e-image-puller.manifest",
+        "etcd.manifest",
+        "glbc.manifest",
+        "kms-plugin-container.manifest",
+        "kube-addon-manager.yaml",
+        "kube-apiserver.manifest",
+        "kube-controller-manager.manifest",
+        "kube-proxy.manifest",
+        "kube-scheduler.manifest",
+        "rescheduler.manifest",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cluster/gce/manifests/kms-plugin-container.manifest

+{
+  "name": "kms-plugin",
+  "image": "gcr.io/google-containers/k8s-cloud-kms-plugin:v0.1.1",
+  "command": ["/k8s-cloud-kms-plugin", "--key-uri={{kms_key_uri}}", "--path-to-unix-socket={{kms_path_to_socket}}", "--gce-config={{gce_conf_path}}", "--logtostderr", "2>\&1"],
+  "livenessProbe": { "httpGet": {"host": "127.0.0.1", "port": 8081, "path": "/healthz"}, "initialDelaySeconds": 3, "timeoutSeconds": 3},
+  "ports":[{ "name": "healthz", "containerPort": 8081, "hostPort": 8081}, { "name": "metrics", "containerPort": 8082, "hostPort": 8082}],
+  "volumeMounts": [{{cloud_config_mount}}, {{kms_socket_mount}}]
+}

cluster/gce/manifests/kube-apiserver.manifest

@@ -15,6 +15,7 @@
 "spec":{
 "hostNetwork": true,
 "containers":[
+    {{kms_plugin_container}}
     {
     "name": "kube-apiserver",
     "image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",
@@ -47,6 +48,8 @@
         "hostPort": 8080}
         ],
     "volumeMounts": [
+        {{kms_socket_mount}}
+        {{encryption_provider_mount}}
         {{cloud_config_mount}}
         {{additional_cloud_config_mount}}
         {{webhook_config_mount}}
@@ -86,6 +89,8 @@
     }
 ],
 "volumes":[
+  {{kms_socket_volume}}
+  {{encryption_provider_volume}}
   {{cloud_config_volume}}
   {{additional_cloud_config_volume}}
   {{webhook_config_volume}}
@@ -133,4 +138,4 @@
         "path": "{{srv_sshproxy_path}}"}
   }
 ]
-}}
+}}
\ No newline at end of file