Merge pull request #69368 from dims/upstream-heptio-kube-conformance

build/lib/release.sh

@@ -29,6 +29,7 @@ readonly RELEASE_TARS="${LOCAL_OUTPUT_ROOT}/release-tars"
 readonly RELEASE_IMAGES="${LOCAL_OUTPUT_ROOT}/release-images"
 
 KUBE_BUILD_HYPERKUBE=${KUBE_BUILD_HYPERKUBE:-y}
+KUBE_BUILD_CONFORMANCE=${KUBE_BUILD_CONFORMANCE:-y}
 
 # Validate a ci version
 #
@@ -289,15 +290,28 @@ function kube::release::build_hyperkube_image() {
   if [[ -n "${save_dir}" ]]; then
     "${DOCKER[@]}" save "${hyperkube_tag}" > "${save_dir}/hyperkube-${arch}.tar"
   fi
-  if [[ -z "${KUBE_DOCKER_IMAGE_TAG-}" || -z "${KUBE_DOCKER_REGISTRY-}" ]]; then
-    # not a release
-    kube::log::status "Deleting hyperkube image ${hyperkube_tag}"
-    "${DOCKER[@]}" rmi "${hyperkube_tag}" &>/dev/null || true
+  kube::log::status "Deleting hyperkube image ${hyperkube_tag}"
+  "${DOCKER[@]}" rmi "${hyperkube_tag}" &>/dev/null || true
+}
+
+function kube::release::build_conformance_image() {
+  local -r arch="$1"
+  local -r registry="$2"
+  local -r version="$3"
+  local -r save_dir="${4-}"
+  kube::log::status "Building conformance image for arch: ${arch}"
+  ARCH="${arch}" REGISTRY="${registry}" VERSION="${version}" \
+    make -C cluster/images/conformance/ build >/dev/null
+
+  local conformance_tag="${registry}/conformance-${arch}:${version}"
+  if [[ -n "${save_dir}" ]]; then
+    "${DOCKER[@]}" save "${conformance_tag}" > "${save_dir}/conformance-${arch}.tar"
   fi
+  kube::log::status "Deleting conformance image ${conformance_tag}"
+  "${DOCKER[@]}" rmi "${conformance_tag}" &>/dev/null || true
 }
 
-# This will take binaries that run on master and creates Docker images
-# that wrap the binary in them. (One docker image per binary)
+# This builds all the release docker images (One docker image per binary)
 # Args:
 #  $1 - binary_dir, the directory to save the tared images to.
 #  $2 - arch, architecture for which we are building docker images.
@@ -395,6 +409,10 @@ EOF
       kube::release::build_hyperkube_image "${arch}" "${docker_registry}" \
         "${docker_tag}" "${images_dir}" &
     fi
+    if [[ "${KUBE_BUILD_CONFORMANCE}" =~ [yY] ]]; then
+      kube::release::build_conformance_image "${arch}" "${docker_registry}" \
+        "${docker_tag}" "${images_dir}" &
+    fi
 
     kube::util::wait-for-jobs || { kube::log::error "previous Docker build failed"; return 1; }
     kube::log::status "Docker builds done"

build/release-images.sh

@@ -29,6 +29,9 @@ CMD_TARGETS="${KUBE_SERVER_IMAGE_TARGETS[*]}"
 if [[ "${KUBE_BUILD_HYPERKUBE}" =~ [yY] ]]; then
     CMD_TARGETS="${CMD_TARGETS} cmd/hyperkube"
 fi
+if [[ "${KUBE_BUILD_CONFORMANCE}" =~ [yY] ]]; then
+    CMD_TARGETS="${CMD_TARGETS} ${KUBE_CONFORMANCE_IMAGE_TARGETS[*]}"
+fi
 
 kube::build::verify_prereqs
 kube::build::build_image

build/root/Makefile

@@ -391,6 +391,7 @@ define RELEASE_IMAGES_HELP_INFO
 #
 # Args:
 #   KUBE_BUILD_HYPERKUBE: Whether to build hyperkube image as well. Set to 'n' to skip.
+#   KUBE_BUILD_CONFORMANCE: Whether to build conformance testing image as well. Set to 'n' to skip.
 #
 # Example:
 #   make release-images
@@ -432,6 +433,7 @@ define QUICK_RELEASE_IMAGES_HELP_INFO
 # Args:
 #   KUBE_FASTBUILD: Whether to cross-compile for other architectures. Set to 'false' to do so.
 #   KUBE_BUILD_HYPERKUBE: Whether to build hyperkube image as well. Set to 'n' to skip.
+#   KUBE_BUILD_CONFORMANCE: Whether to build conformance testing image as well. Set to 'n' to skip.
 #
 # Example:
 #   make quick-release-images

cluster/BUILD

@@ -15,6 +15,7 @@ filegroup(
         ":package-srcs",
         "//cluster/addons:all-srcs",
         "//cluster/gce:all-srcs",
+        "//cluster/images/conformance:all-srcs",
         "//cluster/images/etcd-version-monitor:all-srcs",
         "//cluster/images/etcd/migrate:all-srcs",
         "//cluster/images/hyperkube:all-srcs",

cluster/images/conformance/BUILD

+load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image", "container_layer")
+
+container_layer(
+    name = "cluster-srcs",
+    data_path = "/",
+    directory = "/kubernetes",
+    files = ["//cluster:all-srcs"],
+)
+
+container_layer(
+    name = "bins",
+    directory = "/usr/local/bin",
+    files = [
+        "//cmd/kubectl",
+        "//test/e2e:e2e.test",
+        "//vendor/github.com/onsi/ginkgo/ginkgo",
+    ],
+)
+
+container_image(
+    name = "conformance-internal",
+    base = "@debian-hyperkube-base-amd64//image",
+    cmd = [
+        "/bin/bash",
+        "-c",
+        "/run_e2e.sh",
+    ],
+    env = {
+        "E2E_FOCUS": "\[Conformance\]",
+        "E2E_SKIP": "",
+        "E2E_PARALLEL": "1",
+        "E2E_PROVIDER": "local",
+        "RESULTS_DIR": "/tmp/results",
+        "KUBECONFIG": "",
+    },
+    files = [
+        ":run_e2e.sh",
+    ],
+    layers = [
+        ":cluster-srcs",
+        ":bins",
+    ],
+    stamp = True,
+    workdir = "/usr/local/bin",
+)
+
+container_bundle(
+    name = "conformance",
+    images = {"k8s.gcr.io/conformance-amd64:{STABLE_DOCKER_TAG}": "conformance-internal"},
+    stamp = True,
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cluster/images/conformance/Dockerfile

+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM BASEIMAGE
+
+COPY ginkgo /usr/local/bin/
+COPY e2e.test /usr/local/bin/
+COPY kubectl /usr/local/bin/
+COPY run_e2e.sh /run_e2e.sh
+COPY cluster /kubernetes/cluster
+WORKDIR /usr/local/bin
+
+ENV E2E_FOCUS="\[Conformance\]"
+ENV E2E_SKIP=""
+ENV E2E_PROVIDER="local"
+ENV E2E_PARALLEL="1"
+ENV RESULTS_DIR="/tmp/results"
+ENV KUBECONFIG=""
+
+CMD [ "/bin/bash", "-c", "/run_e2e.sh" ]

cluster/images/conformance/Makefile

+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the conformance image.
+#
+# Usage:
+#   [ARCH=amd64] [REGISTRY="staging-k8s.gcr.io"] make (build|push) VERSION={some_released_version_of_kubernetes}
+
+REGISTRY?=staging-k8s.gcr.io
+ARCH?=amd64
+OUT_DIR?=_output
+GINKGO_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/ginkgo
+KUBECTL_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/kubectl
+E2E_TEST_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/e2e.test
+CLUSTER_DIR?=$(shell pwd)/../../../cluster/
+
+BASEIMAGE=k8s.gcr.io/debian-hyperkube-base-$(ARCH):0.10.2
+TEMP_DIR:=$(shell mktemp -d -t conformanceXXXXXX)
+
+all: build
+
+build:
+
+ifndef VERSION
+    $(error VERSION is undefined)
+endif
+	cp -r ./* ${TEMP_DIR}
+
+	cp ${GINKGO_BIN} ${TEMP_DIR}
+	cp ${KUBECTL_BIN} ${TEMP_DIR}
+	cp ${E2E_TEST_BIN} ${TEMP_DIR}
+	cp -r ${CLUSTER_DIR} ${TEMP_DIR}/cluster
+
+	chmod a+rx ${TEMP_DIR}/ginkgo
+	chmod a+rx ${TEMP_DIR}/kubectl
+	chmod a+rx ${TEMP_DIR}/e2e.test
+
+	cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
+
+ifneq ($(ARCH),amd64)
+	# Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel
+	docker run --rm --privileged multiarch/qemu-user-static:register --reset
+endif
+	docker build --pull -t ${REGISTRY}/conformance-${ARCH}:${VERSION} ${TEMP_DIR}
+	rm -rf "${TEMP_DIR}"
+
+push: build
+	docker push ${REGISTRY}/conformance-${ARCH}:${VERSION}
+ifeq ($(ARCH),amd64)
+	docker rmi ${REGISTRY}/conformance:${VERSION} 2>/dev/null || true
+	docker tag ${REGISTRY}/conformance-${ARCH}:${VERSION} ${REGISTRY}/conformance:${VERSION}
+	docker push ${REGISTRY}/conformance:${VERSION}
+endif
+
+.PHONY: build push all

cluster/images/conformance/OWNERS

+reviewers:
+  - timothysc
+  - dims
+  - ixdy
+  - spiffxp
+approvers:
+  - timothysc
+  - dims
+  - ixdy
+  - spiffxp
+labels:
+- sig/release

cluster/images/conformance/README.md

+### conformance
+
+`conformance` is a standalone container to launch Kubernetes end-to-end tests, for the purposes of conformance testing.
+`conformance` is built for multiple architectures and _the image is pushed automatically on every release._
+
+#### How to release by hand
+
+```console
+# First, build the binaries
+$ build/run.sh make cross
+
+# Build for linux/amd64 (default)
+# export REGISTRY=$HOST/$ORG to switch from staging-k8s.gcr.io
+
+$ make push VERSION={target_version} ARCH=amd64
+# ---> staging-k8s.gcr.io/conformance-amd64:VERSION
+# ---> staging-k8s.gcr.io/conformance:VERSION (image with backwards-compatible naming)
+
+$ make push VERSION={target_version} ARCH=arm
+# ---> staging-k8s.gcr.io/conformance-arm:VERSION
+
+$ make push VERSION={target_version} ARCH=arm64
+# ---> staging-k8s.gcr.io/conformance-arm64:VERSION
+
+$ make push VERSION={target_version} ARCH=ppc64le
+# ---> staging-k8s.gcr.io/conformance-ppc64le:VERSION
+
+$ make push VERSION={target_version} ARCH=s390x
+# ---> staging-k8s.gcr.io/conformance-s390x:VERSION
+```
+
+If you don't want to push the images, run `make` or `make build` instead
+
+
+#### How to setup RBAC needed
+
+```
+kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=default:default
+```
+
+#### How to run a single test
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+  name: e2e-producer-consumer-test
+spec:
+  containers:
+  - name: conformance-container
+    image: gcr.io/heptio-images/kube-conformance:latest
+    image: staging-k8s.gcr.io/conformance-amd64:v1.12.1
+    imagePullPolicy: IfNotPresent
+    env:
+    - name: E2E_FOCUS
+      value: "Pods should be submitted and removed"
+    volumeMounts:
+    - name: output-volume
+      mountPath: /tmp/results
+  volumes:
+  - name: output-volume
+    hostPath:
+      path: /tmp/results
+  restartPolicy: Never
+```
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/images/conformance/README.md?pixel)]()

cluster/images/conformance/run_e2e.sh

+#!/bin/bash
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Shutdown the tests gracefully then save the results
+shutdown () {
+    E2E_SUITE_PID=$(pgrep e2e.test)
+    echo "sending TERM to ${E2E_SUITE_PID}"
+    kill -s TERM "${E2E_SUITE_PID}"
+
+    # Kind of a hack to wait for this pid to finish.
+    # Since it's not a child of this shell we cannot use wait.
+    tail --pid ${E2E_SUITE_PID} -f /dev/null
+    saveResults
+}
+
+saveResults() {
+    cd "${RESULTS_DIR}" || exit
+    tar -czf e2e.tar.gz ./*
+    # mark the done file as a termination notice.
+    echo -n "${RESULTS_DIR}/e2e.tar.gz" > "${RESULTS_DIR}/done"
+}
+
+# We get the TERM from kubernetes and handle it gracefully
+trap shutdown TERM
+
+ginkgo_args=(
+    "--focus=${E2E_FOCUS}"
+    "--skip=${E2E_SKIP}"
+    "--noColor=true"
+)
+
+case ${E2E_PARALLEL} in
+    'y'|'Y')           ginkgo_args+=("--nodes=25") ;;
+    [1-9]|[1-9][0-9]*) ginkgo_args+=("--nodes=${E2E_PARALLEL}") ;;
+esac
+
+echo "/usr/local/bin/ginkgo ${ginkgo_args[@]} /usr/local/bin/e2e.test -- --disable-log-dump --repo-root=/kubernetes --provider=\"${E2E_PROVIDER}\" --report-dir=\"${RESULTS_DIR}\" --kubeconfig=\"${KUBECONFIG}\""
+/usr/local/bin/ginkgo "${ginkgo_args[@]}" /usr/local/bin/e2e.test -- --disable-log-dump --repo-root=/kubernetes --provider="${E2E_PROVIDER}" --report-dir="${RESULTS_DIR}" --kubeconfig="${KUBECONFIG}" | tee ${RESULTS_DIR}/e2e.log &
+# $! is the pid of tee, not ginkgo
+wait $(pgrep ginkgo)
+saveResults

hack/dev-push-conformance.sh

+#!/usr/bin/env bash
+
+# Copyright 2014 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script builds some binaries and then the conformance image.
+# REGISTRY and VERSION must be set.
+# Example usage:
+#   $ export REGISTRY=gcr.io/someone
+#   $ export VERSION=v1.4.0-testfix
+#   ./hack/dev-push-conformance.sh
+# That will build and push gcr.io/someone/conformance-amd64:v1.4.0-testfix
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT="$(dirname "${BASH_SOURCE}")/.."
+source "${KUBE_ROOT}/build/common.sh"
+
+if [[ -z "${REGISTRY:-}" ]]; then
+	echo "REGISTRY must be set"
+	exit -1
+fi
+if [[ -z "${VERSION:-}" ]]; then
+	echo "VERSION must be set"
+	exit -1
+fi
+
+IMAGE="${REGISTRY}/conformance-amd64:${VERSION}"
+
+kube::build::verify_prereqs
+kube::build::build_image
+kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl"
+kube::build::copy_output
+
+make -C "${KUBE_ROOT}/cluster/images/conformance" build
+docker push "${IMAGE}"

hack/lib/golang.sh

@@ -57,6 +57,20 @@ IFS=" " read -ra KUBE_SERVER_IMAGE_TARGETS <<< "$(kube::golang::server_image_tar
 readonly KUBE_SERVER_IMAGE_TARGETS
 readonly KUBE_SERVER_IMAGE_BINARIES=("${KUBE_SERVER_IMAGE_TARGETS[@]##*/}")
 
+# The set of conformance targets we build docker image for
+kube::golang::conformance_image_targets() {
+  # NOTE: this contains cmd targets for kube::release::build_conformance_image
+  local targets=(
+    vendor/github.com/onsi/ginkgo/ginkgo
+    test/e2e/e2e.test
+    cmd/kubectl
+  )
+  echo "${targets[@]}"
+}
+
+IFS=" " read -ra KUBE_CONFORMANCE_IMAGE_TARGETS <<< "$(kube::golang::conformance_image_targets)"
+readonly KUBE_CONFORMANCE_IMAGE_TARGETS
+
 # The set of server targets that we are only building for Kubernetes nodes
 # If you update this list, please also update build/BUILD.
 kube::golang::node_targets() {