Merge pull request #46738 from bdbauer/mysql

test/e2e/testing-manifests/statefulset/mysql-upgrade/configmap.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+data:
+  master.cnf: |
+    [mysqld]
+    log-bin
+  slave.cnf: |
+    [mysqld]
+    super-read-only

test/e2e/testing-manifests/statefulset/mysql-upgrade/service.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+spec:
+  ports:
+  - name: mysql
+    port: 3306
+  clusterIP: None
+  selector:
+    app: mysql
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql-read
+  labels:
+    app: mysql
+spec:
+  ports:
+  - name: mysql
+    port: 3306
+  selector:
+      app: mysql
+  type: LoadBalancer

test/e2e/testing-manifests/statefulset/mysql-upgrade/statefulset.yaml

+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: mysql
+spec:
+  serviceName: mysql
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: mysql
+      annotations:
+        pod.beta.kubernetes.io/init-containers: '[
+          {
+            "name": "init-mysql",
+            "image": "mysql:5.7",
+            "command": ["bash", "-c", "
+              set -ex\n
+              [[ `hostname` =~ -([0-9]+)$ ]] || exit 1\n
+              ordinal=${BASH_REMATCH[1]}\n
+              echo [mysqld] > /mnt/conf.d/server-id.cnf\n
+              echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf\n
+              if [[ $ordinal -eq 0 ]]; then\n
+                cp /mnt/config-map/master.cnf /mnt/conf.d/\n
+              else\n
+                cp /mnt/config-map/slave.cnf /mnt/conf.d/\n
+              fi\n
+            "],
+            "volumeMounts": [
+              {"name": "conf", "mountPath": "/mnt/conf.d"},
+              {"name": "config-map", "mountPath": "/mnt/config-map"}
+            ]
+          },
+          {
+            "name": "clone-mysql",
+            "image": "gcr.io/google-samples/xtrabackup:1.0",
+            "command": ["bash", "-c", "
+              set -ex\n
+              [[ -d /var/lib/mysql/mysql ]] && exit 0\n
+              [[ `hostname` =~ -([0-9]+)$ ]] || exit 1\n
+              ordinal=${BASH_REMATCH[1]}\n
+              [[ $ordinal -eq 0 ]] && exit 0\n
+              ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql\n
+              xtrabackup --prepare --target-dir=/var/lib/mysql\n
+            "],
+            "volumeMounts": [
+              {"name": "data", "mountPath": "/var/lib/mysql", "subPath": "mysql"},
+              {"name": "conf", "mountPath": "/etc/mysql/conf.d"}
+            ]
+          }
+        ]'
+    spec:
+      containers:
+      - name: mysql
+        image: mysql:5.7.15
+        env:
+        - name: MYSQL_ALLOW_EMPTY_PASSWORD
+          value: "1"
+        ports:
+        - name: mysql
+          containerPort: 3306
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+        resources:
+          requests:
+            cpu: 1
+            memory: 1Gi
+        livenessProbe:
+          exec:
+            command: ["mysqladmin", "ping"]
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"]
+          initialDelaySeconds: 5
+          timeoutSeconds: 1
+      - name: xtrabackup
+        image: gcr.io/google-samples/xtrabackup:1.0
+        ports:
+        - name: xtrabackup
+          containerPort: 3307
+        command:
+        - bash
+        - "-c"
+        - |
+          set -ex
+          cd /var/lib/mysql
+
+          if [[ -f xtrabackup_slave_info ]]; then
+            mv xtrabackup_slave_info change_master_to.sql.in
+            rm -f xtrabackup_binlog_info
+          elif [[ -f xtrabackup_binlog_info ]]; then
+            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
+            rm xtrabackup_binlog_info
+            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
+                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
+          fi
+
+          if [[ -f change_master_to.sql.in ]]; then
+            echo "Waiting for mysqld to be ready (accepting connections)"
+            until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
+
+            echo "Initializing replication from clone position"
+            mv change_master_to.sql.in change_master_to.sql.orig
+            mysql -h 127.0.0.1 <<EOF
+          $(<change_master_to.sql.orig),
+            MASTER_HOST='mysql-0.mysql',
+            MASTER_USER='root',
+            MASTER_PASSWORD='',
+            MASTER_CONNECT_RETRY=10;
+          START SLAVE;
+          EOF
+          fi
+
+          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
+            "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+      volumes:
+      - name: conf
+        emptyDir: {}
+      - name: config-map
+        configMap:
+          name: mysql
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+      annotations:
+        volume.alpha.kubernetes.io/storage-class: default
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      resources:
+        requests:
+          storage: 10Gi
+---
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+ name: mysql-pdb 
+ labels:
+   pdb: mysql
+spec:
+ minAvailable: 2
+ selector:
+   matchLabels:
+     app: mysql

test/e2e/testing-manifests/statefulset/mysql-upgrade/tester.yaml

+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: mysql-test-server
+spec:
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: test-server
+    spec:
+      containers:
+      - name: test-server
+        image: gcr.io/google-containers/mysql-e2e-test:0.1
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 2
+          periodSeconds: 2
+---
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: tester-pdb
+  labels:
+    pdb: test-server
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: test-server
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: test-server
+  name: test-server
+spec:
+  ports:
+    - port: 8080
+  selector:
+    app: test-server
+  type: LoadBalancer

test/e2e/upgrades/BUILD

@@ -18,6 +18,7 @@ go_library(
         "horizontal_pod_autoscalers.go",
         "ingress.go",
         "job.go",
+        "mysql.go",
         "persistent_volumes.go",
         "secrets.go",
         "services.go",

test/e2e/upgrades/mysql.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrades
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/pkg/util/version"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+const mysqlManifestPath = "test/e2e/testing-manifests/statefulset/mysql-upgrade"
+
+// MySqlUpgradeTest implements an upgrade test harness that polls a replicated sql database.
+type MySqlUpgradeTest struct {
+	ip               string
+	successfulWrites int
+	nextWrite        int
+	ssTester         *framework.StatefulSetTester
+}
+
+func (MySqlUpgradeTest) Name() string { return "mysql-upgrade" }
+
+func (MySqlUpgradeTest) Skip(upgCtx UpgradeContext) bool {
+	minVersion := version.MustParseSemantic("1.5.0")
+
+	for _, vCtx := range upgCtx.Versions {
+		if vCtx.Version.LessThan(minVersion) {
+			return true
+		}
+	}
+	return false
+}
+
+func mysqlKubectlCreate(ns, file string) {
+	path := filepath.Join(framework.TestContext.RepoRoot, mysqlManifestPath, file)
+	framework.RunKubectlOrDie("create", "-f", path, fmt.Sprintf("--namespace=%s", ns))
+}
+
+func (t *MySqlUpgradeTest) getServiceIP(f *framework.Framework, ns, svcName string) string {
+	svc, err := f.ClientSet.CoreV1().Services(ns).Get(svcName, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	ingress := svc.Status.LoadBalancer.Ingress
+	if len(ingress) == 0 {
+		return ""
+	}
+	return ingress[0].IP
+}
+
+// Setup creates a StatefulSet, HeadlessService, a Service to write to the db, and a Service to read
+// from the db. It then connects to the db with the write Service and populates the db with a table
+// and a few entries. Finally, it connects to the db with the read Service, and confirms the data is
+// available. The db connections are left open to be used later in the test.
+func (t *MySqlUpgradeTest) Setup(f *framework.Framework) {
+	ns := f.Namespace.Name
+	statefulsetPoll := 30 * time.Second
+	statefulsetTimeout := 10 * time.Minute
+	t.ssTester = framework.NewStatefulSetTester(f.ClientSet)
+
+	By("Creating a configmap")
+	mysqlKubectlCreate(ns, "configmap.yaml")
+
+	By("Creating a mysql StatefulSet")
+	t.ssTester.CreateStatefulSet(mysqlManifestPath, ns)
+
+	By("Creating a mysql-test-server deployment")
+	mysqlKubectlCreate(ns, "tester.yaml")
+
+	By("Getting the ingress IPs from the test-service")
+	err := wait.PollImmediate(statefulsetPoll, statefulsetTimeout, func() (bool, error) {
+		if t.ip = t.getServiceIP(f, ns, "test-server"); t.ip == "" {
+			return false, nil
+		}
+		if _, err := t.countNames(); err != nil {
+			framework.Logf("Service endpoint is up but isn't responding")
+			return false, nil
+		}
+		return true, nil
+	})
+	Expect(err).NotTo(HaveOccurred())
+	framework.Logf("Service endpoint is up")
+
+	By("Adding 2 names to the database")
+	Expect(t.addName(strconv.Itoa(t.nextWrite))).NotTo(HaveOccurred())
+	Expect(t.addName(strconv.Itoa(t.nextWrite))).NotTo(HaveOccurred())
+
+	By("Verifying that the 2 names have been inserted")
+	count, err := t.countNames()
+	Expect(err).NotTo(HaveOccurred())
+	Expect(count).To(Equal(2))
+}
+
+// Test continually polls the db using the read and write connections, inserting data, and checking
+// that all the data is readable.
+func (t *MySqlUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) {
+	var writeSuccess, readSuccess, writeFailure, readFailure int
+	By("Continuously polling the database during upgrade.")
+	go wait.Until(func() {
+		_, err := t.countNames()
+		if err != nil {
+			framework.Logf("Error while trying to read data: %v", err)
+			readFailure++
+		} else {
+			readSuccess++
+		}
+	}, framework.Poll, done)
+
+	wait.Until(func() {
+		err := t.addName(strconv.Itoa(t.nextWrite))
+		if err != nil {
+			framework.Logf("Error while trying to write data: %v", err)
+			writeFailure++
+		} else {
+			writeSuccess++
+		}
+	}, framework.Poll, done)
+
+	t.successfulWrites = writeSuccess
+	framework.Logf("Successful reads: %d", readSuccess)
+	framework.Logf("Successful writes: %d", writeSuccess)
+	framework.Logf("Failed reads: %d", readFailure)
+	framework.Logf("Failed writes: %d", writeFailure)
+
+	// TODO: Not sure what the ratio defining a successful test run should be. At time of writing the
+	// test, failures only seem to happen when a race condition occurs (read/write starts, doesn't
+	// finish before upgrade interferes).
+
+	readRatio := float64(readSuccess) / float64(readSuccess+readFailure)
+	writeRatio := float64(writeSuccess) / float64(writeSuccess+writeFailure)
+	if readRatio < 0.75 {
+		framework.Failf("Too many failures reading data. Success ratio: %f", readRatio)
+	}
+	if writeRatio < 0.75 {
+		framework.Failf("Too many failures writing data. Success ratio: %f", writeRatio)
+	}
+}
+
+// Teardown performs one final check of the data's availability.
+func (t *MySqlUpgradeTest) Teardown(f *framework.Framework) {
+	count, err := t.countNames()
+	Expect(err).NotTo(HaveOccurred())
+	Expect(count >= t.successfulWrites).To(BeTrue())
+}
+
+// addName adds a new value to the db.
+func (t *MySqlUpgradeTest) addName(name string) error {
+	val := map[string][]string{"name": {name}}
+	t.nextWrite++
+	r, err := http.PostForm(fmt.Sprintf("http://%s:8080/addName", t.ip), val)
+	if err != nil {
+		return err
+	}
+	defer r.Body.Close()
+	if r.StatusCode != http.StatusOK {
+		b, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf(string(b))
+	}
+	return nil
+}
+
+// countNames checks to make sure the values in testing.users are available, and returns
+// the count of them.
+func (t *MySqlUpgradeTest) countNames() (int, error) {
+	r, err := http.Get(fmt.Sprintf("http://%s:8080/countNames", t.ip))
+	if err != nil {
+		return 0, err
+	}
+	defer r.Body.Close()
+	if r.StatusCode != http.StatusOK {
+		b, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			return 0, err
+		}
+		return 0, fmt.Errorf(string(b))
+	}
+	var count int
+	if err := json.NewDecoder(r.Body).Decode(&count); err != nil {
+		return 0, err
+	}
+	return count, nil
+}