Test cleanup

.drone.yml

@@ -59,14 +59,14 @@ steps:
     event:
     - tag
 
-- name: sonobuoy-e2e-tests
+- name: test
   image: rancher/dapper:v0.4.2
   secrets: [ gcloud_auth ]
   environment:
     GCLOUD_AUTH:
       from_secret: gcloud_auth
   commands:
-  - dapper -f Dockerfile.sonobuoy.dapper
+  - dapper -f Dockerfile.test.dapper
   volumes:
   - name: docker
     path: /var/run/docker.sock
@@ -154,14 +154,14 @@ steps:
     event:
     - tag
 
-- name: sonobuoy-e2e-tests
+- name: test
   image: rancher/dapper:v0.4.2
   secrets: [ gcloud_auth ]
   environment:
     GCLOUD_AUTH:
       from_secret: gcloud_auth
   commands:
-  - dapper -f Dockerfile.sonobuoy.dapper
+  - dapper -f Dockerfile.test.dapper
   volumes:
   - name: docker
     path: /var/run/docker.sock
@@ -232,14 +232,14 @@ steps:
     event:
     - tag
 
-- name: sonobuoy-e2e-tests
+- name: test
   image: rancher/dapper:v0.4.2
   secrets: [ gcloud_auth ]
   environment:
     GCLOUD_AUTH:
       from_secret: gcloud_auth
   commands:
-  - dapper -f Dockerfile.sonobuoy.dapper
+  - dapper -f Dockerfile.test.dapper
   volumes:
   - name: docker
     path: /var/run/docker.sock

Dockerfile.sonobuoy.dapper → Dockerfile.test.dapper

 FROM golang:1.13.5-alpine3.10
 
-RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2
-RUN SONOBUOY_VER=v0.16.2 && \
-    SONOBUOY_PKG=github.com/vmware-tanzu/sonobuoy && \
-    go get -d ${SONOBUOY_PKG} && \
-    cd /go/src/${SONOBUOY_PKG} && \
-    git checkout -b current ${SONOBUOY_VER} && \
-    go build -o /usr/local/bin/sonobuoy
-RUN rm -rf /go/src /go/pkg
+RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2 openssl
 
 ARG DAPPER_HOST_ARCH
 ENV ARCH $DAPPER_HOST_ARCH
 
+RUN if [ "${ARCH}" == "amd64" ] || [ "${ARCH}" == "arm64" ]; then \
+        VERSION=0.17.0 OS=linux && \
+        curl -sL "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_${ARCH}.tar.gz" | \
+        tar -xzf - -C /usr/local/bin; \
+    fi
+
 RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \
             curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt \
         )/bin/linux/${ARCH}/kubectl -o /usr/local/bin/kubectl && \
     chmod a+x /usr/local/bin/kubectl
 
+ENV TEST_CLEANUP true
+
 ENV DAPPER_RUN_ARGS --privileged --network host
 ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME GCLOUD_AUTH
 ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
@@ -26,4 +27,4 @@ ENV HOME ${DAPPER_SOURCE}
 WORKDIR ${DAPPER_SOURCE}
 
 ENTRYPOINT ["./scripts/entry.sh"]
-CMD ["sonobuoy-e2e-tests"]
+CMD ["test"]

scripts/log-upload

@@ -40,7 +40,7 @@ END
 HOME=${TMPDIR}
 PATH=$PATH:${HOME}/gsutil
 
-LOG_TGZ=k3s-log-$(uname -s | tr '[:upper:]' '[:lower:]' )-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(date +%s)-$(basename $1).tgz
+LOG_TGZ=k3s-log-$(date +%s)-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(basename $1).tgz
 
 tar -cz -f ${TMPDIR}/${LOG_TGZ} -C $(dirname $1) $(basename $1)
 gsutil cp ${TMPDIR}/${LOG_TGZ} gs://k3s-ci-logs || exit 1

scripts/sonobuoy

-#!/bin/bash
-set -xe
-
-cd $(dirname $0)/..
-
-. ./scripts/test-helpers
-
-if [ -z "${K3S_IMAGE}" ]; then
-    echo 'K3S_IMAGE environment variable should be defined'
-    exit 1
-fi
-
-# ---
-
-cleanup() {
-    exit_status=$?
-    set +e
-    echo 'Cleaning up'
-    trap - EXIT
-    [ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null
-    if [ "${exit_status}" -ne "0" ]; then
-        dump-logs
-    fi
-    docker rm -f ${CONTAINERS} 2>/dev/null
-    rm ${KUBECONFIG}
-    exit ${exit_status}
-}
-trap cleanup EXIT
-
-# ---
-
-K3S_PORT=$(timeout --foreground 5s bash -c get-port)
-OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT}
-LOGS=$(pwd)/logs/$$
-E2E="${OUTPUT}/e2e"
-E2E_LOG='plugins/e2e/results/global/e2e.log'
-RESULTS="${E2E}/${E2E_LOG}"
-
-SECRET=random-$((100000 + RANDOM % 999999))
-export K3S_AGENT=k3s-agent-${K3S_PORT}
-export K3S_SERVER=k3s-server-${K3S_PORT}
-export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
-export KUBECONFIG=${OUTPUT}/kubeconfig.yaml
-
-mkdir -p ${OUTPUT}
-
-# ---
-
-docker run -d --name ${K3S_SERVER} --privileged \
-    -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \
-    -e K3S_CLUSTER_SECRET=${SECRET} \
-    -e K3S_DEBUG=true \
-    ${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT}
-
-K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER})
-echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}"
-
-docker exec ${K3S_SERVER} check-config || true
-timeout --foreground 1m bash -c wait-for-kubeconfig
-verify-valid-versions ${K3S_SERVER}
-
-# ---
-
-docker run -d --name ${K3S_AGENT} --privileged \
-    -e K3S_CLUSTER_SECRET=${SECRET} \
-    -e K3S_URL=https://${K3S_IP}:${K3S_PORT} \
-    ${K3S_IMAGE} agent
-
-echo "Started ${K3S_AGENT}"
-
-# ---
-
-timeout --foreground 1m bash -c 'wait-for-nodes 2'
-timeout --foreground 3m bash -c 'wait-for-services coredns local-path-provisioner metrics-server'
-
-if [ "$ARCH" = 'arm' ]; then
-    echo "Aborting sonobuoy tests, images not available for $ARCH"
-    exit 0
-fi
-
-echo 'Starting sonobuoy tests'
-sonobuoy-test "${@}"

scripts/sonobuoy-e2e-tests

-#!/bin/bash
-set -e -x
-
-cd $(dirname $0)/..
-
-if [ -z "$K3S_IMAGE" ]; then
-    . ./scripts/version.sh
-    TAG=${TAG:-${VERSION_TAG}${SUFFIX}}
-    REPO=${REPO:-rancher}
-    IMAGE_NAME=${IMAGE_NAME:-k3s}
-    export K3S_IMAGE=${REPO}/${IMAGE_NAME}:${TAG}
-fi
-
-OUTPUT=$(pwd)/dist/artifacts
-mkdir -p ${OUTPUT}
-
-pids=()
-output=()
-
-show-logs() {
-    for pid in "${pids[@]}"; do
-        logdir=$(pwd)/logs/${pid}
-        if [ ! -d $logdir ]; then
-            continue
-        fi
-        echo
-        echo "#- Begin: logs for sonobuoy run pid ${pid}"
-        for log in $(pwd)/logs/${pid}/*.log; do
-            if [ -f ${log} ]; then
-                echo
-                echo "#- Tail: ${log}"
-                tail -10 ${log}
-                echo "#- Done: ${log}"
-                echo
-            fi
-        done
-        echo "#- Finish: logs for sonobuoy run pid ${pid}"
-        echo
-    done
-}
-
-cleanup() {
-    exit_status=$?
-    set +e
-    trap - EXIT INT
-    kill ${pids[@]} 2>/dev/null
-    wait
-    set +x
-    echo -n "Tests "
-    if [ "${exit_status}" -eq "0" ]; then
-        echo "passed"
-    else
-        echo "failed"
-        show-logs
-    fi
-    exit ${exit_status}
-}
-trap cleanup EXIT INT
-
-
-# ---
-
-run-sonobuoy() {
-    output+=(${log_output})
-    E2E_LOG_OUTPUT=${log_output} \
-        ./scripts/sonobuoy ${@} \
-            > >(stdbuf -oL sed "s/^/[${label}] /") 2>&1 &
-    pids+=($!)
-}
-
-log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log label=PARALLEL \
-    run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y
-
-sleep 60
-
-log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log label=SERIAL \
-    run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]'
-
-exit_code=0
-for pid in "${pids[@]}"; do
-    wait $pid || exit_code=$?
-done
-exit ${exit_code}

scripts/test

+#!/bin/bash
+set -e -x
+cd $(dirname $0)/..
+
+. ./scripts/test-helpers
+
+artifacts=$(pwd)/dist/artifacts
+mkdir -p $artifacts
+
+# ---
+
+. ./scripts/test-run-basics
+
+# # ---
+
+E2E_OUTPUT=$artifacts test-run-sonobuoy
+
+# # ---
+
+# if [ "$ARCH" != 'amd64' ]; then
+#     printf "\033[33mSkipping remaining tests, images not available for $ARCH.\033[m\n"
+#     exit 0
+# fi
+
+# # ---
+
+# test-run-sonobuoy mysql
+# test-run-sonobuoy postgres
+# test-run-sonobuoy dqlite
\ No newline at end of file

scripts/test-certs-openssl.cnf

+# db OpenSSL configuration file.
+SAN = "IP:127.0.0.1, IP:172.17.0.1, DNS:host.docker.internal"
+dir = .
+
+[ ca ]
+default_ca = db_ca
+
+[ db_ca ]
+certs            = $dir/certs
+certificate      = $dir/certs/ca.crt
+crl              = $dir/crl.pem
+crl_dir          = $dir/crl
+crlnumber        = $dir/crlnumber
+database         = $dir/index.txt
+email_in_dn      = no
+new_certs_dir    = $dir/newcerts
+private_key      = $dir/private/ca.key
+serial           = $dir/serial
+RANDFILE         = $dir/private/.rand
+name_opt         = ca_default
+cert_opt         = ca_default
+default_days     = 3650
+default_crl_days = 30
+default_md       = sha512
+preserve         = no
+policy           = policy_db
+
+[ policy_db ]
+organizationName = optional
+commonName       = supplied
+
+[ req ]
+default_bits       = 1024
+default_keyfile    = privkey.pem
+distinguished_name = req_distinguished_name
+attributes         = req_attributes
+x509_extensions    = v3_ca
+string_mask        = utf8only
+req_extensions     = db_client
+
+[ req_distinguished_name ]
+countryName                = Country Name (2 letter code)
+countryName_default        = US
+countryName_min            = 2
+countryName_max            = 2
+commonName                 = Common Name (FQDN)
+0.organizationName         = Organization Name (eg, company)
+0.organizationName_default = db-ca
+
+[ req_attributes ]
+
+[ v3_ca ]
+basicConstraints       = CA:true
+keyUsage               = keyCertSign,cRLSign
+subjectKeyIdentifier   = hash
+
+[ db_client ]
+basicConstraints       = CA:FALSE
+extendedKeyUsage       = clientAuth
+keyUsage               = digitalSignature, keyEncipherment
+
+[ db_peer ]
+basicConstraints       = CA:FALSE
+extendedKeyUsage       = clientAuth, serverAuth
+keyUsage               = digitalSignature, keyEncipherment
+subjectAltName         = ${ENV::SAN}
+
+[ db_server ]
+basicConstraints       = CA:FALSE
+extendedKeyUsage       = clientAuth, serverAuth
+keyUsage               = digitalSignature, keyEncipherment
+subjectAltName         = ${ENV::SAN}
\ No newline at end of file

scripts/test-run-basics

+#!/bin/bash
+
+export NUM_SERVERS=1
+export NUM_AGENTS=1
+export SERVER_ARGS='--no-deploy=traefik,coredns,local-storage,metrics-server'
+
+start-test() {
+    docker exec $(cat $TEST_DIR/servers/1/metadata/name) check-config || true
+    verify-valid-versions $(cat $TEST_DIR/servers/1/metadata/name)
+}
+export -f start-test
+
+# --- create a basic cluster and check for valid versions
+LABEL=BASICS run-test

scripts/test-runner

+#!/bin/bash
+set -x -e
+cd $(dirname $0)/..
+
+# ---
+
+for include in $TEST_INCLUDES; do
+  . $include
+done
+
+test-setup
+provision-cluster
+start-test $@

scripts/test-setup-sonobuoy

+#!/bin/bash
+
+export NUM_SERVERS=1
+export NUM_AGENTS=1
+export SERVER_ARGS='--no-deploy=traefik'
+export WAIT_SERVICES='coredns local-path-provisioner metrics-server'
+
+export sonobuoyParallelArgs=(--e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y)
+export sonobuoySerialArgs=(--e2e-focus='\[Serial\].*\[Conformance\]')
+
+start-test() {
+  sonobuoy-test $@
+}
+export -f start-test

scripts/test-setup-sonobuoy-dqlite

+#!/bin/bash
+
+. ./scripts/test-setup-sonobuoy
+
+export NUM_SERVERS=2
+export NUM_AGENTS=0
+
+export SERVER_1_ARGS=--cluster-init
+
+# ---
+
+server-post-hook() {
+  if [ $1 -eq 1 ]; then
+    local url=$(cat $TEST_DIR/servers/1/metadata/url)
+    export SERVER_ARGS="--server $url"
+  fi
+}
+export -f server-post-hook

scripts/test-setup-sonobuoy-mysql

+#!/bin/bash
+
+. ./scripts/test-setup-sonobuoy
+
+# ---
+
+cluster-pre-hook() {
+  # gen-certs
+
+  mkdir -p $TEST_DIR/db/$LABEL_SUFFIX/metadata
+  local testID=$(basename $TEST_DIR)
+  local name=$(echo $LABEL_SUFFIX-$testID | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/name)
+  local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/port)
+  local secret=$(echo "${RANDOM}${RANDOM}${RANDOM}" | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/secret)
+
+  docker run --name $name \
+    --privileged \
+    -p 0.0.0.0:$port:3306 \
+    -v $TEST_DIR/db-ca/:/db-ca \
+    -e MYSQL_ROOT_PASSWORD=$secret \
+    -e MYSQL_ROOT_HOST=% \
+    -d mysql:latest \
+    >/dev/null
+      #  --require-secure-transport=ON \
+      #  --ssl-ca /db-ca/certs/ca.crt \
+      #  --ssl-cert /db-ca/certs/db.crt \
+      #  --ssl-key /db-ca/private/db.key \
+
+  local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/ip)
+  # local host=host.docker.internal
+  local host=172.17.0.1
+
+  DB_CONNECTION_TEST="
+    docker run
+      -v $TEST_DIR/db-ca/:/db-ca
+      --rm mysql
+      mysql
+        -h$host
+        -P$port
+        -uroot
+        -p$secret
+        -e status" \
+    timeout --foreground 1m bash -c "wait-for-db-connection"
+        # --ssl-ca /db-ca/certs/ca.crt
+        # --ssl-cert /db-ca/certs/client.crt
+        # --ssl-key /db-ca/private/client.key
+
+  echo "Started $LABEL_SUFFIX db @ $host"
+  export SERVER_ARGS="${SERVER_ARGS}
+    --datastore-endpoint=mysql://root:$secret@tcp($host:$port)/testdb
+  "
+    # --datastore-cafile /db-ca/certs/ca.crt
+    # --datastore-certfile /db-ca/certs/client.crt
+    # --datastore-keyfile /db-ca/private/client.key
+}
+export -f cluster-pre-hook

scripts/test-setup-sonobuoy-postgres

+#!/bin/bash
+
+. ./scripts/test-setup-sonobuoy
+
+# ---
+
+cluster-pre-hook() {
+  # gen-certs
+
+  mkdir -p $TEST_DIR/db/$LABEL_SUFFIX/metadata
+  local testID=$(basename $TEST_DIR)
+  local name=$(echo $LABEL_SUFFIX-$testID | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/name)
+  local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/port)
+  local secret=$(echo "${RANDOM}${RANDOM}${RANDOM}" | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/secret)
+
+  docker run --name $name \
+    --privileged \
+    -p 0.0.0.0:$port:5432 \
+    -v $TEST_DIR/db-ca/:/db-ca \
+    -e POSTGRES_USER=root \
+    -e POSTGRES_PASSWORD=$secret \
+    -d postgres:latest \
+    >/dev/null
+      # -c ssl=on \
+      # -c ssl_ca_file=/db-ca/certs/ca.crt \
+      # -c ssl_cert_file=/db-ca/certs/db.crt \
+      # -c ssl_key_file=/db-ca/private/db.key \
+
+  local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/ip)
+  # local host=host.docker.internal
+  local host=172.17.0.1
+
+  DB_CONNECTION_TEST="
+    docker run
+      -v $TEST_DIR/db-ca/:/db-ca
+      -e PGPASSWORD=$secret
+      --rm postgres
+      psql
+        -h $host
+        -p $port
+        -U root
+        -c \conninfo" \
+    timeout --foreground 1m bash -c "wait-for-db-connection"
+        # --set sslrootcert=/db-ca/certs/ca.crt
+        # --set sslcert=/db-ca/certs/client.crt
+        # --set sslkey=/db-ca/private/client.key
+
+  echo "Started $LABEL_SUFFIX db @ $host"
+  export SERVER_ARGS="${SERVER_ARGS}
+    --datastore-endpoint=postgres://root:$secret@$host:$port/testdb?sslmode=disable
+  "
+    # --datastore-cafile /db-ca/certs/ca.crt
+    # --datastore-certfile /db-ca/certs/client.crt
+    # --datastore-keyfile /db-ca/private/client.key
+}
+export -f cluster-pre-hook