Merge pull request #610 from erikwilson/sonobuoy-cleanup-3

scripts/sonobuoy

@@ -16,6 +16,8 @@ port-used() {
 }
 export -f port-used
 
+# ---
+
 get-port() {
     while
         PORT=$((10000 + RANDOM % 50000))
@@ -25,39 +27,6 @@ get-port() {
 }
 export -f get-port
 
-K3S_PORT=$(timeout --foreground 5s bash -c get-port)
-OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT}
-mkdir -p ${OUTPUT}
-RESULTS="${OUTPUT}/plugins/e2e/results/e2e.log"
-
-SECRET=random-$((100000 + RANDOM % 999999))
-export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT}
-export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT}
-export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
-export KUBECONFIG=${OUTPUT}/kubeconfig.yaml
-
-# ---
-
-cleanup() {
-    exit_status=$?
-    set +e
-    echo "Cleaning up"
-    docker rm -f ${CONTAINERS} 2>/dev/null
-    rm ${KUBECONFIG}
-    exit ${exit_status}
-}
-trap cleanup EXIT
-
-# ---
-
-docker run -d --name ${K3S_SERVER} --privileged \
-    -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \
-    -e K3S_CLUSTER_SECRET=${SECRET} \
-    ${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT}
-
-K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER})
-echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}"
-
 # ---
 
 fetch-kubeconfig() {
@@ -65,6 +34,8 @@ fetch-kubeconfig() {
 }
 export -f fetch-kubeconfig
 
+# ---
+
 wait-for-kubeconfig() {
     while ! fetch-kubeconfig; do
         echo "Waiting for kubeconfig to become available..."
@@ -73,17 +44,6 @@ wait-for-kubeconfig() {
 }
 export -f wait-for-kubeconfig
 
-timeout --foreground 1m bash -c wait-for-kubeconfig
-
-# ---
-
-docker run -d --name ${K3S_AGENT} --privileged \
-    -e K3S_CLUSTER_SECRET=${SECRET} \
-    -e K3S_URL=https://${K3S_IP}:${K3S_PORT} \
-    ${K3S_IMAGE} agent
-
-echo "Started ${K3S_AGENT}"
-
 # ---
 
 count-ready-nodes() {
@@ -94,6 +54,8 @@ count-ready-nodes() {
 }
 export -f count-ready-nodes
 
+# ---
+
 wait-for-nodes() {
     while [[ "$(count-ready-nodes)" != "2" ]]; do
         echo "Waiting for nodes to be ready..."
@@ -102,8 +64,6 @@ wait-for-nodes() {
 }
 export -f wait-for-nodes
 
-timeout --foreground 1m bash -c wait-for-nodes
-
 # ---
 
 pod-ready() {
@@ -112,6 +72,8 @@ pod-ready() {
 }
 export -f pod-ready
 
+# ---
+
 wait-for-services() {
     for service in coredns; do
         while [[ "$(pod-ready ${service})" != "true" ]]; do
@@ -123,8 +85,6 @@ wait-for-services() {
 }
 export -f wait-for-services
 
-timeout --foreground 1m bash -c wait-for-services
-
 # ---
 
 dump-container-logs() {
@@ -156,9 +116,12 @@ sonobuoy-test() {
         sonobuoy-destruct
         return 1
     fi
-    sonobuoy retrieve ${OUTPUT}
 
-    tar x -z -f ${OUTPUT}/*_sonobuoy_*.tar.gz -C ${OUTPUT}
+    rm -rf ${E2E}
+    mkdir -p ${E2E}
+    sonobuoy retrieve ${E2E}
+
+    tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
     if [ ! -s ${RESULTS} ]; then
         sonobuoy-destruct
         return 1
@@ -175,7 +138,7 @@ sonobuoy-retry-test() {
     LIMIT=300
     for i in $(seq 1 3); do
         sonobuoy-test "${@}" && return
-        echo "Failed sonobuoy try #${i}"
+        echo "*** Failed Sonobuoy try #${i} on port ${K3S_PORT} at ${SECONDS} seconds ***"
         if [ "$SECONDS" -gt "$LIMIT" ]; then
             echo "Stopping sonobuoy re-try after ${SECONDS} seconds (limit ${LIMIT}s)"
             break
@@ -188,9 +151,62 @@ export -f sonobuoy-retry-test
 
 # ---
 
+cleanup() {
+    exit_status=$?
+    set +e
+    echo "Cleaning up"
+    docker rm -f ${CONTAINERS} 2>/dev/null
+    rm ${KUBECONFIG}
+    exit ${exit_status}
+}
+trap cleanup EXIT
+
+# ---
+
+K3S_PORT=$(timeout --foreground 5s bash -c get-port)
+OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT}
+E2E="${OUTPUT}/e2e"
+E2E_LOG="plugins/e2e/results/e2e.log"
+RESULTS="${E2E}/${E2E_LOG}"
+mkdir -p ${OUTPUT}
+
+SECRET=random-$((100000 + RANDOM % 999999))
+export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT}
+export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT}
+export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
+export KUBECONFIG=${OUTPUT}/kubeconfig.yaml
+
+# ---
+
+docker run -d --name ${K3S_SERVER} --privileged \
+    -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \
+    -e K3S_CLUSTER_SECRET=${SECRET} \
+    ${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT}
+
+K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER})
+echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}"
+
+# ---
+
+timeout --foreground 1m bash -c wait-for-kubeconfig
+
+docker run -d --name ${K3S_AGENT} --privileged \
+    -e K3S_CLUSTER_SECRET=${SECRET} \
+    -e K3S_URL=https://${K3S_IP}:${K3S_PORT} \
+    ${K3S_IMAGE} agent
+
+echo "Started ${K3S_AGENT}"
+
+# ---
+
+timeout --foreground 1m bash -c wait-for-nodes
+timeout --foreground 1m bash -c wait-for-services
+
 echo "Starting sonobuoy tests"
 sonobuoy-retry-test "${@}"
 
+# ---
+
 exit_code=0
 status=$(tail -5 ${RESULTS} | grep '^SUCCESS!.*| 0 Failed |' >/dev/null && echo passed || echo failed)
 [ "${status}" = "failed" ] && exit_code=1

scripts/sonobuoy-e2e-tests

@@ -33,12 +33,14 @@ run-sonobuoy() {
 }
 
 
-log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log \
-    run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]'
-
 log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log \
     run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y
 
+sleep 60
+
+log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log \
+    run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]'
+
 
 for pid in "${pids[@]}"; do
     wait $pid || exit $?