Commit c6ef2ddf authored by Fabio Yeon's avatar Fabio Yeon

Fix "validate-cluster.sh" to use "kubectl get componentstatus".

parent 951a1257
......@@ -14,11 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Bring up a Kubernetes cluster.
#
# If the full release name (gs://<bucket>/<release>) is passed in then we take
# that directly. If not then we assume we are doing development stuff and take
# the defaults in the release config.
# Validates that the cluster is healthy.
set -o errexit
set -o nounset
......@@ -28,12 +24,9 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/cluster/kube-env.sh"
source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
get-password
detect-master > /dev/null
detect-minions > /dev/null
MINIONS_FILE=/tmp/minions-$$
trap 'rm -rf "${MINIONS_FILE}"' EXIT
# Make several attempts to deal with slow cluster birth.
attempt=0
while true; do
......@@ -54,62 +47,39 @@ done
echo "Found ${found} nodes."
cat -n "${MINIONS_FILE}"
# On vSphere, use minion IPs as their names
if [[ "${KUBERNETES_PROVIDER}" == "vsphere" || "${KUBERNETES_PROVIDER}" == "vagrant" || "${KUBERNETES_PROVIDER}" == "libvirt-coreos" || "${KUBERNETES_PROVIDER}" == "juju" ]] ; then
MINION_NAMES=("${KUBE_MINION_IP_ADDRESSES[@]}")
fi
attempt=0
while true; do
kubectl_output=$("${KUBE_ROOT}/cluster/kubectl.sh" get componentstatus)
# On AWS we can't really name the minions, so just trust that if the number is right, the right names are there.
if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
MINION_NAMES=("$(cat ${MINIONS_FILE})")
# /healthz validation isn't working for some reason on AWS. So just hope for the best.
# TODO: figure out why and fix, it must be working in some form, or else clusters wouldn't work.
echo "Kubelet health checking on AWS isn't currently supported, assuming everything is good..."
echo -e "${color_green}Cluster validation succeeded${color_norm}"
exit 0
fi
# The "kubectl validate" output is four columns like this:
#
# COMPONENT HEALTH MSG ERR
# controller-manager Healthy ok nil
#
# Parse the output to capture the value of the second column("HEALTH"), then use grep to
# count the number of times it doesn't match "success".
# Because of the header, the actual unsuccessful count is 1 minus the count.
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
# Grep returns an exit status of 1 when line is not found, so we need the : to always return a 0 exit status
count=$(grep -c "${MINION_NAMES[$i]}" "${MINIONS_FILE}") || :
if [[ "${count}" == "0" ]]; then
echo -e "${color_red}Failed to find ${MINION_NAMES[$i]}, cluster is probably broken.${color_norm}"
cat -n "${MINIONS_FILE}"
exit 1
fi
non_success_count=$(echo "${kubectl_output}" | \
sed -n 's/^\([[:alnum:][:punct:]]\+\)\s\+\([[:alnum:][:punct:]]\+\)\s\+.*/\2/p' | \
grep 'Healthy' --invert-match -c)
name="${MINION_NAMES[$i]}"
if [[ "$KUBERNETES_PROVIDER" != "vsphere" && "$KUBERNETES_PROVIDER" != "vagrant" && "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
# Grab fully qualified name
name=$(grep "${MINION_NAMES[$i]}\." "${MINIONS_FILE}")
fi
# Make sure the kubelet is healthy.
# Make several attempts to deal with slow cluster birth.
attempt=0
while true; do
echo -n "Attempt $((attempt+1)) at checking Kubelet installation on node ${MINION_NAMES[$i]} ..."
if [[ "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
curl_output=$(curl -s --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" \
"https://${KUBE_MASTER_IP}/api/v1beta1/proxy/minions/${name}/healthz")
if ((non_success_count > 1)); then
if ((attempt < 5)); then
echo -e "${color_yellow}Cluster not working yet.${color_norm}"
attempt=$((attempt+1))
sleep 30
else
curl_output=$(curl -s \
"http://${KUBE_MASTER_IP}:8080/api/v1beta1/proxy/minions/${name}/healthz")
fi
if [[ "${curl_output}" != "ok" ]]; then
if (( attempt > 5 )); then
echo
echo -e "${color_red}Kubelet failed to install on node ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
echo -e "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)${color_norm}"
echo -e " ${color_yellow}Validate output:${color_norm}"
echo "${kubectl_output}"
echo -e "${color_red}Validation returned one or more failed components. Cluster is probably broken.${color_norm}"
exit 1
fi
else
echo -e " ${color_green}[working]${color_norm}"
break
fi
echo -e " ${color_yellow}[not working yet]${color_norm}"
attempt=$((attempt+1))
sleep 30
done
done
echo "Validate output:"
echo "${kubectl_output}"
echo -e "${color_green}Cluster validation succeeded${color_norm}"
......@@ -66,4 +66,4 @@ kubectl
* [kubectl update](kubectl_update.md) - Update a resource by filename or stdin.
* [kubectl version](kubectl_version.md) - Print the client and server version information.
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392549632 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488963312 +0000 UTC
......@@ -50,4 +50,4 @@ kubectl api-versions
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.39227534 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488505223 +0000 UTC
......@@ -50,4 +50,4 @@ kubectl cluster-info
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392162759 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48831375 +0000 UTC
......@@ -63,4 +63,4 @@ kubectl config SUBCOMMAND
* [kubectl config use-context](kubectl_config_use-context.md) - Sets the current-context in a kubeconfig file
* [kubectl config view](kubectl_config_view.md) - displays Merged kubeconfig settings or a specified kubeconfig file.
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392043616 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488116168 +0000 UTC
......@@ -65,4 +65,4 @@ $ kubectl config set-cluster e2e --insecure-skip-tls-verify=true
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.39119629 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486460859 +0000 UTC
......@@ -58,4 +58,4 @@ $ kubectl config set-context gce --user=cluster-admin
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391488399 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486736724 +0000 UTC
......@@ -78,4 +78,4 @@ $ kubectl set-credentials cluster-admin --client-certificate=~/.kube/admin.crt -
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391323192 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486604006 +0000 UTC
......@@ -52,4 +52,4 @@ kubectl config set PROPERTY_NAME PROPERTY_VALUE
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391618859 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486861123 +0000 UTC
......@@ -51,4 +51,4 @@ kubectl config unset PROPERTY_NAME
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391735806 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.487685494 +0000 UTC
......@@ -50,4 +50,4 @@ kubectl config use-context CONTEXT_NAME
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391848246 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.487888021 +0000 UTC
......@@ -72,4 +72,4 @@ $ kubectl config view -o template --template='{{range .users}}{{ if eq .name "e2
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391073075 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486319118 +0000 UTC
......@@ -63,4 +63,4 @@ $ cat pod.json | kubectl create -f -
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388588064 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48343431 +0000 UTC
......@@ -81,4 +81,4 @@ $ kubectl delete pods --all
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389412973 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483731878 +0000 UTC
......@@ -53,4 +53,4 @@ kubectl describe RESOURCE ID
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388410556 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483293174 +0000 UTC
......@@ -64,4 +64,4 @@ $ kubectl exec -p 123456-7890 -c ruby-container -i -t -- bash -il
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390127525 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484697863 +0000 UTC
......@@ -82,4 +82,4 @@ $ kubectl expose streamer --port=4100 --protocol=udp --service-name=video-stream
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390792874 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485803902 +0000 UTC
......@@ -8,7 +8,7 @@ Display one or many resources
Display one or many resources.
Possible resources include pods (po), replication controllers (rc), services
(svc), minions (mi), or events (ev).
(svc), minions (mi), events (ev), or component statuses (cs).
By specifying the output as 'template' and providing a Go template as the value
of the --template flag, you can filter the attributes of the fetched resource(s).
......@@ -85,4 +85,4 @@ $ kubectl get rc/web service/frontend pods/web-pod-13je7
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.387483074 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.482589064 +0000 UTC
......@@ -81,4 +81,4 @@ $ kubectl label pods foo bar-
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390937166 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486060232 +0000 UTC
......@@ -62,4 +62,4 @@ $ kubectl log -f 123456-7890 ruby-container
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389728881 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484139739 +0000 UTC
......@@ -53,4 +53,4 @@ kubectl namespace [namespace]
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389609191 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483937463 +0000 UTC
......@@ -68,4 +68,4 @@ $ kubectl port-forward -p mypod 0:5000
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390241417 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484899751 +0000 UTC
......@@ -65,4 +65,4 @@ $ kubectl proxy --api-prefix=k8s-api
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390360738 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485099157 +0000 UTC
......@@ -68,4 +68,4 @@ $ kubectl resize --current-replicas=2 --replicas=3 replicationcontrollers foo
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389989377 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484493463 +0000 UTC
......@@ -68,4 +68,4 @@ $ cat frontend-v2.json | kubectl rolling-update frontend-v1 -f -
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.38985117 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484316119 +0000 UTC
......@@ -78,4 +78,4 @@ $ kubectl run-container nginx --image=nginx --overrides='{ "apiVersion": "v1beta
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390501802 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485362986 +0000 UTC
......@@ -72,4 +72,4 @@ $ kubectl stop -f path/to/resources
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390631789 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48555328 +0000 UTC
......@@ -67,4 +67,4 @@ $ kubectl update pods my-pod --patch='{ "apiVersion": "v1beta1", "desiredState":
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388743178 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483572524 +0000 UTC
......@@ -51,4 +51,4 @@ kubectl version
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392395408 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488692518 +0000 UTC
......@@ -17,7 +17,7 @@ Display one or many resources.
.PP
Possible resources include pods (po), replication controllers (rc), services
(svc), minions (mi), or events (ev).
(svc), minions (mi), events (ev), or component statuses (cs).
.PP
By specifying the output as 'template' and providing a Go template as the value
......
......@@ -1682,8 +1682,8 @@ const (
)
type ComponentCondition struct {
Type ComponentConditionType `json:"type"`
Status ConditionStatus `json:"status"`
Type ComponentConditionType `json:"type" description:"type of component condition, currently only Healthy"`
Status ConditionStatus `json:"status" description:"current status of this component condition"`
Message string `json:"message,omitempty" description:"health check message received from the component"`
Error string `json:"error,omitempty" description:"error code from health check attempt (if any)"`
}
......@@ -1694,12 +1694,12 @@ type ComponentStatus struct {
ObjectMeta `json:"metadata,omitempty"`
Name string `json:"name,omitempty" description:"name of the component"`
Conditions []ComponentCondition `json:"conditions,omitempty"`
Conditions []ComponentCondition `json:"conditions,omitempty" description:"Conditions is a list of ComponentCondition objects"`
}
type ComponentStatusList struct {
TypeMeta `json:",inline"`
ListMeta `json:"metadata,omitempty"`
Items []ComponentStatus `json:"items" description:"items is a list of component status objects"`
Items []ComponentStatus `json:"items" description:"items is a list of ComponentStatus objects"`
}
......@@ -65,7 +65,6 @@ func TestValidate(t *testing.T) {
StatusCode: test.code,
},
}
// fake := &http.Client{Transport: fakeRT}
status, data, err := s.DoServerCheck(fakeRT)
expect := fmt.Sprintf("http://%s:%d/healthz", s.Addr, s.Port)
if fakeRT.url != expect {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment