Skip to content

E
etersoft-admin-essentials
Commit c14a5696 authored by Vitaly Lipatov's avatar Vitaly Lipatov
Browse files
Options

route-health: per-gateway failover, iperf3 checks, GRE detection fix

- Flush routes per dead gateway instead of waiting for entire group to die - Add iperf3 bandwidth (success=0 → dead) as third health signal - Ping threshold: >=50% loss is now dead (was 100%) - State tracking per gateway (.state/GROUP/gw-TAG/) - GRE detection: use link UP instead of RX bytes delta (fixes false disconnected status on idle tunnels) - Timer interval: 20s (was 1min) Co-Authored-By: 's avatarClaude Opus 4.6 <noreply@anthropic.com>
parent 6fb0f9be
Show whitespace changes
Inline Side-by-side
Showing with 111 additions and 164 deletions
router/check-vpn-status.sh
View file @ c14a5696
...@@ -105,23 +105,12 @@ fi ...@@ -105,23 +105,12 @@ fi
# --- GRE --- # --- GRE ---
if ip link show gre1 >/dev/null 2>&1; then if ip link show gre1 >/dev/null 2>&1; then
# GRE is stateless — link UP is the only reliable indicator
if ip -o link show gre1 2>/dev/null | grep -q UP; then if ip -o link show gre1 2>/dev/null | grep -q UP; then
# GRE is stateless; check RX bytes are changing via state file
rx=$(cat /sys/class/net/gre1/statistics/rx_bytes 2>/dev/null || echo 0)
state_file="$STATE_DIR/gre1_rx_prev"
prev_rx=$(cat "$state_file" 2>/dev/null || echo 0)
echo "$rx" > "$state_file"
if [ "$rx" != "$prev_rx" ] && [ "$prev_rx" != "0" ]; then
echo 'vpn_status type="gre",connected=1i'
elif [ "$prev_rx" = "0" ]; then
# First run, assume OK if link is UP
echo 'vpn_status type="gre",connected=1i' echo 'vpn_status type="gre",connected=1i'
else else
echo 'vpn_status type="gre",connected=0i' echo 'vpn_status type="gre",connected=0i'
fi fi
else
echo 'vpn_status type="gre",connected=0i'
fi
exit 0 exit 0
fi fi
......
router/route-health.sh
View file @ c14a5696
...@@ -21,10 +21,11 @@ DOWN_THRESHOLD=2 ...@@ -21,10 +21,11 @@ DOWN_THRESHOLD=2
SHOW= SHOW=
[ "$1" = "--show" ] && SHOW=1 [ "$1" = "--show" ] && SHOW=1
# --- Query InfluxDB for packet loss and VPN status --- # --- Query InfluxDB for packet loss, VPN status, and iperf3 bandwidth ---
HEALTH_DATA=$(mktemp) HEALTH_DATA=$(mktemp)
VPN_DATA=$(mktemp) VPN_DATA=$(mktemp)
trap 'rm -f "$HEALTH_DATA" "$VPN_DATA"' EXIT IPERF_DATA=$(mktemp)
trap 'rm -f "$HEALTH_DATA" "$VPN_DATA" "$IPERF_DATA"' EXIT
response=$(curl -sG "$INFLUXDB_URL" \ response=$(curl -sG "$INFLUXDB_URL" \
--data-urlencode "db=$INFLUXDB_DB" \ --data-urlencode "db=$INFLUXDB_DB" \
...@@ -36,6 +37,11 @@ vpn_response=$(curl -sG "$INFLUXDB_URL" \ ...@@ -36,6 +37,11 @@ vpn_response=$(curl -sG "$INFLUXDB_URL" \
--data-urlencode "q=SELECT last(connected) FROM vpn_status WHERE time > now() - 1m GROUP BY gateway" \ --data-urlencode "q=SELECT last(connected) FROM vpn_status WHERE time > now() - 1m GROUP BY gateway" \
--max-time 5 2>/dev/null) --max-time 5 2>/dev/null)
iperf_response=$(curl -sG "$INFLUXDB_URL" \
--data-urlencode "db=$INFLUXDB_DB" \
--data-urlencode "q=SELECT last(success), last(recv_bps) FROM iperf3 WHERE time > now() - 10m GROUP BY gateway" \
--max-time 5 2>/dev/null)
if [ -z "$response" ] ; then if [ -z "$response" ] ; then
log "ERROR: Failed to query InfluxDB" log "ERROR: Failed to query InfluxDB"
exit 1 exit 1
...@@ -45,6 +51,8 @@ fi ...@@ -45,6 +51,8 @@ fi
epm -y assure jq || fatal "jq is required" epm -y assure jq || fatal "jq is required"
echo "$response" | jq -r '.results[0].series[]? | "\(.tags.gateway) \(.values[0][1])"' > "$HEALTH_DATA" echo "$response" | jq -r '.results[0].series[]? | "\(.tags.gateway) \(.values[0][1])"' > "$HEALTH_DATA"
echo "$vpn_response" | jq -r '.results[0].series[]? | "\(.tags.gateway) \(.values[0][1])"' > "$VPN_DATA" 2>/dev/null echo "$vpn_response" | jq -r '.results[0].series[]? | "\(.tags.gateway) \(.values[0][1])"' > "$VPN_DATA" 2>/dev/null
# iperf3: "gateway success recv_bps"
echo "$iperf_response" | jq -r '.results[0].series[]? | "\(.tags.gateway) \(.values[0][1]) \(.values[0][2])"' > "$IPERF_DATA" 2>/dev/null
if [ ! -s "$HEALTH_DATA" ] ; then if [ ! -s "$HEALTH_DATA" ] ; then
log "ERROR: No health data from InfluxDB" log "ERROR: No health data from InfluxDB"
...@@ -58,13 +66,17 @@ if [ -n "$SHOW" ] ; then ...@@ -58,13 +66,17 @@ if [ -n "$SHOW" ] ; then
log "VPN status (last 1m):" log "VPN status (last 1m):"
sed 's/^/ /' "$VPN_DATA" sed 's/^/ /' "$VPN_DATA"
fi fi
if [ -s "$IPERF_DATA" ] ; then
log "iperf3 bandwidth (last 10m):"
sed 's/^/ /' "$IPERF_DATA"
fi
echo echo
fi fi
# Get health status for a monitor tag # Get health status for a monitor tag
# Usage: get_health TAG # Usage: get_health TAG
# Returns: healthy, degraded, dead # Returns: healthy, degraded, dead
# Checks both packet loss (ping) and VPN tunnel status # Checks: VPN tunnel status, iperf3 bandwidth, packet loss (ping)
get_health() get_health()
{ {
local tag="$1" local tag="$1"
...@@ -76,6 +88,13 @@ get_health() ...@@ -76,6 +88,13 @@ get_health()
return return
fi fi
# Check iperf3 — if test failed (success=0), dead
local iperf_success=$(grep "^${tag} " "$IPERF_DATA" 2>/dev/null | awk '{print $2}')
if [ "$iperf_success" = "0" ] ; then
echo "dead"
return
fi
local loss=$(grep "^${tag} " "$HEALTH_DATA" | awk '{print $2}') local loss=$(grep "^${tag} " "$HEALTH_DATA" | awk '{print $2}')
if [ -z "$loss" ] ; then if [ -z "$loss" ] ; then
...@@ -84,9 +103,8 @@ get_health() ...@@ -84,9 +103,8 @@ get_health()
fi fi
echo "$loss" | awk '{ echo "$loss" | awk '{
if ($1 >= 100) print "dead" if ($1 >= 50) print "dead"
else if ($1 < 50) print "healthy" else print "healthy"
else print "degraded"
}' }'
} }
...@@ -127,7 +145,9 @@ eval_group_health() ...@@ -127,7 +145,9 @@ eval_group_health()
local ld="${loss:-no_data}%" ; [ -z "$loss" ] && ld="no_data" local ld="${loss:-no_data}%" ; [ -z "$loss" ] && ld="no_data"
local vpn=$(grep "^${tag} " "$VPN_DATA" 2>/dev/null | awk '{print $2}') local vpn=$(grep "^${tag} " "$VPN_DATA" 2>/dev/null | awk '{print $2}')
local vpn_mark="" ; [ "$vpn" = "0" ] && vpn_mark=",vpn=DOWN" local vpn_mark="" ; [ "$vpn" = "0" ] && vpn_mark=",vpn=DOWN"
group_detail="${group_detail:+$group_detail, }$tag=$ld($st$vpn_mark)" local iperf_s=$(grep "^${tag} " "$IPERF_DATA" 2>/dev/null | awk '{print $2}')
local iperf_mark="" ; [ "$iperf_s" = "0" ] && iperf_mark=",iperf=FAIL"
group_detail="${group_detail:+$group_detail, }$tag=$ld($st$vpn_mark$iperf_mark)"
[ "$st" = "healthy" ] && any_healthy=1 [ "$st" = "healthy" ] && any_healthy=1
[ "$st" != "dead" ] && all_dead="" [ "$st" != "dead" ] && all_dead=""
done < "$gwdir/gateway" done < "$gwdir/gateway"
...@@ -190,73 +210,114 @@ flush_gw_routes() ...@@ -190,73 +210,114 @@ flush_gw_routes()
done done
} }
# --- Evaluate health for all monitored groups --- # --- Per-gateway health evaluation and failover ---
healthy_count=0 need_reload=
dead_count=0
dead_groups=""
healthy_groups=""
for routes_dir in "$ROUTES_DIR" "$ROUTES6_DIR" ; do for routes_dir in "$ROUTES_DIR" "$ROUTES6_DIR" ; do
[ -d "$routes_dir" ] || continue [ -d "$routes_dir" ] || continue
ipcmd=$(ipcmd_for "$routes_dir")
for gwdir in "$routes_dir"/*/ ; do for gwdir in "$routes_dir"/*/ ; do
[ -d "$gwdir" ] || continue [ -d "$gwdir" ] || continue
[ -f "$gwdir/gateway" ] || continue
name=$(basename "$gwdir") name=$(basename "$gwdir")
state_path="$routes_dir/$name"
group_has_healthy=""
eval_group_health "$gwdir" "$routes_dir" || continue # First pass: determine which gateways are healthy
while IFS= read -r line ; do
[ -z "$line" ] && continue
echo "$line" | grep -q '^#' && continue
parse_gw_line "$line" "$ipcmd"
[ -n "$gw_ip" ] || continue
local_tag=$(find_gw_monitor "$routes_dir" "$gw_ip" "$ipcmd")
[ -n "$local_tag" ] || continue
st=$(get_health "$local_tag")
[ "$st" = "healthy" ] && group_has_healthy=1
done < "$gwdir/gateway"
state_path="$routes_dir/$name" # Second pass: process each gateway
while IFS= read -r line ; do
[ -z "$line" ] && continue
echo "$line" | grep -q '^#' && continue
parse_gw_line "$line" "$ipcmd"
[ -n "$gw_ip" ] || continue
tag=$(find_gw_monitor "$routes_dir" "$gw_ip" "$ipcmd")
[ -n "$tag" ] || continue
[ -n "$SHOW" ] && log "[$name] $group_detail -> $group_status" st=$(get_health "$tag")
loss=$(grep "^${tag} " "$HEALTH_DATA" | awk '{print $2}')
case "$group_status" in ld="${loss:-no_data}%" ; [ -z "$loss" ] && ld="no_data"
healthy) vpn=$(grep "^${tag} " "$VPN_DATA" 2>/dev/null | awk '{print $2}')
healthy_count=$((healthy_count + 1)) vpn_mark="" ; [ "$vpn" = "0" ] && vpn_mark=",vpn=DOWN"
healthy_groups="${healthy_groups:+$healthy_groups }$state_path" iperf_s=$(grep "^${tag} " "$IPERF_DATA" 2>/dev/null | awk '{print $2}')
;; iperf_mark="" ; [ "$iperf_s" = "0" ] && iperf_mark=",iperf=FAIL"
dead)
dead_count=$((dead_count + 1))
dead_groups="${dead_groups:+$dead_groups }$state_path"
;;
esac
done
done
[ -n "$SHOW" ] && log "Summary: healthy=$healthy_count dead=$dead_count" [ -n "$SHOW" ] && log "[$name] $tag=$ld($st$vpn_mark$iperf_mark) gw=$gw_ip"
# Nothing monitored — exit gw_state_dir="$STATE_DIR/$state_path/gw-$tag"
[ "$healthy_count" -eq 0 ] && [ "$dead_count" -eq 0 ] && exit 0 mkdir -p "$gw_state_dir"
# --- Manage default route for groups with set-default --- if [ "$st" = "dead" ] ; then
for routes_dir in "$ROUTES_DIR" "$ROUTES6_DIR" ; do # --- Dead gateway: flapping protection, then flush ---
[ -d "$routes_dir" ] || continue down_count=0
ipcmd=$(ipcmd_for "$routes_dir") [ -f "$gw_state_dir/down_count" ] && read -r down_count < "$gw_state_dir/down_count"
down_count=$((down_count + 1))
for gwdir in "$routes_dir"/*/ ; do if [ "$down_count" -lt "$DOWN_THRESHOLD" ] ; then
[ -d "$gwdir" ] || continue log "[$name] $tag down ($down_count/$DOWN_THRESHOLD), waiting"
[ -z "$SHOW" ] && echo "$down_count" > "$gw_state_dir/down_count"
continue
fi
# Already flushed?
if [ -f "$gw_state_dir/health" ] && [ "$(cat "$gw_state_dir/health")" = "down" ] ; then
[ -n "$SHOW" ] && log "[$name] $tag already down"
continue
fi
log "[$name] $tag dead (${down_count}x), flushing routes via $gw_ip"
if [ -z "$SHOW" ] ; then
flush_gw_routes "$STATE_DIR/$state_path" "$gw_ip" "$ipcmd"
echo "down" > "$gw_state_dir/health"
echo "$down_count" > "$gw_state_dir/down_count"
fi
else
# --- Healthy gateway: reset counters, recover if was down ---
[ -z "$SHOW" ] && rm -f "$gw_state_dir/down_count"
if [ -f "$gw_state_dir/health" ] && [ "$(cat "$gw_state_dir/health")" = "down" ] ; then
log "[$name] $tag recovered"
if [ -z "$SHOW" ] ; then
echo "up" > "$gw_state_dir/health"
# Invalidate per-list state to trigger route reload
for list_state in "$STATE_DIR/$state_path"/*/ ; do
[ -d "$list_state" ] || continue
rm -f "$list_state/hash" "$list_state/resolved"
done
need_reload=1
fi
fi
fi
done < "$gwdir/gateway"
# --- Manage default route for groups with set-default ---
has_option "$gwdir" "set-default" || continue has_option "$gwdir" "set-default" || continue
name=$(basename "$gwdir")
best_gw="" best_gw=""
# Read gateways in priority order, pick first healthy
while IFS= read -r line ; do while IFS= read -r line ; do
[ -z "$line" ] && continue [ -z "$line" ] && continue
echo "$line" | grep -q '^#' && continue echo "$line" | grep -q '^#' && continue
parse_gw_line "$line" "$ipcmd" parse_gw_line "$line" "$ipcmd"
[ -z "$gw_ip" ] && continue [ -z "$gw_ip" ] && continue
gw_tag=$(find_gw_monitor "$routes_dir" "$gw_ip" "$ipcmd") gw_tag=$(find_gw_monitor "$routes_dir" "$gw_ip" "$ipcmd")
if [ -z "$gw_tag" ] ; then if [ -z "$gw_tag" ] ; then
[ -n "$SHOW" ] && log "[$name] $gw_ip: no monitor, assuming healthy" [ -n "$SHOW" ] && log "[$name] $gw_ip: no monitor, assuming healthy"
best_gw="$gw_ip" best_gw="$gw_ip"
break break
fi fi
status=$(get_health "$gw_tag") status=$(get_health "$gw_tag")
[ -n "$SHOW" ] && log "[$name] $gw_ip ($gw_tag): $status" [ -n "$SHOW" ] && log "[$name] default candidate $gw_ip ($gw_tag): $status"
if [ "$status" = "healthy" ] ; then if [ "$status" = "healthy" ] ; then
best_gw="$gw_ip" best_gw="$gw_ip"
break break
...@@ -271,118 +332,15 @@ for routes_dir in "$ROUTES_DIR" "$ROUTES6_DIR" ; do ...@@ -271,118 +332,15 @@ for routes_dir in "$ROUTES_DIR" "$ROUTES6_DIR" ; do
cur_default=$(resolve_default_gw "$ipcmd") cur_default=$(resolve_default_gw "$ipcmd")
if [ "$cur_default" != "$best_gw" ] ; then if [ "$cur_default" != "$best_gw" ] ; then
log "[$name] Switching default: $cur_default -> $best_gw" log "[$name] Switching default: $cur_default -> $best_gw"
if [ -z "$SHOW" ] ; then [ -z "$SHOW" ] && $ipcmd route replace default via "$best_gw"
$ipcmd route replace default via "$best_gw"
fi
else else
[ -n "$SHOW" ] && log "[$name] Default already via $best_gw" [ -n "$SHOW" ] && log "[$name] Default already via $best_gw"
fi fi
done done
done done
# --- Process dead groups --- # Reload routes for recovered gateways
for state_path in $dead_groups ; do
name=$(basename "$state_path")
routes_dir=$(dirname "$state_path")
gwdir="$state_path"
mkdir -p "$STATE_DIR/$state_path"
ipcmd=$(ipcmd_for "$routes_dir")
# Flapping protection: require DOWN_THRESHOLD consecutive checks
down_count=0
[ -f "$STATE_DIR/$state_path/down_count" ] && read -r down_count < "$STATE_DIR/$state_path/down_count"
down_count=$((down_count + 1))
if [ "$down_count" -lt "$DOWN_THRESHOLD" ] ; then
log "[$name] Down ($down_count/$DOWN_THRESHOLD), waiting"
[ -z "$SHOW" ] && echo "$down_count" > "$STATE_DIR/$state_path/down_count"
continue
fi
# Safety: don't flush if no healthy alternatives (last resort)
if [ "$healthy_count" -eq 0 ] ; then
log "[$name] Down but no healthy alternatives, keeping routes"
[ -z "$SHOW" ] && echo "$down_count" > "$STATE_DIR/$state_path/down_count"
continue
fi
# Already flushed?
if [ -f "$STATE_DIR/$state_path/health" ] && [ "$(cat "$STATE_DIR/$state_path/health")" = "down" ] ; then
[ -n "$SHOW" ] && log "[$name] Already down"
continue
fi
# Check if this is a metric group with per-gateway failover
has_metric=""
[ -f "$gwdir/gateway" ] && grep -v '^#' "$gwdir/gateway" | grep -q 'metric' && has_metric=1
if [ -n "$has_metric" ] ; then
# Metric mode: only remove routes via this dead gateway, keep others
# Find dead gateway IP from monitor tag
tag=$(read_value "$gwdir/monitor")
dead_gw=""
while IFS= read -r gw_line ; do
[ -z "$gw_line" ] && continue
echo "$gw_line" | grep -q '^#' && continue
parse_gw_line "$gw_line" "$ipcmd"
[ -z "$gw_ip" ] && continue
gw_tag=$(find_gw_monitor "$routes_dir" "$gw_ip" "$ipcmd")
if [ "$gw_tag" = "$tag" ] ; then
dead_gw="$gw_ip"
break
fi
done < "$gwdir/gateway"
if [ -n "$dead_gw" ] ; then
log "[$name] Dead (${down_count}x confirmed), removing routes via $dead_gw (metric failover)"
if [ -z "$SHOW" ] ; then
flush_gw_routes "$STATE_DIR/$state_path" "$dead_gw" "$ipcmd"
fi
else
log "[$name] Dead (${down_count}x confirmed), cannot identify dead gateway IP"
fi
else
# Non-metric: flush all per-list tables
log "[$name] Dead (${down_count}x confirmed), flushing all per-list tables"
if [ -z "$SHOW" ] ; then
flush_group_tables "$STATE_DIR/$state_path" "$ipcmd"
fi
fi
if [ -z "$SHOW" ] ; then
echo "down" > "$STATE_DIR/$state_path/health"
echo "$down_count" > "$STATE_DIR/$state_path/down_count"
fi
done
# --- Recover healthy groups that were previously down ---
need_reload=
for state_path in $healthy_groups ; do
name=$(basename "$state_path")
# Reset down_count for consistently healthy groups
[ -z "$SHOW" ] && rm -f "$STATE_DIR/$state_path/down_count"
# Check if was previously down
[ -f "$STATE_DIR/$state_path/health" ] || continue
[ "$(cat "$STATE_DIR/$state_path/health")" = "down" ] || continue
log "[$name] Recovered, scheduling route reload"
if [ -z "$SHOW" ] ; then
echo "up" > "$STATE_DIR/$state_path/health"
# Remove per-list state hashes to trigger reload on next route-update run
for list_state in "$STATE_DIR/$state_path"/*/ ; do
[ -d "$list_state" ] || continue
rm -f "$list_state/hash" "$list_state/resolved"
done
need_reload=1
fi
done
# Reload routes for recovered groups
if [ -n "$need_reload" ] ; then if [ -n "$need_reload" ] ; then
log "Running route-update.sh for recovered groups" log "Running route-update.sh for recovered gateways"
[ -z "$SHOW" ] && ./route-update.sh [ -z "$SHOW" ] && ./route-update.sh
fi fi
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment