Skip to content

E
etersoft-admin-essentials
Commit f98973c6 authored by Vitaly Lipatov's avatar Vitaly Lipatov
Browse files
Options

router: per-list tables, gateway metric, per-gateway failover

Per-list tables: each .list file gets its own routing table (auto-allocated 200-250) instead of one table per group. Enables BGP redistribution per list. Gateway metric: "IP metric N" syntax in gateway file. Multiple gateways with metric get separate route entries (preference-based) instead of ECMP multipath. Per-gateway failover: route-health.sh removes routes only via dead gateway in metric groups, keeping fallback routes alive. Refactored process_routes() into check_list_changed(), resolve_list_file(), load_list_routes() subfunctions for readability. Co-Authored-By: 's avatarClaude Opus 4.6 <noreply@anthropic.com>
parent 70b191be
Hide whitespace changes
Inline Side-by-side
Showing with 494 additions and 299 deletions
router/functions
View file @ f98973c6
...@@ -25,6 +25,16 @@ table_by_name() { awk -v n="$1" '$2 == n { print $1; exit }' /etc/iproute2/rt_ta ...@@ -25,6 +25,16 @@ table_by_name() { awk -v n="$1" '$2 == n { print $1; exit }' /etc/iproute2/rt_ta
# Resolve "default" keyword to actual default gateway # Resolve "default" keyword to actual default gateway
resolve_default_gw() { $1 route show default | awk '/default/ {print $3; exit}' ; } resolve_default_gw() { $1 route show default | awk '/default/ {print $3; exit}' ; }
# Parse gateway line: "IP [metric N]" → sets gw_ip, gw_metric
# Usage: parse_gw_line "91.232.225.122 metric 10" "ip"
parse_gw_line()
{
local line="$1" ipcmd="${2:-ip}"
local raw=$(echo "$line" | awk '{print $1}')
gw_ip=$(resolve_gw "$raw" "$ipcmd")
gw_metric=$(echo "$line" | awk '/metric/{for(i=1;i<=NF;i++) if($i=="metric") print $(i+1)}')
}
# Resolve gateway value: "default" → system default, hostname → IP, IP → as-is # Resolve gateway value: "default" → system default, hostname → IP, IP → as-is
resolve_gw() resolve_gw()
{ {
......
router/route-health.sh
View file @ f98973c6
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
# Health-check failover for route groups via Telegraf/InfluxDB # Health-check failover for route groups via Telegraf/InfluxDB
# Monitors gateway health and flushes dead routes when live alternatives exist # Monitors gateway health and flushes dead routes when live alternatives exist
# Manages default route for groups with set-default option # Manages default route for groups with set-default option
# Supports per-list tables and per-gateway failover (metric mode)
# #
# Usage: route-health.sh [--show] # Usage: route-health.sh [--show]
...@@ -69,6 +70,51 @@ get_health() ...@@ -69,6 +70,51 @@ get_health()
}' }'
} }
# Find monitor tag for a gateway IP by scanning all groups
# Usage: find_gw_monitor ROUTES_DIR GW_IP IPCMD
find_gw_monitor()
{
local routes_dir="$1" gw_ip="$2" ipcmd="$3"
for check_dir in "$routes_dir"/*/ ; do
[ -d "$check_dir" ] || continue
[ -f "$check_dir/monitor" ] || continue
local check_ip=$(resolve_gw "$(read_value "$check_dir/gateway")" "$ipcmd" 2>/dev/null)
if [ "$check_ip" = "$gw_ip" ] ; then
read_value "$check_dir/monitor"
return
fi
done
}
# Flush per-list tables for a group
# Usage: flush_group_tables STATE_GROUP_DIR IPCMD
flush_group_tables()
{
local group_state="$1" ipcmd="$2"
for list_state in "$group_state"/*/ ; do
[ -d "$list_state" ] || continue
[ -f "$list_state/table" ] || continue
local t ; read -r t < "$list_state/table"
$ipcmd route flush table "$t" 2>/dev/null
done
}
# Remove routes via specific gateway from all per-list tables
# Usage: flush_gw_routes STATE_GROUP_DIR GW_IP IPCMD
flush_gw_routes()
{
local group_state="$1" dead_gw="$2" ipcmd="$3"
for list_state in "$group_state"/*/ ; do
[ -d "$list_state" ] || continue
[ -f "$list_state/table" ] || continue
local t ; read -r t < "$list_state/table"
# Remove routes via dead gateway
$ipcmd route show table "$t" 2>/dev/null | grep "via $dead_gw" | while read -r route ; do
$ipcmd route del $route table "$t" 2>/dev/null
done
done
}
# --- Evaluate health for all monitored groups --- # --- Evaluate health for all monitored groups ---
healthy_count=0 healthy_count=0
dead_count=0 dead_count=0
...@@ -129,24 +175,12 @@ for routes_dir in "$ROUTES_DIR" "$ROUTES6_DIR" ; do ...@@ -129,24 +175,12 @@ for routes_dir in "$ROUTES_DIR" "$ROUTES6_DIR" ; do
[ -z "$line" ] && continue [ -z "$line" ] && continue
echo "$line" | grep -q '^#' && continue echo "$line" | grep -q '^#' && continue
gw_ip=$(resolve_gw "$line" "$ipcmd") parse_gw_line "$line" "$ipcmd"
[ -z "$gw_ip" ] && continue [ -z "$gw_ip" ] && continue
# Find monitor tag for this gateway IP gw_tag=$(find_gw_monitor "$routes_dir" "$gw_ip" "$ipcmd")
# Look through all groups for one whose gateway matches
gw_tag=""
for check_dir in "$routes_dir"/*/ ; do
[ -d "$check_dir" ] || continue
[ -f "$check_dir/monitor" ] || continue
check_ip=$(resolve_gw "$(read_value "$check_dir/gateway")" "$ipcmd" 2>/dev/null)
if [ "$check_ip" = "$gw_ip" ] ; then
gw_tag=$(read_value "$check_dir/monitor")
break
fi
done
if [ -z "$gw_tag" ] ; then if [ -z "$gw_tag" ] ; then
# No monitor for this gateway — assume healthy (first unmonitored wins)
[ -n "$SHOW" ] && log "[$name] $gw_ip: no monitor, assuming healthy" [ -n "$SHOW" ] && log "[$name] $gw_ip: no monitor, assuming healthy"
best_gw="$gw_ip" best_gw="$gw_ip"
break break
...@@ -182,16 +216,11 @@ done ...@@ -182,16 +216,11 @@ done
for state_path in $dead_groups ; do for state_path in $dead_groups ; do
name=$(basename "$state_path") name=$(basename "$state_path")
routes_dir=$(dirname "$state_path") routes_dir=$(dirname "$state_path")
gwdir="$state_path"
mkdir -p "$STATE_DIR/$state_path" mkdir -p "$STATE_DIR/$state_path"
# Read table number from group config or rt_tables ipcmd=$(ipcmd_for "$routes_dir")
table=""
if [ -f "$state_path/table" ] ; then
table=$(read_value "$state_path/table")
else
table=$(table_by_name "$name")
fi
# Flapping protection: require DOWN_THRESHOLD consecutive checks # Flapping protection: require DOWN_THRESHOLD consecutive checks
down_count=0 down_count=0
...@@ -217,11 +246,44 @@ for state_path in $dead_groups ; do ...@@ -217,11 +246,44 @@ for state_path in $dead_groups ; do
continue continue
fi fi
ipcmd=$(ipcmd_for "$routes_dir") # Check if this is a metric group with per-gateway failover
has_metric=""
[ -f "$gwdir/gateway" ] && grep -v '^#' "$gwdir/gateway" | grep -q 'metric' && has_metric=1
if [ -n "$has_metric" ] ; then
# Metric mode: only remove routes via this dead gateway, keep others
# Find dead gateway IP from monitor tag
tag=$(read_value "$gwdir/monitor")
dead_gw=""
while IFS= read -r gw_line ; do
[ -z "$gw_line" ] && continue
echo "$gw_line" | grep -q '^#' && continue
parse_gw_line "$gw_line" "$ipcmd"
[ -z "$gw_ip" ] && continue
gw_tag=$(find_gw_monitor "$routes_dir" "$gw_ip" "$ipcmd")
if [ "$gw_tag" = "$tag" ] ; then
dead_gw="$gw_ip"
break
fi
done < "$gwdir/gateway"
if [ -n "$dead_gw" ] ; then
log "[$name] Dead (${down_count}x confirmed), removing routes via $dead_gw (metric failover)"
if [ -z "$SHOW" ] ; then
flush_gw_routes "$STATE_DIR/$state_path" "$dead_gw" "$ipcmd"
fi
else
log "[$name] Dead (${down_count}x confirmed), cannot identify dead gateway IP"
fi
else
# Non-metric: flush all per-list tables
log "[$name] Dead (${down_count}x confirmed), flushing all per-list tables"
if [ -z "$SHOW" ] ; then
flush_group_tables "$STATE_DIR/$state_path" "$ipcmd"
fi
fi
log "[$name] Dead (${down_count}x confirmed), flushing table $table" if [ -z "$SHOW" ] ; then
if [ -z "$SHOW" ] && [ -n "$table" ] ; then
$ipcmd route flush table "$table" 2>/dev/null
echo "down" > "$STATE_DIR/$state_path/health" echo "down" > "$STATE_DIR/$state_path/health"
echo "$down_count" > "$STATE_DIR/$state_path/down_count" echo "$down_count" > "$STATE_DIR/$state_path/down_count"
fi fi
...@@ -242,8 +304,11 @@ for state_path in $healthy_groups ; do ...@@ -242,8 +304,11 @@ for state_path in $healthy_groups ; do
log "[$name] Recovered, scheduling route reload" log "[$name] Recovered, scheduling route reload"
if [ -z "$SHOW" ] ; then if [ -z "$SHOW" ] ; then
echo "up" > "$STATE_DIR/$state_path/health" echo "up" > "$STATE_DIR/$state_path/health"
# Remove route-update state to trigger reload on next run # Remove per-list state hashes to trigger reload on next route-update run
rm -f "$STATE_DIR/$state_path/hash" "$STATE_DIR/$state_path/resolved" for list_state in "$STATE_DIR/$state_path"/*/ ; do
[ -d "$list_state" ] || continue
rm -f "$list_state/hash" "$list_state/resolved"
done
need_reload=1 need_reload=1
fi fi
done done
......
router/route-update.sh
View file @ f98973c6
...@@ -65,20 +65,26 @@ MODES OF OPERATION: ...@@ -65,20 +65,26 @@ MODES OF OPERATION:
--help, -h Show this help and exit. --help, -h Show this help and exit.
DIRECTORY STRUCTURE: DIRECTORY STRUCTURE:
routes.d/GROUP/gateway Gateway: IP, hostname, or "default" (one per line). routes.d/GROUP/gateway Gateway(s), one per line: IP, hostname, or "default".
Hostnames are resolved via dig. Multiple lines = multipath. Hostnames are resolved via dig. Multiple lines = multipath.
With "metric N" suffix: preference-based failover.
routes.d/GROUP/options Optional flags, one per line: routes.d/GROUP/options Optional flags, one per line:
set-default Managed by route-health.sh (default route failover). set-default Managed by route-health.sh (default route failover).
routes.d/GROUP/table (legacy) Routing table number override.
routes.d/GROUP/*.list Symlinks to files with IPs, subnets, or domains. routes.d/GROUP/*.list Symlinks to files with IPs, subnets, or domains.
routes6.d/ Same structure for IPv6 groups. routes6.d/ Same structure for IPv6 groups.
GATEWAY METRIC:
Without metric: multiple gateways use ECMP multipath (equal-cost).
With metric: each gateway gets its own route entry with metric value.
Lower metric = preferred. Failover: route-health.sh removes dead gateway routes.
Example:
91.232.225.122 metric 10 # preferred
91.232.225.112 metric 20 # fallback
TABLE RESOLUTION: TABLE RESOLUTION:
Table number is determined by (in order): Each .list file gets its own routing table (per-list tables).
1. routes.d/GROUP/table file (legacy, if present) Table number is auto-allocated in range 200-250 and registered in rt_tables.
2. /etc/iproute2/rt_tables entry matching GROUP name
3. Last numeric component of gateway IP (e.g. 91.232.225.122 → 122),
auto-registered in rt_tables for future use.
ip rule pref = table_number × 10. ip rule pref = table_number × 10.
EXAMPLES: EXAMPLES:
...@@ -145,58 +151,74 @@ detect_volatile_domains() ...@@ -145,58 +151,74 @@ detect_volatile_domains()
fi fi
} }
# Look up table number: first from rt_tables by name, then from last IP octet # Allocate a free table number in range 200-250 (step 10)
# If derived from IP, auto-creates entry in rt_tables alloc_table()
{
local name="$1"
local used=$(awk '/^[0-9]/ {print $1}' /etc/iproute2/rt_tables | sort -n)
local num=200
while [ $num -le 250 ] ; do
if ! echo "$used" | grep -q "^${num}$" ; then
echo "$num $name" >> /etc/iproute2/rt_tables
vlog "Allocated table $num for $name"
echo "$num"
return
fi
num=$((num + 1))
done
log "ERROR: Cannot allocate table for $name: range 200-250 exhausted" >&2
return 1
}
# Look up table number by name; allocate if not found
lookup_table() lookup_table()
{ {
local name="$1" gateway="$2" local name="$1"
local num=$(table_by_name "$name") local num=$(table_by_name "$name")
if [ -n "$num" ] ; then echo "$num" ; return ; fi if [ -n "$num" ] ; then echo "$num" ; return ; fi
alloc_table "$name"
# Fallback: last component of gateway IP (91.232.225.122 → 122)
num=$(echo "$gateway" | sed -E 's/.*[.:]([0-9]+)$/\1/')
if [ -n "$num" ] ; then
echo "$num $name" >> /etc/iproute2/rt_tables
vlog "Added rt_tables entry: $num $name" >&2
fi
echo "$num"
} }
# Read gateway and table from a group directory # Read gateway config from a group directory
# Sets: gw, table, route_via, opt_set_default # Sets: gw, opt_set_default, has_metric
read_group_config() read_group_config()
{ {
local dir="$1" local dir="$1"
local ipcmd="${2:-ip}" local ipcmd="${2:-ip}"
local name=$(basename "$dir") local name=$(basename "$dir")
gw="" ; table="" ; route_via="" ; opt_set_default="" gw="" ; opt_set_default="" ; has_metric=""
if [ -f "$dir/gateway" ] ; then
local raw_gw=$(read_value "$dir/gateway")
gw=$(resolve_gw "$raw_gw" "$ipcmd")
if [ -z "$gw" ] ; then
echo "[$name] Cannot resolve gateway '$raw_gw', skipping" >&2
return 1
fi
else
echo "[$name] No gateway file, skipping" >&2
return 1
fi
# Table from file (legacy) or by name lookup [ -f "$dir/gateway" ] || { echo "[$name] No gateway file, skipping" >&2 ; return 1 ; }
if [ -f "$dir/table" ] ; then
table=$(read_value "$dir/table")
else
table=$(lookup_table "$name" "$gw")
fi
if [ -z "$table" ] ; then # Parse first gateway for gw variable (used for display)
echo "[$name] Cannot determine table number, skipping" >&2 local first_line=$(read_value "$dir/gateway")
return 1 parse_gw_line "$first_line" "$ipcmd"
fi gw="$gw_ip"
[ -z "$gw" ] && { echo "[$name] Cannot resolve gateway '$first_line'" >&2 ; return 1 ; }
# Detect metric mode
grep -v '^#' "$dir/gateway" | grep -q 'metric' && has_metric=1
# Options
has_option "$dir" "set-default" && opt_set_default=1 has_option "$dir" "set-default" && opt_set_default=1
}
# Build route_via suffix for a list's table (used by --add/--del/--flush)
# Sets: route_via, table
# Args: group_dir ipcmd list_name
build_route_via()
{
local dir="$1" ipcmd="${2:-ip}" list_name="$3"
local name=$(basename "$dir")
# Determine table
if [ -n "$list_name" ] ; then
table=$(lookup_table "$list_name")
elif [ -f "$dir/table" ] ; then
table=$(read_value "$dir/table")
else
table=$(lookup_table "$name")
fi
[ -z "$table" ] && { echo "[$name] Cannot determine table number" >&2 ; return 1 ; }
# Build route suffix: single vs multipath # Build route suffix: single vs multipath
local gw_count=$(read_values "$dir/gateway" | wc -l) local gw_count=$(read_values "$dir/gateway" | wc -l)
...@@ -234,6 +256,7 @@ if [ "$ACTION" = "add" ] || [ "$ACTION" = "del" ] ; then ...@@ -234,6 +256,7 @@ if [ "$ACTION" = "add" ] || [ "$ACTION" = "del" ] ; then
ipcmd=$(ipcmd_for "$groupdir") ipcmd=$(ipcmd_for "$groupdir")
read_group_config "$groupdir" "$ipcmd" || exit 1 read_group_config "$groupdir" "$ipcmd" || exit 1
build_route_via "$groupdir" "$ipcmd" || exit 1
if [ "$ACTION" = "add" ] ; then if [ "$ACTION" = "add" ] ; then
if is_ipv4 "$ADD_DEL_TARGET" || is_ipv6 "$ADD_DEL_TARGET" ; then if is_ipv4 "$ADD_DEL_TARGET" || is_ipv6 "$ADD_DEL_TARGET" ; then
...@@ -262,257 +285,332 @@ if [ -n "$FLUSH_GROUP" ] ; then ...@@ -262,257 +285,332 @@ if [ -n "$FLUSH_GROUP" ] ; then
groupdir=$(find_group "$FLUSH_GROUP") || exit 1 groupdir=$(find_group "$FLUSH_GROUP") || exit 1
ipcmd=$(ipcmd_for "$groupdir") ipcmd=$(ipcmd_for "$groupdir")
read_group_config "$groupdir" "$ipcmd" || exit 1 read_group_config "$groupdir" "$ipcmd" || exit 1
log "Flushing table $table (group $FLUSH_GROUP)"
[ -z "$SHOW" ] && $ipcmd route flush table "$table" 2>/dev/null
exit
fi
# --- Process a routes directory (works for both IPv4 and IPv6) --- # Flush all per-list tables in this group
# Usage: process_routes ROUTES_DIR resolve_func ip_cmd state_base="$STATE_DIR/$( [ "$ipcmd" = "ip -6" ] && echo "$ROUTES6_DIR" || echo "$ROUTES_DIR" )/$FLUSH_GROUP"
# resolve_func: get_ipv4_list_bulk or get_ipv6_list_bulk flushed=0
# ip_cmd: "ip" or "ip -6" for list_state in "$state_base"/*/ ; do
process_routes() [ -d "$list_state" ] || continue
{ [ -f "$list_state/table" ] || continue
local routes_dir="$1" t="" ; read -r t < "$list_state/table"
local resolve_func="$2" log "Flushing table $t ($(basename "$list_state") in group $FLUSH_GROUP)"
local ipcmd="$3" if [ -z "$SHOW" ] ; then
local label="$4" # "(v6)" or "" $ipcmd route flush table "$t" 2>/dev/null
$ipcmd rule del lookup "$t" 2>/dev/null
[ -d "$routes_dir" ] || return 0 fi
flushed=1
done
for gwdir in "$routes_dir"/*/ ; do # Fallback: flush by group name if no per-list state exists
[ -d "$gwdir" ] || continue if [ "$flushed" = "0" ] ; then
local name=$(basename "$gwdir") build_route_via "$groupdir" "$ipcmd" || exit 1
log "Flushing table $table (group $FLUSH_GROUP)"
[ -z "$SHOW" ] && $ipcmd route flush table "$table" 2>/dev/null
fi
exit
fi
read_group_config "$gwdir" "$ipcmd" || continue # --- Per-list processing subfunctions ---
# These use module-level variables set by the caller:
# _tag, _ipcmd, _gwdir, _resolve_func, _label, _table, _pref, _state
# _has_metric, _gw (first gateway IP), _f (list file path)
# Convention: _tag = "[group/list]" for log messages
local pref=$(rule_pref "$table") # Check if list needs processing (hash + rules/routes integrity)
local state="$routes_dir/$name" # Sets: _current_hash, _need_reload, _need_resolve
# Returns 0 if needs processing, 1 if should skip
check_list_changed()
{
_current_hash=$( { echo "dns=$EXTRA_DNS"; cat "$_gwdir/gateway" "$_gwdir/options" "$_f"; } 2>/dev/null | md5sum | awk '{print $1}')
_need_reload= ; _need_resolve=1
if [ -z "$FORCE" ] && [ -z "$RESOLVE" ] && [ -f "$STATE_DIR/$_state/hash" ] ; then
local saved_hash
read -r saved_hash < "$STATE_DIR/$_state/hash"
vlog "$_tag$_label hash: saved=$saved_hash current=$_current_hash"
if [ "$_current_hash" = "$saved_hash" ] ; then
local rule_count=$($_ipcmd -N rule show | grep -c "lookup $_table")
local route_count=$($_ipcmd route show table "$_table" 2>/dev/null | grep -c '^[^[:space:]]')
vlog "$_tag$_label hash match, checking state: rules=$rule_count routes=$route_count"
if [ "$rule_count" = "0" ] ; then
_need_reload=1
vlog "$_tag$_label ip rule for table $_table missing"
elif [ "$route_count" = "0" ] ; then
_need_reload=1
vlog "$_tag$_label route table $_table empty"
elif [ -f "$STATE_DIR/$_state/resolved" ] ; then
local expected=$(wc -l < "$STATE_DIR/$_state/resolved")
local threshold=$(( expected - expected / 100 ))
if [ "$route_count" -lt "$threshold" ] ; then
_need_reload=1
vlog "$_tag$_label route count $route_count < $threshold (expected $expected)"
fi
fi
if [ -n "$_need_reload" ] ; then
log "$_tag$_label Rules/routes lost, forcing reload"
else
vlog "$_tag$_label no changes, skipping"
return 1
fi
fi
else
vlog "$_tag$_label state: force=$FORCE resolve=$RESOLVE hash_exists=$([ -f "$STATE_DIR/$_state/hash" ] && echo yes || echo no)"
fi
ensure_state_dir "$state" # Detect routes/rules lost even when hash changed
vlog "[$name]$label table=$table pref=$pref gw=$gw dir=$gwdir" if [ -z "$FORCE" ] && [ -z "$_need_reload" ] ; then
local cur_rules=$($_ipcmd -N rule show | grep -c "lookup $_table")
local cur_routes=$($_ipcmd route show table "$_table" 2>/dev/null | grep -c '^[^[:space:]]')
if [ "$cur_rules" = "0" ] || [ "$cur_routes" = "0" ] ; then
_need_reload=1
log "$_tag$_label Rules/routes lost, forcing reload"
fi
fi
# Collect .list files (follow symlinks) return 0
local lists=$(find -L "$gwdir" -maxdepth 1 -name '*.list' -type f 2>/dev/null | sort) }
if [ -z "$lists" ] ; then # Resolve list file to IPs, manage history
log "[$name]$label No .list files, flushing table $table" # Produces: _resolved_new file
if [ -z "$SHOW" ] ; then # Uses: _need_resolve (may skip actual resolve if only gateway changed)
$ipcmd route flush table "$table" 2>/dev/null resolve_list_file()
$ipcmd rule del lookup "$table" 2>/dev/null {
rm -f "$STATE_DIR/$state/hash" "$STATE_DIR/$state/resolved" _list_hash=$(md5_lists "$_f")
fi _resolved_new="$STATE_DIR/$_state/resolved.new"
continue
# Skip re-resolve if only gateway/options changed (list unchanged)
if [ -z "$FORCE" ] && [ -z "$RESOLVE" ] && [ -f "$STATE_DIR/$_state/list_hash" ] && [ -f "$STATE_DIR/$_state/resolved" ] ; then
local saved_list_hash
read -r saved_list_hash < "$STATE_DIR/$_state/list_hash"
if [ "$saved_list_hash" = "$_list_hash" ] ; then
_need_resolve=
cp "$STATE_DIR/$_state/resolved" "$_resolved_new"
log "$_tag$_label Gateway/options changed, reusing resolved IPs"
fi fi
fi
# Detect table number change (e.g. rt_tables edited manually) [ -z "$_need_resolve" ] && return 0
if [ -f "$STATE_DIR/$state/table" ] ; then
local saved_table if [ -n "$RESOLVE" ] ; then
read -r saved_table < "$STATE_DIR/$state/table" log "$_tag$_label Re-resolving..."
if [ "$saved_table" != "$table" ] ; then else
log "[$name]$label Table changed $saved_table → $table, flushing old" local changed_files=""
if [ -z "$SHOW" ] ; then local hash_file="$STATE_DIR/$_state/hash"
$ipcmd route flush table "$saved_table" 2>/dev/null if [ -f "$hash_file" ] ; then
$ipcmd rule del lookup "$saved_table" 2>/dev/null local cf
rm -f "$STATE_DIR/$state/hash" for cf in "$_gwdir/gateway" "$_gwdir/options" "$_f" ; do
fi [ "$cf" -nt "$hash_file" ] && changed_files="$changed_files $(basename "$cf")"
fi done
fi fi
log "$_tag$_label Changes detected${changed_files:+ in:$changed_files}, resolving..."
fi
# Compute hashes: lists only (for resolve skip) and full (for change detect) vlog "$_tag$_label resolving..."
local lists_hash=$(md5_lists $lists) if grep -v '^#' "$_f" | grep -q '[a-zA-Z]' ; then
local current_hash=$( { echo "dns=$EXTRA_DNS"; cat "$gwdir/gateway" "$gwdir/options" $lists; } 2>/dev/null | md5sum | awk '{print $1}') cat_expanded "$_f" | grep -v '^#' | grep -v '^$' | $_resolve_func | \
local need_reload= need_resolve=1 grep -v '^#' | grep -v '^$' > "$_resolved_new.tmp"
else
if [ -z "$FORCE" ] && [ -z "$RESOLVE" ] && [ -f "$STATE_DIR/$state/hash" ] ; then if [ "$_ipcmd" = "ip -6" ] ; then
local saved_hash grep -v '^#' "$_f" | grep -v '^$' | grep ':' > "$_resolved_new.tmp"
read -r saved_hash < "$STATE_DIR/$state/hash"
vlog "[$name]$label hash: saved=$saved_hash current=$current_hash"
if [ "$current_hash" = "$saved_hash" ] ; then
# Check if rules/routes were lost (e.g. after network restart)
local rule_count=$($ipcmd -N rule show | grep -c "lookup $table")
local route_count=$($ipcmd route show table "$table" 2>/dev/null | grep -c '^[^[:space:]]')
vlog "[$name]$label hash match, checking state: rules=$rule_count routes=$route_count"
if [ "$rule_count" = "0" ] ; then
need_reload=1
vlog "[$name]$label ip rule for table $table missing"
elif [ "$route_count" = "0" ] ; then
need_reload=1
vlog "[$name]$label route table $table empty"
elif [ -f "$STATE_DIR/$state/resolved" ] ; then
local expected=$(wc -l < "$STATE_DIR/$state/resolved")
local threshold=$(( expected - expected / 100 ))
if [ "$route_count" -lt "$threshold" ] ; then
need_reload=1
vlog "[$name]$label route count $route_count < $threshold (expected $expected, 1% tolerance)"
fi
fi
if [ -n "$need_reload" ] ; then
log "[$name]$label Rules/routes lost, forcing reload"
else
vlog "[$name]$label no changes, skipping"
continue
fi
fi
else else
vlog "[$name]$label state: force=$FORCE resolve=$RESOLVE hash_exists=$([ -f "$STATE_DIR/$state/hash" ] && echo yes || echo no)" grep -v '^#' "$_f" | grep -v '^$' > "$_resolved_new.tmp"
fi fi
fi
sed 's|/32$||' "$_resolved_new.tmp" | sort -u > "$_resolved_new"
rm -f "$_resolved_new.tmp"
# Accumulate resolve history (DNS round-robin)
local history_dir="$STATE_DIR/$_state/resolve_history"
mkdir -p "$history_dir"
rm -f "$history_dir/$HISTORY_SIZE"
local i=$((HISTORY_SIZE - 1))
while [ $i -ge 1 ] ; do
[ -f "$history_dir/$i" ] && mv "$history_dir/$i" "$history_dir/$((i + 1))"
i=$((i - 1))
done
cp "$_resolved_new" "$history_dir/1"
sort -u "$history_dir"/* > "${_resolved_new}.merged"
mv "${_resolved_new}.merged" "$_resolved_new"
vlog "$_tag$_label resolve history: $(ls "$history_dir" | wc -l) snapshots, $(wc -l < "$_resolved_new") unique IPs"
# Detect volatile domains
if grep -v '^#' "$_f" | grep -q '[a-zA-Z]' && [ "$_ipcmd" = "ip" ] ; then
local _vname=$(echo "$_tag" | tr -d '[]')
detect_volatile_domains "$_state" "$_vname" "$_label" "$_f"
fi
}
# Detect routes/rules lost even when hash changed (e.g. DNS availability) # Load routes into table, remove stale, verify, ensure ip rule
if [ -z "$FORCE" ] && [ -z "$need_reload" ] ; then load_list_routes()
local cur_rules=$($ipcmd -N rule show | grep -c "lookup $table") {
local cur_routes=$($ipcmd route show table "$table" 2>/dev/null | grep -c '^[^[:space:]]') local count=$(wc -l < "$_resolved_new")
if [ "$cur_rules" = "0" ] || [ "$cur_routes" = "0" ] ; then
need_reload=1 # Check if resolved IPs actually changed
log "[$name]$label Rules/routes lost, forcing reload" vlog "$_tag$_label resolved $count IPs, checking against saved state"
if [ -z "$FORCE" ] && [ -z "$_need_reload" ] && [ -f "$STATE_DIR/$_state/resolved" ] ; then
if diff -q "$STATE_DIR/$_state/resolved" "$_resolved_new" >/dev/null 2>&1 ; then
if [ -f "$STATE_DIR/$_state/gateway" ] && diff -q "$STATE_DIR/$_state/gateway" "$_gwdir/gateway" >/dev/null 2>&1 ; then
log "$_tag$_label Resolved IPs unchanged, updating hash only"
echo "$_current_hash" > "$STATE_DIR/$_state/hash"
rm -f "$_resolved_new"
return 1 # signal: skip loading
fi fi
log "$_tag$_label Gateway changed, reloading routes"
fi fi
fi
# Skip re-resolve if only gateway/options changed (lists unchanged) local gw_display=$(read_values "$_gwdir/gateway" | awk '{print $1}' | paste -sd,)
local resolved_new="$STATE_DIR/$state/resolved.new" log "$_tag$_label Loading $count routes into table $_table via $gw_display"
if [ -z "$FORCE" ] && [ -z "$RESOLVE" ] && [ -f "$STATE_DIR/$state/lists_hash" ] && [ -f "$STATE_DIR/$state/resolved" ] ; then
local saved_lists_hash
read -r saved_lists_hash < "$STATE_DIR/$state/lists_hash"
if [ "$saved_lists_hash" = "$lists_hash" ] ; then
need_resolve=
cp "$STATE_DIR/$state/resolved" "$resolved_new"
log "[$name]$label Gateway/options changed, reusing resolved IPs"
fi
fi
if [ -n "$need_resolve" ] ; then if [ -n "$SHOW" ] ; then
if [ -n "$RESOLVE" ] ; then echo " Would load $count routes into table $_table"
log "[$name]$label Re-resolving..." head -5 "$_resolved_new" | sed 's/^/ /'
else [ "$count" -gt 5 ] && echo " ... ($count total)"
# Report which files changed (newer than saved hash) rm -f "$_resolved_new"
local changed_files="" return 1
local hash_file="$STATE_DIR/$state/hash" fi
if [ -f "$hash_file" ] ; then
for f in "$gwdir/gateway" "$gwdir/options" $lists ; do
if [ "$f" -nt "$hash_file" ] ; then
changed_files="$changed_files $(basename "$f")"
fi
done
fi
log "[$name]$label Changes detected${changed_files:+ in:$changed_files}, resolving..."
fi
# Resolve each list file individually, cache results per file # Zero-downtime reload: replace all, then remove stale
local resolved_parts_dir="$STATE_DIR/$state/resolved_parts" if [ -n "$_has_metric" ] ; then
mkdir -p "$resolved_parts_dir" {
for f in $lists ; do while IFS= read -r gw_line ; do
local bname=$(basename "$f") [ -z "$gw_line" ] && continue
local cached="$resolved_parts_dir/$bname" echo "$gw_line" | grep -q '^#' && continue
# Re-resolve only changed files (or all if no cache) parse_gw_line "$gw_line" "$_ipcmd"
if [ -f "$cached" ] && [ -f "$hash_file" ] && ! [ "$f" -nt "$hash_file" ] ; then [ -z "$gw_ip" ] && continue
vlog "[$name]$label $bname: unchanged, using cache" sed "s|^|route replace |; s|$| via $gw_ip table $_table${gw_metric:+ metric $gw_metric}|" "$_resolved_new"
continue done < "$_gwdir/gateway"
fi } | $_ipcmd -batch - 2>&1 | grep -v "^$" | head -5
vlog "[$name]$label $bname: resolving..." else
if grep -v '^#' "$f" | grep -q '[a-zA-Z]' ; then local route_via
# Domain file: resolve local gw_count=$(read_values "$_gwdir/gateway" | wc -l)
cat_expanded "$f" | grep -v '^#' | grep -v '^$' | $resolve_func | \ if [ "$gw_count" -le 1 ] ; then
grep -v '^#' | grep -v '^$' > "$cached.tmp" route_via="via $_gw table $_table"
else else
# IP-only file: just filter route_via="table $_table"
if [ "$ipcmd" = "ip -6" ] ; then local g
grep -v '^#' "$f" | grep -v '^$' | grep ':' > "$cached.tmp" for g in $(read_values "$_gwdir/gateway") ; do
else g=$(resolve_gw "$g" "$_ipcmd")
grep -v '^#' "$f" | grep -v '^$' > "$cached.tmp" [ -z "$g" ] && continue
fi route_via="$route_via nexthop via $g weight 1"
fi
mv "$cached.tmp" "$cached"
done
cat "$resolved_parts_dir"/* 2>/dev/null | sed 's|/32$||' | sort -u > "$resolved_new"
# Accumulate resolve history to handle DNS balancing (round-robin)
# Keeps last HISTORY_SIZE snapshots; IPs not seen for HISTORY_SIZE runs expire
local history_dir="$STATE_DIR/$state/resolve_history"
mkdir -p "$history_dir"
rm -f "$history_dir/$HISTORY_SIZE"
local i=$((HISTORY_SIZE - 1))
while [ $i -ge 1 ] ; do
[ -f "$history_dir/$i" ] && mv "$history_dir/$i" "$history_dir/$((i + 1))"
i=$((i - 1))
done done
cp "$resolved_new" "$history_dir/1"
sort -u "$history_dir"/* > "${resolved_new}.merged"
mv "${resolved_new}.merged" "$resolved_new"
vlog "[$name]$label resolve history: $(ls "$history_dir" | wc -l) snapshots, $(wc -l < "$resolved_new") unique IPs"
# Detect domains using DNS balancing (only for IPv4, where dig A applies)
if [ -n "$domain_lists" ] && [ "$ipcmd" = "ip" ] ; then
detect_volatile_domains "$state" "$name" "$label" $domain_lists
fi
fi fi
sed "s|^|route replace |; s|$| $route_via|" "$_resolved_new" | \
$_ipcmd -batch - 2>&1 | grep -v "^$" | head -5
fi
# Check if resolved IPs actually changed # Remove stale routes
vlog "[$name]$label resolved $(wc -l < "$resolved_new") IPs, checking against saved state" local stale=$(mktemp)
if [ -z "$FORCE" ] && [ -z "$need_reload" ] && [ -f "$STATE_DIR/$state/resolved" ] ; then local resolved_norm=$(mktemp)
if diff -q "$STATE_DIR/$state/resolved" "$resolved_new" >/dev/null 2>&1 ; then sed 's|/32$||' "$_resolved_new" | sort > "$resolved_norm"
# IPs same, but gateway may have changed $_ipcmd route show table "$_table" 2>/dev/null | awk '/^[^\t ]/{print $1}' | sed 's|/32$||' | sort -u | \
if [ -f "$STATE_DIR/$state/gateway" ] && diff -q "$STATE_DIR/$state/gateway" "$gwdir/gateway" >/dev/null 2>&1 ; then comm -23 - "$resolved_norm" > "$stale"
log "[$name]$label Resolved IPs unchanged, updating hash only" rm -f "$resolved_norm"
echo "$current_hash" > "$STATE_DIR/$state/hash" if [ -s "$stale" ] ; then
rm -f "$resolved_new" vlog "$_tag$_label removing $(wc -l < "$stale") stale routes"
continue sed "s|^|route del |; s|$| table $_table|" "$stale" | \
fi $_ipcmd -batch - 2>/dev/null
log "[$name]$label Gateway changed, reloading routes" fi
fi rm -f "$stale"
fi
local count=$(wc -l < "$resolved_new") # Verify route count (unique dst IPs)
local gw_display=$(read_values "$gwdir/gateway" | paste -sd,) local actual=$($_ipcmd route show table "$_table" 2>/dev/null | awk '/^[^\t ]/{print $1}' | sort -u | wc -l)
log "[$name]$label Loading $count routes into table $table via $gw_display" if [ "$actual" != "$count" ] ; then
log "$_tag$_label WARNING: route count mismatch: expected $count, got $actual"
else
vlog "$_tag$_label route count OK: $actual"
fi
if [ -n "$SHOW" ] ; then # Ensure ip rule exists
echo " Would flush table $table and load $count routes" if ! $_ipcmd -N rule show | grep -q "lookup $_table.*pref $_pref" ; then
head -5 "$resolved_new" | sed 's/^/ /' vlog "$_tag$_label adding ip rule: lookup $_table pref $_pref"
[ "$count" -gt 5 ] && echo " ... ($count total)" $_ipcmd rule add lookup "$_table" pref "$_pref" 2>/dev/null
rm -f "$resolved_new" fi
continue
fi
# Zero-downtime reload: replace all, then remove stale # Save per-list state
sed "s|^|route replace |; s|$| $route_via|" "$resolved_new" | \ echo "$_current_hash" > "$STATE_DIR/$_state/hash"
$ipcmd -batch - 2>&1 | grep -v "^$" | head -5 echo "$_list_hash" > "$STATE_DIR/$_state/list_hash"
local stale=$(mktemp) echo "$_table" > "$STATE_DIR/$_state/table"
local resolved_norm=$(mktemp) cp "$_gwdir/gateway" "$STATE_DIR/$_state/gateway"
sed 's|/32$||' "$resolved_new" | sort > "$resolved_norm" mv "$_resolved_new" "$STATE_DIR/$_state/resolved"
$ipcmd route show table "$table" 2>/dev/null | awk '/^[^\t ]/{print $1}' | sed 's|/32$||' | sort | \
comm -23 - "$resolved_norm" > "$stale"
rm -f "$resolved_norm"
if [ -s "$stale" ] ; then
vlog "[$name]$label removing $(wc -l < "$stale") stale routes"
sed "s|^|route del |; s|$| table $table|" "$stale" | \
$ipcmd -batch - 2>/dev/null
fi
rm -f "$stale"
# Verify route count matches expected return 0
local actual=$($ipcmd route show table "$table" 2>/dev/null | grep -c '^[^[:space:]]') }
if [ "$actual" != "$count" ] ; then
log "[$name]$label WARNING: route count mismatch: expected $count, got $actual"
else
vlog "[$name]$label route count OK: $actual"
fi
# Ensure ip rule exists # --- Process a routes directory (works for both IPv4 and IPv6) ---
if ! $ipcmd -N rule show | grep -q "lookup $table.*pref $pref" ; then # Per-list tables: each .list file gets its own routing table
vlog "[$name]$label adding ip rule: lookup $table pref $pref" process_routes()
$ipcmd rule add lookup "$table" pref "$pref" 2>/dev/null {
fi local routes_dir="$1"
_resolve_func="$2"
_ipcmd="$3"
_label="$4" # "(v6)" or ""
[ -d "$routes_dir" ] || return 0
for _gwdir in "$routes_dir"/*/ ; do
[ -d "$_gwdir" ] || continue
local name=$(basename "$_gwdir")
read_group_config "$_gwdir" "$_ipcmd" || continue
_gw="$gw" ; _has_metric="$has_metric"
local group_state="$routes_dir/$name"
ensure_state_dir "$group_state"
vlog "[$name]$_label gw=$_gw has_metric=${_has_metric:-0} dir=$_gwdir"
# Save gateway in group state (shared by all lists)
[ -z "$SHOW" ] && cp "$_gwdir/gateway" "$STATE_DIR/$group_state/gateway" 2>/dev/null
# Collect .list files (follow symlinks)
local lists=$(find -L "$_gwdir" -maxdepth 1 -name '*.list' -type f 2>/dev/null | sort)
# Note: set-default is handled by route-health.sh, not here if [ -z "$lists" ] ; then
log "[$name]$_label No .list files, flushing all per-list tables"
if [ -z "$SHOW" ] ; then
for list_state in "$STATE_DIR/$group_state"/*/ ; do
[ -d "$list_state" ] || continue
[ -f "$list_state/table" ] || continue
local t ; read -r t < "$list_state/table"
$_ipcmd route flush table "$t" 2>/dev/null
$_ipcmd rule del lookup "$t" 2>/dev/null
done
rm -rf "$STATE_DIR/$group_state"/*/
fi
continue
fi
# Save state # --- Per-list loop ---
echo "$current_hash" > "$STATE_DIR/$state/hash" for _f in $lists ; do
echo "$lists_hash" > "$STATE_DIR/$state/lists_hash" local bname=$(basename "$_f")
echo "$table" > "$STATE_DIR/$state/table" local list_name="${bname%.list}"
cp "$gwdir/gateway" "$STATE_DIR/$state/gateway" _table=$(lookup_table "$list_name")
mv "$resolved_new" "$STATE_DIR/$state/resolved" [ -z "$_table" ] && { log "[$name/$list_name]$_label Cannot allocate table" >&2 ; continue ; }
_pref=$(rule_pref "$_table")
_state="$group_state/$list_name"
_tag="[$name/$list_name]"
ensure_state_dir "$_state"
vlog "$_tag$_label table=$_table pref=$_pref"
# Detect table number change
if [ -f "$STATE_DIR/$_state/table" ] ; then
local saved_table
read -r saved_table < "$STATE_DIR/$_state/table"
if [ "$saved_table" != "$_table" ] ; then
log "$_tag$_label Table changed $saved_table → $_table, flushing old"
if [ -z "$SHOW" ] ; then
$_ipcmd route flush table "$saved_table" 2>/dev/null
$_ipcmd rule del lookup "$saved_table" 2>/dev/null
rm -f "$STATE_DIR/$_state/hash"
fi
fi
fi
log "[$name]$label Done" check_list_changed || continue
resolve_list_file
load_list_routes && log "$_tag$_label Done"
done
done done
} }
...@@ -526,16 +624,38 @@ cleanup_state() ...@@ -526,16 +624,38 @@ cleanup_state()
for sdir in "$STATE_DIR/$routes_dir"/*/ ; do for sdir in "$STATE_DIR/$routes_dir"/*/ ; do
[ -d "$sdir" ] || continue [ -d "$sdir" ] || continue
local name=$(basename "$sdir") local name=$(basename "$sdir")
if [ ! -d "$routes_dir/$name" ] ; then if [ ! -d "$routes_dir/$name" ] ; then
local table="" # Group removed — flush all per-list tables
[ -f "$sdir/table" ] && read -r table < "$sdir/table" log "Cleaning orphaned state for group $name"
log "Cleaning orphaned state for $name" if [ -z "$SHOW" ] ; then
if [ -z "$SHOW" ] && [ -n "$table" ] ; then for list_state in "$sdir"/*/ ; do
$ipcmd route flush table "$table" 2>/dev/null [ -d "$list_state" ] || continue
$ipcmd rule del lookup "$table" 2>/dev/null [ -f "$list_state/table" ] || continue
local t ; read -r t < "$list_state/table"
$ipcmd route flush table "$t" 2>/dev/null
$ipcmd rule del lookup "$t" 2>/dev/null
done
rm -rf "$sdir"
fi fi
[ -z "$SHOW" ] && rm -rf "$sdir" continue
fi fi
# Group exists — check for removed .list files
for list_state in "$sdir"/*/ ; do
[ -d "$list_state" ] || continue
local lname=$(basename "$list_state")
if [ ! -f "$routes_dir/$name/${lname}.list" ] ; then
local t=""
[ -f "$list_state/table" ] && read -r t < "$list_state/table"
log "Cleaning orphaned state for $name/$lname (table $t)"
if [ -z "$SHOW" ] && [ -n "$t" ] ; then
$ipcmd route flush table "$t" 2>/dev/null
$ipcmd rule del lookup "$t" 2>/dev/null
fi
[ -z "$SHOW" ] && rm -rf "$list_state"
fi
done
done done
done done
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment