Commit bd2a6117 authored by Vitaly Lipatov's avatar Vitaly Lipatov

route-update: improve volatile detection and add IP validation

Volatile detection: - Remove single-record restriction (count<=1) that missed multi-record domains like youtube.com with 4 rotating AAAA records - Add diff-resolvers check: domain is volatile if local and extra DNS return different IPs (catches cached TTL > threshold cases) - expand_volatile_subnets now also processes domains with accumulated volatile_ips from prior runs, not only current volatile_domains (fixes race where cached TTL > threshold causes empty volatile_domains but IPs were already collected) IP validation: - Validate resolved IPs with python3 ipaddress before ip-batch loading - Filter out invalid entries (e.g. malformed IPv6) with WARNING log - Prevents ip-batch failures from corrupting route tables Co-Authored-By: 's avatarClaude Opus 4.6 <noreply@anthropic.com>
parent c130e4a6
...@@ -163,21 +163,30 @@ detect_volatile_domains() ...@@ -163,21 +163,30 @@ detect_volatile_domains()
[ -s "$domains" ] || { rm -f "$volatile_file" ; return 0 ; } [ -s "$domains" ] || { rm -f "$volatile_file" ; return 0 ; }
# Parallel dig via temp script (avoids quoting issues with xargs) # Parallel dig via temp script (avoids quoting issues with xargs)
# Volatile = low TTL OR different IPs from local vs extra DNS resolver
cat > "$checker" << 'CHECKER' cat > "$checker" << 'CHECKER'
#!/bin/sh #!/bin/sh
rtype="$1"; threshold="$2"; domain="$3" rtype="$1"; threshold="$2"; domain="$3"; extra_dns="$4"
output=$(dig +noall +answer "$domain" "$rtype" 2>/dev/null | grep "IN[[:space:]]*${rtype}[[:space:]]") output=$(dig +noall +answer "$domain" "$rtype" 2>/dev/null | grep "IN[[:space:]]*${rtype}[[:space:]]")
count=$(echo "$output" | grep -c .) count=$(echo "$output" | grep -c .)
ttl=$(echo "$output" | head -1 | awk '{print $2}') ttl=$(echo "$output" | head -1 | awk '{print $2}')
if [ "$count" -le 1 ] && [ "${ttl:-9999}" -le "$threshold" ] ; then reason=""
printf "%s\tttl=%s records=%s\n" "$domain" "$ttl" "$count" if [ "${ttl:-9999}" -le "$threshold" ] ; then
reason="ttl=${ttl}"
elif [ -n "$extra_dns" ] ; then
local_ips=$(echo "$output" | awk '{print $NF}' | sort)
extra_ips=$(dig @$extra_dns +noall +answer "$domain" "$rtype" 2>/dev/null | grep "IN[[:space:]]*${rtype}[[:space:]]" | awk '{print $NF}' | sort)
if [ -n "$extra_ips" ] && [ "$local_ips" != "$extra_ips" ] ; then
reason="diff-resolvers"
fi
fi fi
[ -n "$reason" ] && printf "%s\t%s records=%s\n" "$domain" "$reason" "$count"
CHECKER CHECKER
chmod +x "$checker" chmod +x "$checker"
xargs -P 10 -I{} "$checker" "$rtype" "$ttl_threshold" {} < "$domains" > "$volatile_file" xargs -P 10 -I{} "$checker" "$rtype" "$ttl_threshold" {} "$EXTRA_DNS" < "$domains" > "$volatile_file"
if [ -s "$volatile_file" ] ; then if [ -s "$volatile_file" ] ; then
vlog "[$name]$label volatile domains (TTL≤${ttl_threshold}s, single $rtype): $(wc -l < "$volatile_file")" vlog "[$name]$label volatile domains (TTL≤${ttl_threshold}s or diff resolvers): $(wc -l < "$volatile_file")"
[ -n "$VERBOSE" ] && sed "s|^| [$name]$label |" "$volatile_file" >&2 || true [ -n "$VERBOSE" ] && sed "s|^| [$name]$label |" "$volatile_file" >&2 || true
fi fi
} }
...@@ -193,10 +202,21 @@ expand_volatile_subnets() ...@@ -193,10 +202,21 @@ expand_volatile_subnets()
local vip_dir="$STATE_DIR/$state/volatile_ips" local vip_dir="$STATE_DIR/$state/volatile_ips"
mkdir -p "$vip_dir" mkdir -p "$vip_dir"
# Build combined domain list: volatile_domains + domains with saved volatile_ips
local all_domains=$(mktemp)
[ -s "$volatile_file" ] && awk -F' ' '{print $1}' "$volatile_file" > "$all_domains"
# Also include domains with accumulated IPs from prior runs
for f in "$vip_dir"/* ; do
[ -s "$f" ] && basename "$f" >> "$all_domains"
done 2>/dev/null
sort -u "$all_domains" -o "$all_domains"
[ -s "$all_domains" ] || { rm -f "$all_domains" ; return 0 ; }
# Phase 1: parallel dig for all volatile domains # Phase 1: parallel dig for all volatile domains
local fresh_dir=$(mktemp -d) local fresh_dir=$(mktemp -d)
local digger=$(mktemp) local digger=$(mktemp)
trap "rm -rf $fresh_dir $digger" RETURN trap "rm -rf $fresh_dir $digger $all_domains" RETURN
cat > "$digger" << 'DIGGER' cat > "$digger" << 'DIGGER'
#!/bin/sh #!/bin/sh
domain="$1"; outdir="$2"; extra_dns="$3" domain="$1"; outdir="$2"; extra_dns="$3"
...@@ -205,12 +225,11 @@ dig +short "$domain" AAAA 2>/dev/null | grep ':' > "$out" ...@@ -205,12 +225,11 @@ dig +short "$domain" AAAA 2>/dev/null | grep ':' > "$out"
[ -n "$extra_dns" ] && dig @$extra_dns +short "$domain" AAAA 2>/dev/null | grep ':' >> "$out" || true [ -n "$extra_dns" ] && dig @$extra_dns +short "$domain" AAAA 2>/dev/null | grep ':' >> "$out" || true
DIGGER DIGGER
chmod +x "$digger" chmod +x "$digger"
awk -F' ' '{print $1}' "$volatile_file" | \ xargs -P 10 -I{} "$digger" {} "$fresh_dir" "$EXTRA_DNS" < "$all_domains"
xargs -P 10 -I{} "$digger" {} "$fresh_dir" "$EXTRA_DNS"
# Phase 2: sequential merge + subnet computation # Phase 2: sequential merge + subnet computation
local added=0 local added=0
while IFS=' ' read -r domain _info ; do while read -r domain ; do
[ -z "$domain" ] && continue [ -z "$domain" ] && continue
local ip_file="$vip_dir/$domain" local ip_file="$vip_dir/$domain"
local fresh="$fresh_dir/$domain" local fresh="$fresh_dir/$domain"
...@@ -233,7 +252,7 @@ DIGGER ...@@ -233,7 +252,7 @@ DIGGER
added=$((added + scount)) added=$((added + scount))
vlog "${tag}${label} volatile $domain: $ip_count IPs → $scount subnet(s): $subnets" vlog "${tag}${label} volatile $domain: $ip_count IPs → $scount subnet(s): $subnets"
fi fi
done < "$volatile_file" done < "$all_domains"
rm -rf "$fresh_dir" "$digger" rm -rf "$fresh_dir" "$digger"
[ "$added" -gt 0 ] && log "${tag}${label} Added $added covering subnet(s) for volatile IPv6 domains" || true [ "$added" -gt 0 ] && log "${tag}${label} Added $added covering subnet(s) for volatile IPv6 domains" || true
...@@ -578,7 +597,9 @@ resolve_list_file() ...@@ -578,7 +597,9 @@ resolve_list_file()
[ "$_ipcmd" = "ip -6" ] && _rtype="AAAA" [ "$_ipcmd" = "ip -6" ] && _rtype="AAAA"
detect_volatile_domains "$_state" "$_vname" "$_label" "$_rtype" "$_f" detect_volatile_domains "$_state" "$_vname" "$_label" "$_rtype" "$_f"
# For IPv6 volatile domains: add covering subnets # For IPv6 volatile domains: add covering subnets
if [ "$_ipcmd" = "ip -6" ] && [ -s "$STATE_DIR/$_state/volatile_domains" ] ; then # Also run if volatile_ips accumulated from prior runs (even if current detect found nothing)
local _vip_dir="$STATE_DIR/$_state/volatile_ips"
if [ "$_ipcmd" = "ip -6" ] && { [ -s "$STATE_DIR/$_state/volatile_domains" ] || [ -d "$_vip_dir" ] && ls "$_vip_dir"/* >/dev/null 2>&1 ; } ; then
expand_volatile_subnets "$_state" "$_tag" "$_label" expand_volatile_subnets "$_state" "$_tag" "$_label"
fi fi
fi fi
...@@ -614,6 +635,30 @@ load_list_routes() ...@@ -614,6 +635,30 @@ load_list_routes()
return 1 return 1
fi fi
# Validate resolved IPs before loading (filter out garbage, warn)
local _valid=$(mktemp)
python3 -c '
import sys, ipaddress
for line in sys.stdin:
line = line.strip()
if not line: continue
try:
ipaddress.ip_network(line, strict=False)
print(line)
except ValueError:
print(line, file=sys.stderr)
' < "$_resolved_new" > "$_valid" 2>"$_valid.bad"
local _invalid=$(wc -l < "$_valid.bad")
if [ "$_invalid" -gt 0 ] ; then
log "$_tag$_label WARNING: filtered out $_invalid invalid IPs before loading"
head -5 "$_valid.bad" | while read -r bad ; do
log "$_tag$_label invalid: $bad"
done
fi
rm -f "$_valid.bad"
mv "$_valid" "$_resolved_new"
count=$(wc -l < "$_resolved_new")
# Zero-downtime reload: replace all, then remove stale # Zero-downtime reload: replace all, then remove stale
if [ -n "$_has_metric" ] ; then if [ -n "$_has_metric" ] ; then
{ {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment