router: use adnshost for bulk async DNS resolution (22x speedup)

router/functions

+
+get_ipv4_list()
+{
+    echo "$1" | grep -q "[a-z]" || return 0
+    dig "$1" A | grep -v "^;" | grep "IN[[:space:]]*A[[:space:]]" | sed -e "s|.*[[:space:]]||" | sort
+    dig @8.8.8.8 "$1" A | grep -v "^;" | grep "IN[[:space:]]*A[[:space:]]" | sed -e "s|.*[[:space:]]||" | sort
+}
+
+get_ipv6_list()
+{
+    echo "$1" | grep -q "[a-z]" || return 0
+    dig "$1" AAAA | grep -v "^;" | grep "IN[[:space:]]*AAAA[[:space:]]" | sed -e "s|.*[[:space:]]||" | sort
+    dig @8.8.8.8 "$1" AAAA | grep -v "^;" | grep "IN[[:space:]]*AAAA[[:space:]]" | sed -e "s|.*[[:space:]]||" | sort
+}
+
+is_ipv4()
+{
+    # https://disnetern.ru/search-ip/
+    echo "$1" | grep -q -E "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
+}
+
+## Несколько диапазонов в одной строке
+#  $ expand_range "r[1-2][a-b].test"
+#  r1a.test
+#  r1b.test
+#  r2a.test
+#  r2b.test
+  
+expand_range() {
+      local input="$1"
+      # Ищем паттерн [X-Y] где X и Y - цифры или буквы
+      if [[ $input =~ ^(.*)\[([[:alnum:]])-([[:alnum:]])\](.*)$ ]]; then
+          local prefix="${BASH_REMATCH[1]}"
+          local start="${BASH_REMATCH[2]}"
+          local end="${BASH_REMATCH[3]}"
+          local suffix="${BASH_REMATCH[4]}"
+
+          # Рекурсия для обработки нескольких диапазонов
+          for char in $(eval echo "{$start..$end}"); do
+              expand_range "${prefix}${char}${suffix}"
+          done
+      else
+          echo "$input"
+      fi
+}
+
+cat_expanded() {
+      local file="$1"
+      while IFS= read -r line || [[ -n "$line" ]]; do
+          [[ -z "$line" || "$line" == \#* ]] && continue
+          expand_range "$line"
+      done < "$file"
+}
+
+# Bulk resolve domains to IPv4 via adnshost (async) with dig fallback for CNAMEs
+# Usage: get_ipv4_list_bulk < domains.txt
+#    or: get_ipv4_list_bulk domains.txt
+get_ipv4_list_bulk()
+{
+    # read from file arg or stdin
+    local domains=$(mktemp)
+    local adns_out=$(mktemp)
+    trap "rm -f $domains $adns_out" RETURN
+
+    # Read domains, skip IPs (output them directly)
+    while read -r entry ; do
+        [ -z "$entry" ] && continue
+        if is_ipv4 "$entry" ; then
+            echo "$entry"
+        else
+            echo "$entry" >> "$domains"
+        fi
+    done < "${1:-/dev/stdin}"
+
+    [ -s "$domains" ] || return 0
+
+    # Bulk async resolve: local resolver + Google DNS
+    # -a: async, -Fi: inline parseable (answers+errors to stdout), -f: read from stdin
+    {
+        adnshost -a -t a -Fi -f < "$domains"
+        adnshost -a -t a -Fi -f --config "nameserver 8.8.8.8" < "$domains"
+    } > "$adns_out" 2>/dev/null
+
+    # Output resolved IPs grouped by domain
+    awk '/ A / { print $1, $3 }' "$adns_out" | sort -u | awk '{
+        if (domain != $1) { print ""; print "# " $1; domain = $1 }
+        print $2
+    }'
+
+    # Fallback to dig for CNAME/tempfail domains
+    grep "^;" "$adns_out" | grep -v 'nxdomain\|nodata\|querydomaintoolong' | \
+        awk '{ for(i=4;i<=NF;i++) if($i ~ /^[a-z0-9].*\.[a-z]/) { print $i; break } }' | \
+        sort -u | while read domain ; do
+            echo
+            echo "# $domain (dig fallback)"
+            get_ipv4_list "$domain"
+        done
+}

router/get_ips_z-i.sh

@@ -4,5 +4,44 @@
 #cat dump.csv |cut -f1 -d';'|tr '|' '\n'| tr -d ' '|sort -u
 #cat dump.csv |cut -f1 -d';'|tr '|' '\n'| tr -d ' '|sed '1d'|sort -u
 #curl --silent https://reestr.rublacklist.net/api/ips | sed -e 's|"||g' | sed -e 's|;|\n|g'
-curl --silent https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv | cut -d";" -f1 | tr '|' '\n' | tr -d ' ' | grep -v Updated
-cat a_manual.list | grep -v "^#"
+# https://antifilter.download/
+# Нужно переделать на маршруты через Северен? Как-то продетектить, какие адреса через него доступны, какие нет?
+
+curl --silent https://community.antifilter.download/list/community.lst || exit 1
+#curl --silent https://antifilter.download/list/allyouneed.lst
+curl --silent https://antifilter.download/list/ipresolve.lst || exit 1
+curl --silent https://antifilter.download/list/subnet.lst || exit 1
+
+#curl --silent https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv | cut -d";" -f1 | tr '|' '\n' | tr -d ' ' | grep -v Updated
+#cat a_manual.list | grep -v "^#"
+
+# https://reestr.rublacklist.net/api/v3/ips/
+# https://reestr.rublacklist.net/api/v3/dpi/
+
+. ./functions
+
+
+get_domains()
+{
+    #curl --silent https://gitlab.eterfund.ru/eterfund/egw-route/raw/master/ogw.list || exit 1
+    #cat a_ogw_manual.list
+    ./fetch_lists.sh >/dev/null
+    cat_expanded /root/egw-route/ogw.list
+    cat_expanded /root/egw-route/youtube.list
+# TODO: надо кэшировать
+    cat_expanded /root/egw-route/whatsapp.list
+    cat_expanded /root/egw-route/telegram.list
+}
+
+get_domains | grep -v "^#" | get_ipv4_list_bulk
+
+# Old slow method (sequential dig per domain):
+#for domain in $(get_domains | grep -v "^#") ; do
+#    echo
+#    if is_ipv4 $domain ; then
+#        echo "$domain"
+#        continue
+#    fi
+#    echo "# $domain"
+#    get_ipv4_list $domain
+#done