Skip to content

E
etersoft-admin-essentials
Commit 385b6521 authored by Vitaly Lipatov's avatar Vitaly Lipatov
Browse files
Options

web-api: unify check — use speed test for both domain and file URL modes

Remove old parallel gateway check (check_site, _check_one, _find_assets, _check_throttle) and dead JS SSE handlers (gateway, throttle events). Both SSE and JSON paths now use the same run_speed_test with identical gateway order and early-exit logic. Co-Authored-By: 's avatarClaude Opus 4.6 <noreply@anthropic.com>
parent e526a3c2
Hide whitespace changes
Inline Side-by-side
Showing with 32 additions and 293 deletions
router/route-web-api.py
View file @ 385b6521
......@@ -56,44 +56,9 @@ CHECK_GATEWAYS = [
("gre.beget.ogw", "socks5://91.232.225.124:1080", None),
("ikev2.beget.ogw", "socks5://91.232.225.130:1080", None),
]
CHECK_TIMEOUT = 10
_list_lock = threading.Lock()
def _check_one(name, proxy, url, ipver="-4"):
"""Check a single gateway, return (name, ipver, status, http_code)."""
try:
cmd = ["curl", ipver]
if proxy:
cmd.extend(["--proxy", proxy])
cmd.extend(["-o", "/dev/null", "-s",
"-w", "%{http_code}", "-m", str(CHECK_TIMEOUT), "-L", url])
result = subprocess.run(
cmd,
capture_output=True, text=True, timeout=CHECK_TIMEOUT + 5,
)
code = int(result.stdout.strip()) if result.stdout.strip() else 0
rc = result.returncode
except subprocess.TimeoutExpired:
return (name, ipver, "PROXY?", 0)
except ValueError:
code = 0
rc = -1
if code == 0:
# curl exit 7=connect refused, 97=SOCKS error → proxy issue
if rc in (7, 97):
status = "PROXY?"
else:
status = "BLOCK"
elif 200 <= code <= 399:
status = "OK"
else:
status = "HTTP%d" % code
return (name, ipver, status, code)
def resolve_domain(domain):
"""Resolve domain to list of unique IP addresses."""
ips = set()
......@@ -200,79 +165,6 @@ def get_whois(domain):
return []
THROTTLE_TIMEOUT = 8
THROTTLE_SIZE_LIMIT = 32768 # 32KB — above typical TCP initial window
def _find_assets(proxy, base_url):
"""Download HTML page via proxy and extract CSS/JS asset URLs."""
try:
cmd = ["curl", "-4"]
if proxy:
cmd.extend(["--proxy", proxy])
cmd.extend(["-s", "-m", "10", "-L", base_url])
result = subprocess.run(
cmd,
capture_output=True, timeout=15,
)
if result.returncode != 0:
return []
page = result.stdout.decode("utf-8", errors="ignore")
except (subprocess.TimeoutExpired, OSError):
return []
if len(result.stdout) >= THROTTLE_SIZE_LIMIT:
return [] # page itself is large, no need to check assets
parsed_url = urllib.parse.urlparse(base_url)
origin = "%s://%s" % (parsed_url.scheme, parsed_url.netloc)
assets = []
for m in re.finditer(
r'(?:href|src)=["\']([^"\']*\.(?:css|js)(?:\?[^"\']*)?)["\']', page
):
href = m.group(1)
if href.startswith("//"):
href = "https:" + href
elif href.startswith("/"):
href = origin + href
elif not href.startswith("http"):
href = base_url.rstrip("/") + "/" + href
assets.append(href)
return assets[:3]
def _check_throttle(name, proxy, asset_url):
"""Download a single asset via proxy and detect throttling.
Returns (name, result_dict) or (name, None).
"""
try:
cmd = ["curl", "-4"]
if proxy:
cmd.extend(["--proxy", proxy])
cmd.extend(["-o", "/dev/null", "-s",
"-w", "%{size_download} %{time_total}",
"-m", str(THROTTLE_TIMEOUT), "-L", asset_url])
result = subprocess.run(
cmd,
capture_output=True, text=True, timeout=THROTTLE_TIMEOUT + 5,
)
parts = result.stdout.strip().split()
if len(parts) < 2:
return (name, None)
size = int(float(parts[0]))
elapsed = float(parts[1])
fname = asset_url.split("/")[-1].split("?")[0]
if 0 < size < THROTTLE_SIZE_LIMIT and elapsed >= THROTTLE_TIMEOUT - 1:
return (name, {"throttled": True, "asset": fname,
"size": size, "time": round(elapsed, 1)})
if size >= THROTTLE_SIZE_LIMIT:
return (name, {"throttled": False})
except (subprocess.TimeoutExpired, OSError, ValueError):
pass
return (name, None)
SPEED_CHECK_TIMEOUT = 5
# Order for speed test: likely-slow first, stop after two successes in a row
......@@ -321,87 +213,6 @@ def _check_speed(name, proxy, file_url):
return (name, {"error": str(e)})
def check_site(domain, file_url=None):
"""Check domain: resolve IPs, find in route lists, whois, test gateways.
When file_url is given, skip gateway HTTP checks — speed test replaces them.
"""
url = "https://%s/" % domain
checks = {}
throttle = {}
whois_info = []
if file_url:
# File URL mode: only resolve, routes, whois (speed test is streamed separately)
whois_info = get_whois(domain)
else:
# Domain mode: full gateway checks + throttle detection
has_v6 = False
try:
socket.getaddrinfo(domain, None, socket.AF_INET6, socket.SOCK_STREAM)
has_v6 = True
except socket.gaierror:
pass
workers = len(CHECK_GATEWAYS) * 2 + 2
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
gw_futures = []
for name, proxy_v4, proxy_v6 in CHECK_GATEWAYS:
gw_futures.append(pool.submit(_check_one, name, proxy_v4, url, "-4"))
if has_v6 and proxy_v6:
gw_futures.append(pool.submit(_check_one, name, proxy_v6, url, "-6"))
whois_future = pool.submit(get_whois, domain)
assets_future = pool.submit(_find_assets, CHECK_GATEWAYS[0][1], url)
for future in concurrent.futures.as_completed(gw_futures):
name, ipver, status, code = future.result()
if name not in checks:
checks[name] = {}
if ipver == "-4":
checks[name]["status"] = status
checks[name]["http_code"] = code
else:
checks[name]["status_v6"] = status
checks[name]["http_code_v6"] = code
for name, _pv4, _pv6 in CHECK_GATEWAYS:
if name not in checks:
checks[name] = {}
if "status_v6" not in checks[name]:
checks[name]["status_v6"] = "NOIP"
checks[name]["http_code_v6"] = 0
whois_info = whois_future.result()
assets = assets_future.result()
if assets:
asset_url = assets[0]
with concurrent.futures.ThreadPoolExecutor(max_workers=len(CHECK_GATEWAYS)) as pool:
throttle_futures = [
pool.submit(_check_throttle, name, proxy, asset_url)
for name, proxy, _pv6 in CHECK_GATEWAYS
]
for future in concurrent.futures.as_completed(throttle_futures):
gw_name, result = future.result()
if result:
throttle[gw_name] = result
ips = resolve_domain(domain)
routes = find_in_routes(domain, ips)
result = {
"domain": domain, "ips": ips, "routes": routes,
"whois": whois_info,
}
if checks:
result["checks"] = checks
if file_url:
result["file_url"] = file_url
if throttle:
result["throttle"] = throttle
return result
def run_speed_test(file_url, callback):
"""Run sequential speed test, calling callback(gw_name, result) for each.
......@@ -631,7 +442,7 @@ OPENAPI_SPEC = {
"/api/check": {
"post": {
"summary": "Проверка блокировки",
"description": "Проверяет доступность домена/IP напрямую (policy routing) и через все шлюзы параллельно (curl через SOCKS5).",
"description": "Проверяет скорость доступа к домену/URL через шлюзы последовательно (curl через SOCKS5). SSE при Accept: text/event-stream, иначе JSON.",
"requestBody": {"required": True, "content": {"application/json": {"schema": {
"type": "object",
"required": ["domain"],
......@@ -648,7 +459,6 @@ OPENAPI_SPEC = {
},
}}}},
"400": {"description": "Некорректный домен"},
"429": {"description": "Проверка уже выполняется"},
},
}
},
......@@ -1190,7 +1000,6 @@ async function checkDomain() {
// Two parallel requests: SSE for gateways, JSON for resolve
const resolveP = api('resolve', {domain: v}).then(renderResolve).catch(() => {});
const gwResults = {};
const speedResults = {};
try {
const resp = await fetch('/api/check', {
......@@ -1217,33 +1026,7 @@ async function checkDomain() {
if (!evData) continue;
const parsed = JSON.parse(evData);
if (evType === 'check') {
// file_url mode: update header
if (parsed.file_url) $('check-domain').textContent = parsed.file_url;
} else if (evType === 'gateway') {
const n = parsed.name;
if (!gwResults[n]) gwResults[n] = {};
if (parsed.ipver === '-4') {
gwResults[n].status = parsed.status;
gwResults[n].http_code = parsed.http_code;
} else {
gwResults[n].status_v6 = parsed.status;
gwResults[n].http_code_v6 = parsed.http_code;
}
let el = document.getElementById('gw-' + n);
if (!el) {
el = document.createElement('span');
el.id = 'gw-' + n;
gwWrap.appendChild(el);
}
const info = gwResults[n];
const st = info.status || '\\u2026';
const cls = st === 'OK' ? 'ok' : st === 'BLOCK' ? 'block' : st === 'PROXY?' ? 'proxy' : 'other';
el.className = 'check-gw ' + cls;
let txt = n + ': ' + st;
if (info.status_v6 && info.status_v6 !== 'PROXY?' && info.status_v6 !== 'NOIP') {
txt += ' v6: ' + info.status_v6;
}
el.textContent = txt;
} else if (evType === 'speed') {
const gw = parsed.gateway;
const info = parsed.result;
......@@ -1253,28 +1036,18 @@ async function checkDomain() {
el.className = info.error ? 'check-gw proxy' : 'check-gw other';
el.textContent = gw + ': ' + (info.error || info.speed_str);
gwWrap.appendChild(el);
} else if (evType === 'throttle') {
const el = document.getElementById('gw-' + parsed.gateway);
if (el && el.classList.contains('ok')) {
const r = parsed.result;
el.className = 'check-gw slow';
el.textContent = el.textContent.split(':')[0] + ': SLOW';
el.title = r.asset + ': ' + r.size + ' \\u0431\\u0430\\u0439\\u0442 \\u0437\\u0430 ' + r.time + '\\u0441';
}
} else if (evType === 'done') {
if (Object.keys(speedResults).length) {
let maxSpd = 0;
for (const r of Object.values(speedResults)) { if (r.speed > maxSpd) maxSpd = r.speed; }
for (const [gw, r] of Object.entries(speedResults)) {
const el = document.getElementById('gw-' + gw);
if (!el || r.error) continue;
if (maxSpd > 0 && r.speed < maxSpd / 10) {
el.className = 'check-gw slow';
el.textContent = gw + ': SLOW (' + r.speed_str + ')';
} else {
el.className = 'check-gw ok';
el.textContent = gw + ': OK (' + r.speed_str + ')';
}
let maxSpd = 0;
for (const r of Object.values(speedResults)) { if (r.speed > maxSpd) maxSpd = r.speed; }
for (const [gw, r] of Object.entries(speedResults)) {
const el = document.getElementById('gw-' + gw);
if (!el || r.error) continue;
if (maxSpd > 0 && r.speed < maxSpd / 10) {
el.className = 'check-gw slow';
el.textContent = gw + ': SLOW (' + r.speed_str + ')';
} else {
el.className = 'check-gw ok';
el.textContent = gw + ': OK (' + r.speed_str + ')';
}
}
}
......@@ -1755,65 +1528,31 @@ class RouteHandler(http.server.BaseHTTPRequestHandler):
base["file_url"] = file_url
self.send_sse("check", base)
if file_url:
# File URL mode: speed test only
def on_speed(gw_name, res):
self.send_sse("speed", {"gateway": gw_name, "result": res})
run_speed_test(file_url, on_speed)
else:
# Domain mode: gateways + throttle
url = "https://%s/" % domain
has_v6 = False
try:
socket.getaddrinfo(domain, None, socket.AF_INET6, socket.SOCK_STREAM)
has_v6 = True
except socket.gaierror:
pass
with concurrent.futures.ThreadPoolExecutor(max_workers=len(CHECK_GATEWAYS) * 2 + 2) as pool:
gw_futures = {}
for name, proxy_v4, proxy_v6 in CHECK_GATEWAYS:
f4 = pool.submit(_check_one, name, proxy_v4, url, "-4")
gw_futures[f4] = True
if has_v6 and proxy_v6:
f6 = pool.submit(_check_one, name, proxy_v6, url, "-6")
gw_futures[f6] = True
assets_future = pool.submit(_find_assets, CHECK_GATEWAYS[0][1], url)
for future in concurrent.futures.as_completed(gw_futures):
name, ipver, status, code = future.result()
self.send_sse("gateway", {
"name": name, "ipver": ipver,
"status": status, "http_code": code,
})
assets = assets_future.result()
# Phase 2: throttle detection
if assets:
asset_url = assets[0]
with concurrent.futures.ThreadPoolExecutor(max_workers=len(CHECK_GATEWAYS)) as pool:
throttle_futures = [
pool.submit(_check_throttle, name, proxy, asset_url)
for name, proxy, _pv6 in CHECK_GATEWAYS
]
for future in concurrent.futures.as_completed(throttle_futures):
gw_name, result = future.result()
if result and result.get("throttled"):
self.send_sse("throttle", {
"gateway": gw_name, "result": result,
})
# Unified speed test: same order, same logic for both modes
test_url = file_url if file_url else "https://%s/" % domain
def on_speed(gw_name, res):
self.send_sse("speed", {"gateway": gw_name, "result": res})
run_speed_test(test_url, on_speed)
self.send_sse("done", {})
else:
# JSON response (curl/scripts)
result = check_site(domain, file_url=file_url)
test_url = file_url if file_url else "https://%s/" % domain
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
whois_future = pool.submit(get_whois, domain)
ips = resolve_domain(domain)
routes = find_in_routes(domain, ips)
whois_info = whois_future.result()
speed = {}
def on_speed(gw_name, res):
speed[gw_name] = res
run_speed_test(test_url, on_speed)
result = {
"domain": domain, "ips": ips, "routes": routes,
"whois": whois_info, "speed": speed,
}
if file_url:
speed = {}
def on_speed(gw_name, res):
speed[gw_name] = res
run_speed_test(file_url, on_speed)
result["speed"] = speed
result["file_url"] = file_url
self.send_json(result)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment