From 84c3dffbce36e72519f4cbfa31caaae0853be546 Mon Sep 17 00:00:00 2001 From: Andrew Pimenov Date: Wed, 12 Aug 2026 21:36:54 +0300 Subject: [PATCH] detect captive portals up front, flag off-country speedtest servers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New Phase 0 in diagnose(): three generate_204-style probes run before every other module, using detect-and-tag rather than short-circuit — a portal is rarely total (DNS/ICMP often pass while only HTTP is intercepted), and an aborted run would look identical to a dead link. On detection every later phase still runs, tagged with a crit finding and a banner at the top of the report. Caught live before it shipped: detectportal.firefox.com/success.txt (one of the original three default check URLs) returns 200 "success" on a genuinely clean connection, which the generate_204-shaped classifier misread as a portal redirect. Swapped for a second real generate_204 endpoint instead of teaching the classifier a second success shape. Also redesigned the speedtest server geo-sanity check after checking the bundled Ookla CLI's actual JSON output: its server object never carries lat/lon (documented schema, not a version gap), so speed.server_other_country compares server_country against the client's own country instead of a haversine distance that no tier can actually supply. Part of plans/zesty-zooming-wilkinson.md (gitignored). --- ARCHITECTURE.md | 9 ++- ARCHITECTURE.ru.md | 9 ++- README.md | 3 +- README.ru.md | 3 +- config.yaml | 11 ++++ src/netsleuth/cli.py | 35 +++++++++- src/netsleuth/config.py | 20 ++++++ src/netsleuth/exporter.py | 46 ++++++++++++- src/netsleuth/interpret.py | 52 ++++++++++++++- src/netsleuth/models.py | 28 ++++++++ src/netsleuth/probes/captive_portal.py | 73 +++++++++++++++++++++ src/netsleuth/speed.py | 3 + tests/test_captive_portal.py | 89 ++++++++++++++++++++++++++ tests/test_config.py | 12 ++++ tests/test_exporter.py | 5 +- tests/test_interpret.py | 18 ++++++ tests/test_speed_fastcom.py | 1 + 17 files changed, 406 insertions(+), 11 deletions(-) create mode 100644 src/netsleuth/probes/captive_portal.py create mode 100644 tests/test_captive_portal.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e9740b0..6582638 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -37,6 +37,7 @@ src/netsleuth/ dpi_check.py --my-server: single-host TCP RST/filtering self-diagnostic. Not a scanner — see Key decisions. nat64.py One RFC 7050 AAAA query (ipv4only.arpa) to detect NAT64/464XLAT translation. hop_asn.py Per-hop ASN/AS-name/country/PTR enrichment via Team Cymru DNS, in-run deduped. + captive_portal.py Phase-0 gate: three generate_204-style probes for a public-wifi login wall. ``` **File-size exceptions.** The ~200-line guideline (Global Constraints) held for most modules, but these grew past it during implementation and were kept as single files rather than split, because each one is a single cohesive responsibility that splitting would only scatter across more files with a thinner reason to draw the line anywhere in particular: @@ -134,6 +135,12 @@ The AS prefix benchmark (`probes/prefix_benchmark.py`) gets the same treatment f **Per-hop ASN enrichment is a separate phase after the trace completes, not folded into traceroute itself.** `TraceHop.asn`/`as_name` existed in the model since the original design but were never populated by any writer — `probes/hop_asn.py::enrich_hops()` fills them (plus a new `country` field and a PTR fallback for backends that don't resolve hostnames themselves) by collecting the unique set of hop IPs across every trace and firing one Team Cymru DNS query pair per IP, deduped in-run. This runs as its own `run_module("path_enrich", ...)` step in `cli.py` after the latency/path/TLS fan-out and before speed, and mutates the already-collected `TraceHop` objects in place; a lookup failure folds into that step's warnings only; it can never downgrade the trace's own `ModuleResult.status`. Team Cymru's DNS service has no rate limit (unlike the `cached_json()`-wrapped RIPEstat/CAIDA calls), so no disk cache is needed — only in-run dedup of repeated hop IPs. +**Captive portal detection tags, never short-circuits.** `ModuleResult.status` is informational everywhere else in the pipeline, and nothing in `diagnose()` branches on it — introducing the codebase's first status-driven control flow for this one feature would contradict "failure is a value, not control flow" above. A captive portal is also rarely total: DNS and ICMP often pass while only HTTP is intercepted, and a short-circuited run would produce a report indistinguishable from a dead link. So `probes/captive_portal.py` runs as Phase 0, before everything else, and on detection adds a `crit` finding plus a banner at the top of the report — every other phase still runs and is honestly labeled as measured from behind the portal, which is more information than refusing to measure at all. + +**The three default captive-portal check URLs must share one success shape, and picking them is not interchangeable.** `check_captive_portal()` expects every URL to behave like a `generate_204` endpoint: HTTP 204 with an empty body on success. Mozilla's `detectportal.firefox.com/success.txt` was tried as a third default and immediately produced a false positive on a real network — its *correct*, non-portal response is HTTP 200 with the body `success`, which is indistinguishable from `classify_portal_response()`'s "200 with a body" portal signal. Caught by the same live-smoke-test discipline as the international-loop bug above, not by a unit test (the unit tests fed the function literal response shapes and never modeled a real endpoint's actual behavior). Fixed by swapping in a second genuine `generate_204` endpoint (`clients3.google.com`) instead of teaching the classifier a second success shape — one uniform contract across all check URLs is simpler than a per-URL expected-body table for three entries. + +**Speedtest server geo-sanity compares countries, not coordinates.** The original design called for a haversine distance in km between the client and the speedtest server, which needs both ends' lat/lon. Live inspection of the bundled Ookla CLI's own JSON output (`speedtest.exe --format=json`) showed its `server` object carries `location` (a city name) and `country`, never `lat`/`lon` — that's the documented schema, not a version quirk. `speed_findings()` compares `SpeedResult.server_country` against the client's own geolocated country instead: coarser than a distance, but real, always available when a tier reports a country, and it still catches the case that matters (the test hit a server in another country entirely). + **Python 3.14 is the floor, not 3.10.** The project moved off `>=3.10` when this batch of probes was added; none of the new code actually depends on a 3.11+-only construct (`ssl.SSLObject.start_tls()`, `asyncio.timeout()`, `datetime.UTC`) — the two-sequential-connection TLS design above was chosen independently of what the interpreter allows. The version floor moved because there was no longer a reason to hold it back, not because a specific new feature required it. ## Storage @@ -152,4 +159,4 @@ Five sections back the L4-L7 probes, all opt-in at the CLI level and all with a ## Tests -`uv run pytest -q`. Roughly 740 tests covering the parsers, normalizers, verdict engine, DNSBL decoding, throughput math, ping statistics, jitter percentiles, serialization and the report diff. Glue is deliberately untested: live HTTP, real ICMP, Typer wiring, `psutil`/`ctypes` calls and the `--watch` timing loop. Traceroute fixtures live in `tests/fixtures/traceroute/{windows,linux,darwin}/` and include a cp866-encoded Russian Windows sample, because that encoding is exactly what breaks naive parsers. DoH response fixtures (RFC 8484 JSON form) live in `tests/fixtures/api/doh_*.json` alongside the other provider fixtures. +`uv run pytest -q`. Roughly 758 tests covering the parsers, normalizers, verdict engine, DNSBL decoding, throughput math, ping statistics, jitter percentiles, serialization and the report diff. Glue is deliberately untested: live HTTP, real ICMP, Typer wiring, `psutil`/`ctypes` calls and the `--watch` timing loop. Traceroute fixtures live in `tests/fixtures/traceroute/{windows,linux,darwin}/` and include a cp866-encoded Russian Windows sample, because that encoding is exactly what breaks naive parsers. DoH response fixtures (RFC 8484 JSON form) live in `tests/fixtures/api/doh_*.json` alongside the other provider fixtures. diff --git a/ARCHITECTURE.ru.md b/ARCHITECTURE.ru.md index 08a24c6..d49de60 100644 --- a/ARCHITECTURE.ru.md +++ b/ARCHITECTURE.ru.md @@ -37,6 +37,7 @@ src/netsleuth/ dpi_check.py --my-server: одиночная самопроверка TCP RST/фильтрации. Не сканер — см. Ключевые решения. nat64.py Один AAAA-запрос по RFC 7050 (ipv4only.arpa) для детекта трансляции NAT64/464XLAT. hop_asn.py Обогащение хопов ASN/именем AS/страной/PTR через DNS Team Cymru, дедуп в рамках прогона. + captive_portal.py Фаза 0: три generate_204-пробы на признаки логин-стены публичного wifi. ``` **Исключения по размеру файла.** Лимит ~200 строк (Общие ограничения) выдержан для большинства модулей, но эти выросли за него в процессе реализации и остались одним файлом, а не были разбиты, потому что каждый — единая цельная ответственность, и разбиение только раскидало бы её по файлам без чёткой причины проводить границу где-то конкретно: @@ -134,6 +135,12 @@ src/netsleuth/ **Обогащение хопов по ASN — отдельная фаза после завершения трассировки, а не часть самого traceroute.** Поля `TraceHop.asn`/`as_name` существовали в модели с самого начала, но ни один писатель их не заполнял — `probes/hop_asn.py::enrich_hops()` заполняет их (плюс новое поле `country` и PTR-фоллбэк для бэкендов, которые сами не резолвят имена), собирая уникальный набор IP хопов по всем трассировкам и делая по одной паре DNS-запросов Team Cymru на IP, с дедупом в рамках прогона. Это отдельный шаг `run_module("path_enrich", ...)` в `cli.py` после веера latency/path/TLS и перед speed, мутирующий уже собранные объекты `TraceHop` на месте; ошибка поиска попадает только в предупреждения этого шага и никогда не понижает `ModuleResult.status` самой трассировки. У DNS-сервиса Team Cymru нет лимита запросов (в отличие от обёрнутых в `cached_json()` вызовов RIPEstat/CAIDA), поэтому дисковый кэш не нужен — только дедуп повторяющихся IP хопов в рамках прогона. +**Детект captive portal размечает, но никогда не обрывает прогон.** `ModuleResult.status` — информационное поле везде в конвейере, и ничто в `diagnose()` не ветвится по нему — введение первого статус-управляемого control flow ради одной фичи противоречило бы принципу «отказ — это значение, а не поток управления» выше. К тому же captive portal редко тотален: DNS и ICMP часто проходят, пока перехватывается только HTTP, и оборванный прогон дал бы отчёт, неотличимый от мёртвой ссылки. Поэтому `probes/captive_portal.py` работает как Фаза 0, до всего остального, и при обнаружении добавляет находку `crit` плюс баннер в начале отчёта — все остальные фазы всё равно выполняются и честно помечены как измеренные из-за портала, а это больше информации, чем отказ измерять вовсе. + +**Три URL captive-portal-проверки по умолчанию обязаны иметь одну и ту же форму успеха, и подобрать их — не взаимозаменяемая мелочь.** `check_captive_portal()` ожидает, что каждый URL ведёт себя как `generate_204`-эндпоинт: HTTP 204 с пустым телом при успехе. `detectportal.firefox.com/success.txt` Mozilla пробовался как третий дефолт и сразу дал ложное срабатывание на реальной сети — его *корректный*, не-портальный ответ — HTTP 200 с телом `success`, что неотличимо от сигнала «200 с телом» в `classify_portal_response()`, означающего портал. Поймано той же дисциплиной живого смоук-теста, что и баг с международной петлёй выше, а не юнит-тестом (юнит-тесты подавали функции буквальные формы ответа и никогда не моделировали реальное поведение конкретного эндпоинта). Исправлено заменой на второй настоящий `generate_204`-эндпоинт (`clients3.google.com`) вместо обучения классификатора второй форме успеха — единый контракт на все URL проще, чем таблица ожидаемых тел по каждому URL ради трёх записей. + +**Geo-sanity сервера speedtest сравнивает страны, а не координаты.** Изначальный дизайн предполагал расстояние по haversine в км между клиентом и сервером speedtest, для чего нужны lat/lon обеих сторон. Живая проверка собственного JSON-вывода бандл-бинарника Ookla CLI (`speedtest.exe --format=json`) показала, что его объект `server` несёт `location` (название города) и `country`, но никогда `lat`/`lon` — это документированная схема, а не особенность версии. `speed_findings()` вместо этого сравнивает `SpeedResult.server_country` с собственной геолоцированной страной клиента: грубее расстояния, но реально, всегда доступно, когда уровень каскада сообщает страну, и всё равно ловит именно тот случай, что важен — тест попал на сервер совсем в другой стране. + **Python 3.14 — минимальная версия, не 3.10.** Проект ушёл с `>=3.10` одновременно с этим набором проб; ни один новый код на деле не зависит от конструкций только для 3.11+ (`ssl.SSLObject.start_tls()`, `asyncio.timeout()`, `datetime.UTC`) — схема TLS с двумя последовательными соединениями выше выбрана независимо от того, что позволяет интерпретатор. Планка версии сдвинулась потому, что не осталось причин её удерживать, а не потому, что это потребовалось для конкретной новой функции. ## Хранение @@ -152,4 +159,4 @@ src/netsleuth/ ## Тесты -`uv run pytest -q`. Около 740 тестов: парсеры, нормализаторы, движок вердиктов, декодирование DNSBL, математика пропускной способности, статистика пингов, перцентили джиттера, сериализация и сравнение отчётов. Клей намеренно не тестируется: живой HTTP, настоящий ICMP, обвязка Typer, вызовы `psutil`/`ctypes` и цикл `--watch`. Фикстуры трассировок лежат в `tests/fixtures/traceroute/{windows,linux,darwin}/` и включают образец русской Windows в cp866 — именно эта кодировка ломает наивные парсеры. Фикстуры DoH-ответов (формат RFC 8484 JSON) лежат в `tests/fixtures/api/doh_*.json` рядом с фикстурами других провайдеров. +`uv run pytest -q`. Около 758 тестов: парсеры, нормализаторы, движок вердиктов, декодирование DNSBL, математика пропускной способности, статистика пингов, перцентили джиттера, сериализация и сравнение отчётов. Клей намеренно не тестируется: живой HTTP, настоящий ICMP, обвязка Typer, вызовы `psutil`/`ctypes` и цикл `--watch`. Фикстуры трассировок лежат в `tests/fixtures/traceroute/{windows,linux,darwin}/` и включают образец русской Windows в cp866 — именно эта кодировка ломает наивные парсеры. Фикстуры DoH-ответов (формат RFC 8484 JSON) лежат в `tests/fixtures/api/doh_*.json` рядом с фикстурами других провайдеров. diff --git a/README.md b/README.md index 2bcd430..39160cd 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,14 @@ None of this is required — netsleuth degrades gracefully without any of it — ## What it does +- **Captive portal check** — runs first, before anything else: three `generate_204`-style probes catch a public-wifi login wall before it makes every other measurement look like a broken connection. - **Identity** — egress IPv4 *and* IPv6, reverse DNS, ASN, organisation, geo, and whether the address looks residential, mobile, hosting or business. Cross-checked across six independent providers so one rate limit does not blank the answer. Also flags carrier-grade NAT (RFC 6598) and NAT64/464XLAT translation (RFC 7050) when present. - **VPN / proxy assessment** — a weighted verdict (`none` / `likely` / `confirmed`) built from tunnel interfaces, MTU anomalies, gateway-vs-egress mismatch, Cloudflare WARP flags, provider proxy/hosting flags, PeeringDB network type, timezone-vs-geo mismatch, and DNS resolver ASN mismatch. - **DNS leak test** — enumerates resolvers *per network adapter*, so it catches the classic case where the tunnel adapter is clean but the Wi-Fi adapter still holds the ISP resolver. Also detects EDNS Client Subnet leakage. - **BGP intelligence** — upstreams, peers, downstreams, announced prefixes, route stability, IXP presence, and CAIDA ASRank customer-cone size. - **Reputation** — locally cached FireHOL blocklists (no query leaves your machine), Shodan InternetDB exposure, and optional classic DNSBL zones. - **Latency & path** — per-host ping statistics with real jitter/loss/mdev/p95/p99, and a traceroute that never needs administrator rights. The first hop is diagnosed separately from the rest of the route, so loss or slowness there is correctly attributed to your own router/Wi-Fi rather than the ISP. Every hop is annotated with its ASN, AS name and country, and flags a route that detours through a third country on the way to the destination. -- **Bandwidth** — a cascading speedtest (Ookla binary → Cloudflare → fast.com → optional M-Lab NDT7) with separate download and upload bufferbloat grades. +- **Bandwidth** — a cascading speedtest (Ookla binary → Cloudflare → fast.com → optional M-Lab NDT7) with separate download and upload bufferbloat grades, and a flag when the test server landed in a different country than you. - **TLS handshake RTT** (`--tls`) — TCP RTT, TLS 1.3 handshake time and time-to-first-byte to reference services, flagging servers whose handshake is disproportionately slow relative to the network (CPU-bound TLS termination, not a network problem). - **DNS: system vs DoH** (`--dns-advanced`) — compares your system resolver's answers and latency against Cloudflare/Google/Quad9 DoH, and probes a bogus resolver IP to detect a transparent DNS proxy intercepting port 53. - **Path diversity / Anycast** (`--path-diversity`) — reads Cloudflare's `CF-RAY` edge code to find which PoP is actually serving your traffic, and flags an international routing loop when it's in a different country than you are. diff --git a/README.ru.md b/README.ru.md index 4c0d151..e71ceb2 100644 --- a/README.ru.md +++ b/README.ru.md @@ -57,13 +57,14 @@ netsleuth ## Что он делает +- **Проверка captive portal** — выполняется первой, до всего остального: три пробы в стиле `generate_204` ловят логин-стену публичного wifi до того, как она сделает все остальные замеры похожими на сломанное соединение. - **Идентификация** — внешний IPv4 *и* IPv6, обратный DNS, ASN, организация, гео и тип адреса: домашний, мобильный, хостинговый или корпоративный. Проверяется по шести независимым источникам, поэтому один rate-limit не обнуляет результат. Также определяет carrier-grade NAT (RFC 6598) и трансляцию NAT64/464XLAT (RFC 7050), если они есть. - **Оценка VPN/прокси** — взвешенный вердикт (`none` / `likely` / `confirmed`) из наличия туннельного интерфейса, аномалии MTU, расхождения шлюза и внешнего IP, флагов Cloudflare WARP, флагов proxy/hosting от провайдеров данных, типа сети в PeeringDB, расхождения таймзоны ОС и гео, а также несовпадения ASN DNS-резолвера. - **Тест утечки DNS** — перечисляет резолверы *по каждому сетевому адаптеру*, поэтому ловит классический случай, когда туннельный адаптер чистый, а на Wi-Fi-адаптере остался резолвер провайдера. Также определяет утечку EDNS Client Subnet. - **BGP-разведка** — аплинки, пиры, клиенты, анонсируемые префиксы, стабильность маршрутов, присутствие на IX и размер клиентского конуса по CAIDA ASRank. - **Репутация** — локально кэшируемые списки FireHOL (ни один запрос не покидает машину), экспозиция по Shodan InternetDB и опционально классические зоны DNSBL. - **Задержки и маршрут** — статистика пингов по каждому хосту с настоящими джиттером/потерями/mdev/p95/p99 и трассировка, которой никогда не нужны права администратора. Первый хоп диагностируется отдельно от остального маршрута, поэтому потери или медлительность там правильно относятся к вашему роутеру/Wi-Fi, а не к провайдеру. Каждый хоп размечен своим ASN, именем AS и страной, а маршрут, уходящий крюком через третью страну, отмечается отдельно. -- **Скорость** — каскадный замер (бинарник Ookla → Cloudflare → fast.com → опционально M-Lab NDT7) с раздельными оценками bufferbloat на приём и передачу. +- **Скорость** — каскадный замер (бинарник Ookla → Cloudflare → fast.com → опционально M-Lab NDT7) с раздельными оценками bufferbloat на приём и передачу и отметкой, если сервер замера оказался в другой стране. - **TLS handshake RTT** (`--tls`) — TCP RTT, время TLS 1.3-рукопожатия и время до первого байта к эталонным сервисам; отдельно отмечает сервер, чьё рукопожатие непропорционально медленнее сети (упирается в CPU TLS-терминатора, а не в сеть). - **DNS: системный vs DoH** (`--dns-advanced`) — сравнивает ответы и задержку системного резолвера с Cloudflare/Google/Quad9 DoH, а также проверяет фиктивный IP резолвера на признаки прозрачного DNS-прокси, перехватывающего 53 порт. - **Диверсификация маршрута / Anycast** (`--path-diversity`) — по заголовку Cloudflare `CF-RAY` определяет, какая точка (PoP) реально обслуживает трафик, и отмечает международную петлю маршрута, если она в другой стране, чем клиент. diff --git a/config.yaml b/config.yaml index eaa3b8a..3ba8190 100644 --- a/config.yaml +++ b/config.yaml @@ -117,6 +117,17 @@ path_diversity: - { label: "discord", host: "discord.com" } max_targets: 3 +# Runs first, before any other probe. Checks a handful of known generate_204-style +# endpoints for a redirect/tampered response, the classic signature of a public-wifi +# login wall. check_urls is capped at 4 in code. +captive_portal: + enabled: true + check_urls: + - "http://cp.cloudflare.com/generate_204" + - "http://connectivitycheck.gstatic.com/generate_204" + - "http://clients3.google.com/generate_204" + expected_status: 204 + output: logs_dir: "./logs" cache_dir: "./.cache" diff --git a/src/netsleuth/cli.py b/src/netsleuth/cli.py index fa6bab2..abf78b6 100644 --- a/src/netsleuth/cli.py +++ b/src/netsleuth/cli.py @@ -22,6 +22,7 @@ from netsleuth.exporter import build_report, dump_json, egress_asn, render_markd from netsleuth.iface import available_interfaces_hint, resolve_bind_target from netsleuth.interpret import ( assess_vpn, + captive_portal_findings, cgnat_findings, dns_advanced_findings, dpi_findings, @@ -36,9 +37,20 @@ from netsleuth.interpret import ( tls_findings, ) from netsleuth.ip_geo import dual_stack_mismatch, gather_identity -from netsleuth.models import BindTarget, Finding, IpGeo, LocalNet, ModuleResult, ProbeError, SpeedResult, VpnContext +from netsleuth.models import ( + BindTarget, + CaptivePortal, + Finding, + IpGeo, + LocalNet, + ModuleResult, + ProbeError, + SpeedResult, + VpnContext, +) from netsleuth.netinfo import collect_local_net, detect_capabilities, iface_for_ip, is_tunnel_iface, primary_interface_ip from netsleuth.orchestration import gather_modules, run_module, utc_now_iso +from netsleuth.probes.captive_portal import check_captive_portal from netsleuth.probes.dns_leak import collect_dns_leak from netsleuth.probes.hop_asn import enrich_hops from netsleuth.probes.latency import ping_fanout, tcp_connect_rtt @@ -476,6 +488,23 @@ async def diagnose(settings: Settings, options: Options) -> tuple[dict, list[Pat http2=True, headers={"User-Agent": f"netsleuth/{__version__}"}, ) as client: + # Phase 0 — captive portal gate. Detect-and-tag, never short-circuit: every + # later phase still runs, and the report says plainly that it was measured + # from behind a portal rather than silently looking like a normal run. + cp_cfg = settings.captive_portal + if cp_cfg.enabled: + modules["captive_portal"] = await run_module( + "captive_portal", + check_captive_portal( + client, cp_cfg.check_urls, expected_status=cp_cfg.expected_status, timeout=timeouts.http_seconds + ), + timeout=timeouts.module_seconds, + ) + else: + modules["captive_portal"] = ModuleResult( + name="captive_portal", status="skipped", warnings=["captive portal check disabled in config"] + ) + # Phase 1 — local facts and identity. Blocking: everything below needs the ASN. modules["connection"] = await run_module( "connection", asyncio.to_thread(collect_local_net), timeout=timeouts.module_seconds @@ -623,10 +652,11 @@ async def diagnose(settings: Settings, options: Options) -> tuple[dict, list[Pat latency_findings(pings, settings.thresholds) + [f for trace in traces for f in path_findings(trace, local, settings.thresholds)] + [f for trace in traces for f in path_asn_findings(trace, geo.country_code)] - + speed_findings(speed, settings.thresholds.bufferbloat_ms) + + speed_findings(speed, settings.thresholds.bufferbloat_ms, geo.country) + l7_findings + cgnat_findings(local, traces) + dual_stack_findings(bundle.get("nat64_prefix"), local) + + captive_portal_findings(modules["captive_portal"].data or CaptivePortal()) ) # Phase 6 — export @@ -648,6 +678,7 @@ async def diagnose(settings: Settings, options: Options) -> tuple[dict, list[Pat "path_diversity": options.path_diversity, "prefix_bench": options.prefix_bench, }, + "captive_portal": bool((modules["captive_portal"].data or CaptivePortal()).detected), "dpi_target": options.dpi_target, "formats": sorted(options.formats), "host_os": f"{platform.system()} {platform.release()}", diff --git a/src/netsleuth/config.py b/src/netsleuth/config.py index 96d13bc..b337fd3 100644 --- a/src/netsleuth/config.py +++ b/src/netsleuth/config.py @@ -201,6 +201,25 @@ class DnsAdvancedConfig(BaseModel): bogus_probe_name: str = "cloudflare.com" +class CaptivePortalConfig(BaseModel): + enabled: bool = True + check_urls: list[str] = Field( + default_factory=lambda: [ + "http://cp.cloudflare.com/generate_204", + "http://connectivitycheck.gstatic.com/generate_204", + "http://clients3.google.com/generate_204", + ] + ) + expected_status: int = 204 + + @field_validator("check_urls") + @classmethod + def _cap_check_urls(cls, value: list[str]) -> list[str]: + if len(value) > 4: + raise ValueError("captive_portal.check_urls may list at most 4 URLs") + return value + + class PathDiversityConfig(BaseModel): targets: list[HostSpec] = Field( default_factory=lambda: [ @@ -263,6 +282,7 @@ class Settings(BaseSettings): dpi_check: DpiCheckConfig = DpiCheckConfig() dns_advanced: DnsAdvancedConfig = DnsAdvancedConfig() path_diversity: PathDiversityConfig = PathDiversityConfig() + captive_portal: CaptivePortalConfig = CaptivePortalConfig() ipinfo_token: SecretStr | None = Field(default=None, alias="IPINFO_TOKEN") peeringdb_api_key: SecretStr | None = Field(default=None, alias="PEERINGDB_API_KEY") diff --git a/src/netsleuth/exporter.py b/src/netsleuth/exporter.py index d6a0016..2a9c48f 100644 --- a/src/netsleuth/exporter.py +++ b/src/netsleuth/exporter.py @@ -10,8 +10,9 @@ from netsleuth.config import FORMAT_EXTENSIONS from netsleuth.interpret import overall_verdict from netsleuth.models import Finding, ModuleResult, to_jsonable -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 SECTION_ORDER = ( + "captive_portal", "connection", "ip_geo", "vpn_assessment", @@ -125,6 +126,7 @@ _WARNING_RU = { "speedtest skipped: --quick": "speedtest пропущен: --quick", "speedtest skipped in target mode": "speedtest пропущен в режиме цели", "no throughput measured": "пропускная способность не измерена", + "captive portal check disabled in config": "проверка captive portal отключена в конфиге", } @@ -229,15 +231,21 @@ def _header(report: dict, emoji: bool, lang: str = "en") -> list[str]: def _tldr(report: dict, emoji: bool, lang: str = "en") -> list[str]: found = _findings(report) heading = _t(lang, "## TL;DR", "## Коротко") + banner = ( + [_t(lang, "> ⚠️ Every measurement below was taken from behind a captive portal.", "> ⚠️ Все измерения ниже сделаны из-за captive portal."), ""] + if (report.get("meta") or {}).get("captive_portal") + else [] + ) if not found: - return [heading, "", _t(lang, "No problems found on this connection.", "Проблем на этом соединении не найдено."), ""] - return [heading, ""] + [ + return banner + [heading, "", _t(lang, "No problems found on this connection.", "Проблем на этом соединении не найдено."), ""] + return banner + [heading, ""] + [ f"- {badge(f.get('severity', 'info'), emoji, lang)} **{_f(f, 'title', lang)}** — {_f(f, 'detail', lang)}" for f in found[:5] ] + [""] _SECTION_TITLES = { + "captive_portal": "## Captive portal check", "connection": "## Connection & identity", "ip_geo": "## Connection & identity", "vpn_assessment": "## VPN / proxy assessment", @@ -254,6 +262,7 @@ _SECTION_TITLES = { } _SECTION_TITLES_RU = { + "captive_portal": "## Проверка captive portal", "connection": "## Подключение и идентификация", "ip_geo": "## Подключение и идентификация", "vpn_assessment": "## Оценка VPN/прокси", @@ -311,6 +320,35 @@ def _forced_interface_rows(meta: dict, lang: str = "en") -> list[list[str]]: return rows +_PORTAL_VERDICT_RU = {"clean": "чисто", "portal": "портал", "suspect": "под вопросом", "error": "ошибка"} + + +def _captive_portal(report: dict, lang: str = "en") -> list[str]: + title = _section_title("captive_portal", lang) + note = unavailable(report, "captive_portal", lang) + if note: + return [title, "", note, ""] + cp = _data(report, "captive_portal") or {} + checks = cp.get("checks") or [] + lines = [title, ""] + _table( + [_t(lang, "URL", "URL"), _t(lang, "Status", "Статус"), _t(lang, "Verdict", "Вердикт"), _t(lang, "Evidence", "Доказательство")], + [ + [ + c.get("url", ""), + _num(c.get("status")), + c.get("verdict", "") if lang == "en" else _PORTAL_VERDICT_RU.get(c.get("verdict"), c.get("verdict", "")), + c.get("evidence") or "—", + ] + for c in checks + ], + lang, + ) + verdict_word = cp.get("verdict", "clean") if lang == "en" else _PORTAL_VERDICT_RU.get(cp.get("verdict"), cp.get("verdict")) + note_text = (cp.get("note_ru") or cp.get("note") or "") if lang == "ru" else (cp.get("note") or "") + lines += ["", f"{_t(lang, 'Verdict', 'Вердикт')}: **{verdict_word}**" + (f" — {note_text}" if note_text else "")] + return lines + [""] + + def _connection(report: dict, lang: str = "en") -> list[str]: title = _section_title("connection", lang) conn_note = unavailable(report, "connection", lang) @@ -637,6 +675,7 @@ def _speed(report: dict, lang: str = "en") -> list[str]: rows = [ [_t(lang, "Method", "Метод"), method_word], [_t(lang, "Server", "Сервер"), speed.get("server") or "—"], + [_t(lang, "Server country", "Страна сервера"), speed.get("server_country") or "—"], [_t(lang, "Download", "Скачивание"), f"{_num(speed.get('download_mbps'))} {_t(lang, 'Mbps', 'Мбит/с')}"], [_t(lang, "Upload", "Отдача"), f"{_num(speed.get('upload_mbps'))} {_t(lang, 'Mbps', 'Мбит/с')}"], [_t(lang, "Idle RTT", "RTT в простое"), f"{_num(speed.get('idle_rtt_ms'))} {_t(lang, 'ms', 'мс')}"], @@ -900,6 +939,7 @@ def render_markdown( link_text = "English" if lang == "ru" else "Русский" lines = [lines[0], "", f"[{link_text}]({sibling})"] + lines[1:] lines += _tldr(report, emoji, lang) + lines += _captive_portal(report, lang) lines += _connection(report, lang) lines += _vpn(report, lang) lines += _bgp(report, lang) diff --git a/src/netsleuth/interpret.py b/src/netsleuth/interpret.py index 9296fb3..5ce1850 100644 --- a/src/netsleuth/interpret.py +++ b/src/netsleuth/interpret.py @@ -3,6 +3,7 @@ from __future__ import annotations from netsleuth.config import Band, BufferbloatBands, Thresholds, VpnBands from netsleuth.models import ( + CaptivePortal, CfTrace, DnsAdvanced, DnsLeak, @@ -226,6 +227,38 @@ def path_findings(trace: TraceResult, local: LocalNet | None = None, t: Threshol return findings +def captive_portal_findings(cp: CaptivePortal) -> list[Finding]: + if cp.detected: + return [ + Finding( + id="net.captive_portal", + severity="crit", + title="Behind a captive portal", + detail=cp.note, + metric="portal_url", + value=cp.portal_url, + advice="Every measurement below was taken from behind the portal. Sign in through a browser, then rerun.", + title_ru="За captive portal", + detail_ru=cp.note_ru or cp.note, + advice_ru="Все измерения ниже сделаны из-за captive portal. Авторизуйтесь через браузер и перезапустите.", + ) + ] + if cp.verdict == "suspect": + return [ + Finding( + id="net.captive_portal_suspect", + severity="warn", + title="Captive portal check was inconclusive", + detail=cp.note, + advice="No check returned a clean expected response, but none looked like a portal redirect either; treat other findings with a grain of salt.", + title_ru="Проверка captive portal неоднозначна", + detail_ru=cp.note_ru or cp.note, + advice_ru="Ни одна проверка не вернула чистый ожидаемый ответ, но и на редирект портала не похоже; к остальным находкам стоит относиться с осторожностью.", + ) + ] + return [] + + def cgnat_findings(local: LocalNet, traces: list[TraceResult]) -> list[Finding]: evidence = local.cgnat_evidence detected = local.cgnat @@ -445,7 +478,7 @@ def bufferbloat_consequence(grade: str, lang: str = "en") -> str: return table.get(grade, table["?"]) -def speed_findings(speed: SpeedResult, bands: BufferbloatBands) -> list[Finding]: +def speed_findings(speed: SpeedResult, bands: BufferbloatBands, client_country: str | None = None) -> list[Finding]: if speed.method == "none": tried = ", ".join(a.tier for a in speed.tier_attempts) or "none" return [ @@ -486,6 +519,23 @@ def speed_findings(speed: SpeedResult, bands: BufferbloatBands) -> list[Finding] + " Включение SQM/fq_codel на роутере — стандартное решение.", ) ) + if client_country and speed.server_country and client_country.casefold() != speed.server_country.casefold(): + findings.append( + Finding( + id="speed.server_other_country", + severity="info", + title="Speedtest server is in another country", + detail=f"The speedtest server is in {speed.server_country}, you are in {client_country}.", + metric="server_country", + value=speed.server_country, + advice="A distant server can understate real throughput and overstate latency; " + "pin a closer one with --speedtest-server if one exists.", + title_ru="Сервер speedtest в другой стране", + detail_ru=f"Сервер speedtest находится в {speed.server_country}, а вы — в {client_country}.", + advice_ru="Далёкий сервер может занижать реальную пропускную способность и завышать задержку; " + "закрепите более близкий через --speedtest-server, если такой есть.", + ) + ) return findings diff --git a/src/netsleuth/models.py b/src/netsleuth/models.py index a09f065..d6b09fd 100644 --- a/src/netsleuth/models.py +++ b/src/netsleuth/models.py @@ -24,6 +24,7 @@ PORT_STATES = ("open", "closed", "filtered", "reset", "error") DPI_VERDICTS = ("clean", "partial_filtering", "reset_injection", "unreachable", "unknown") RESOLVER_KINDS = ("system", "doh") EDGE_SOURCES = ("cf_ray", "cf_trace", "server_timing", "none") +PORTAL_VERDICTS = ("clean", "portal", "suspect", "error") @dataclass @@ -347,6 +348,7 @@ class SpeedResult: bufferbloat_grade: str | None = None cfL4_stats: CfL4Stats | None = None netflix_oca_onnet: bool | None = None + server_country: str | None = None @dataclass @@ -470,6 +472,32 @@ class PathDiversity: note_ru: str | None = None +@dataclass +class PortalCheck: + url: str + status: int | None = None + verdict: str = "error" + evidence: str = "" + + def __post_init__(self) -> None: + if self.verdict not in PORTAL_VERDICTS: + raise ValueError(f"unknown PortalCheck verdict: {self.verdict!r}") + + +@dataclass +class CaptivePortal: + detected: bool = False + verdict: str = "clean" + checks: list[PortalCheck] = field(default_factory=list) + portal_url: str | None = None + note: str = "" + note_ru: str | None = None + + def __post_init__(self) -> None: + if self.verdict not in PORTAL_VERDICTS: + raise ValueError(f"unknown CaptivePortal verdict: {self.verdict!r}") + + def to_jsonable(obj: Any) -> Any: if obj is None or isinstance(obj, (str, bool, int)): return obj diff --git a/src/netsleuth/probes/captive_portal.py b/src/netsleuth/probes/captive_portal.py new file mode 100644 index 0000000..ce51948 --- /dev/null +++ b/src/netsleuth/probes/captive_portal.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from urllib.parse import urlsplit + +import httpx + +from netsleuth.models import CaptivePortal, PortalCheck + +_REDIRECT_STATUSES = (301, 302, 307, 511) + + +def classify_portal_response( + url: str, + status: int | None, + body: str, + location: str | None, + expected_status: int = 204, +) -> tuple[str, str]: + if status is None: + return "error", "no response" + if status == expected_status and not body: + return "clean", f"{status} with an empty body, as expected" + if status in _REDIRECT_STATUSES: + return "portal", f"redirected ({status}) to {location or '?'}" + if location: + requested_host = urlsplit(url).netloc + redirect_host = urlsplit(location).netloc + if redirect_host and redirect_host != requested_host: + return "portal", f"redirected to a different host ({redirect_host})" + if status == 200 and body: + return "portal", f"200 with a non-empty body instead of the expected {expected_status}" + return "suspect", f"unexpected status {status}" + + +async def check_captive_portal( + client: httpx.AsyncClient, + urls: list[str], + *, + expected_status: int = 204, + timeout: float, +) -> CaptivePortal: + checks: list[PortalCheck] = [] + for url in urls: + try: + response = await client.get(url, follow_redirects=False, timeout=timeout) + verdict, evidence = classify_portal_response( + url, response.status_code, response.text, response.headers.get("location"), expected_status + ) + except httpx.HTTPError as exc: + verdict, evidence = "error", str(exc) + checks.append(PortalCheck(url=url, status=None, verdict=verdict, evidence=evidence)) + continue + checks.append(PortalCheck(url=url, status=response.status_code, verdict=verdict, evidence=evidence)) + + portal_checks = [c for c in checks if c.verdict == "portal"] + if portal_checks: + return CaptivePortal( + detected=True, + verdict="portal", + checks=checks, + portal_url=portal_checks[0].url, + note=f"Captive portal detected: {portal_checks[0].evidence}.", + note_ru=f"Обнаружен captive portal: {portal_checks[0].evidence}.", + ) + if checks and all(c.verdict in ("suspect", "error") for c in checks): + return CaptivePortal( + detected=False, + verdict="suspect", + checks=checks, + note="Every check returned an ambiguous response; a captive portal is possible but not confirmed.", + note_ru="Все проверки вернули неоднозначный ответ; captive portal возможен, но не подтверждён.", + ) + return CaptivePortal(detected=False, verdict="clean", checks=checks) diff --git a/src/netsleuth/speed.py b/src/netsleuth/speed.py index af54c63..9a5a618 100644 --- a/src/netsleuth/speed.py +++ b/src/netsleuth/speed.py @@ -142,6 +142,7 @@ async def tier_ookla( upload_mbps=round(mbps(upload.get("bytes", 0), (upload.get("elapsed", 0) or 0) / 1000), 3), server=f"{server_info.get('name', '')} ({server_info.get('location', '')})".strip(), idle_rtt_ms=(payload.get("ping") or {}).get("latency"), + server_country=server_info.get("country") or None, ) @@ -198,6 +199,7 @@ async def tier_fastcom(client: httpx.AsyncClient, cfg: Speedtest, timeout: float upload_mbps=None, server=targets[0].get("url").split("/")[2] if targets and targets[0].get("url") else None, netflix_oca_onnet=on_net, + server_country=(targets[0].get("location") or {}).get("country") if targets else None, ) @@ -230,6 +232,7 @@ async def tier_ndt7( download_mbps=round(mbps(total, elapsed), 3), upload_mbps=None, server=results[0].get("machine"), + server_country=(results[0].get("location") or {}).get("country") or None, ) diff --git a/tests/test_captive_portal.py b/tests/test_captive_portal.py new file mode 100644 index 0000000..140a31c --- /dev/null +++ b/tests/test_captive_portal.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import httpx + +from netsleuth.interpret import captive_portal_findings +from netsleuth.models import CaptivePortal +from netsleuth.probes.captive_portal import check_captive_portal, classify_portal_response + +URL = "http://cp.cloudflare.com/generate_204" + + +def test_classify_204_with_empty_body_is_clean(): + assert classify_portal_response(URL, 204, "", None) == ("clean", "204 with an empty body, as expected") + + +def test_classify_204_with_a_body_is_suspect(): + verdict, _ = classify_portal_response(URL, 204, "unexpected", None) + assert verdict == "suspect" + + +def test_classify_200_with_a_body_is_portal(): + verdict, _ = classify_portal_response(URL, 200, "login", None) + assert verdict == "portal" + + +def test_classify_302_to_the_same_host_is_still_a_redirect_status_portal(): + verdict, evidence = classify_portal_response(URL, 302, "", "http://cp.cloudflare.com/login") + assert verdict == "portal" + assert "302" in evidence + + +def test_classify_302_to_a_different_host_is_portal(): + verdict, evidence = classify_portal_response(URL, 302, "", "http://portal.example.net/login") + assert verdict == "portal" + + +def test_classify_511_is_portal(): + assert classify_portal_response(URL, 511, "", None)[0] == "portal" + + +def test_classify_500_is_suspect(): + assert classify_portal_response(URL, 500, "", None)[0] == "suspect" + + +def test_classify_no_response_is_error(): + assert classify_portal_response(URL, None, "", None) == ("error", "no response") + + +async def test_check_captive_portal_detects_a_redirect(httpx_mock): + httpx_mock.add_response(url=URL, status_code=302, headers={"location": "http://portal.example.net/login"}) + async with httpx.AsyncClient() as client: + result = await check_captive_portal(client, [URL], timeout=5.0) + assert result.detected is True + assert result.verdict == "portal" + assert result.portal_url == URL + + +async def test_check_captive_portal_clean_connection(httpx_mock): + httpx_mock.add_response(url=URL, status_code=204) + async with httpx.AsyncClient() as client: + result = await check_captive_portal(client, [URL], timeout=5.0) + assert result.detected is False + assert result.verdict == "clean" + + +async def test_check_captive_portal_all_ambiguous_is_suspect(httpx_mock): + httpx_mock.add_response(url=URL, status_code=500) + async with httpx.AsyncClient() as client: + result = await check_captive_portal(client, [URL], timeout=5.0) + assert result.detected is False + assert result.verdict == "suspect" + + +def test_captive_portal_findings_fires_crit_when_detected(): + cp = CaptivePortal(detected=True, verdict="portal", portal_url=URL, note="redirected", note_ru="редирект") + findings = captive_portal_findings(cp) + assert [f.id for f in findings] == ["net.captive_portal"] + assert findings[0].severity == "crit" + + +def test_captive_portal_findings_fires_warn_when_suspect(): + cp = CaptivePortal(detected=False, verdict="suspect", note="ambiguous", note_ru="неоднозначно") + findings = captive_portal_findings(cp) + assert [f.id for f in findings] == ["net.captive_portal_suspect"] + assert findings[0].severity == "warn" + + +def test_captive_portal_findings_silent_when_clean(): + assert captive_portal_findings(CaptivePortal(detected=False, verdict="clean")) == [] diff --git a/tests/test_config.py b/tests/test_config.py index a45df90..3ca4665 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -191,6 +191,18 @@ def test_dpi_check_concurrency_is_capped_as_a_rate_limit(tmp_path): load_settings(config_path=path, env_file=tmp_path / "nope.env") +def test_captive_portal_check_urls_are_capped(tmp_path): + path = tmp_path / "config.yaml" + path.write_text( + "captive_portal:\n check_urls: " + + str([f"http://example{i}.com/generate_204" for i in range(5)]) + + "\n", + encoding="utf-8", + ) + with pytest.raises(ValidationError, match="at most 4"): + load_settings(config_path=path, env_file=tmp_path / "nope.env") + + def test_prefix_bench_max_prefixes_is_capped_so_it_cannot_scan_a_whole_as(tmp_path): path = tmp_path / "config.yaml" path.write_text("prefix_bench:\n max_prefixes: 1000\n", encoding="utf-8") diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 5d2c187..4e94b6b 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -81,6 +81,7 @@ def test_missing_modules_are_rendered_as_skipped_sections_not_dropped(): assert report["bgp"]["status"] == "skipped" assert report["bgp"]["data"] is None assert set(SECTION_ORDER) == { + "captive_portal", "connection", "ip_geo", "vpn_assessment", @@ -590,6 +591,7 @@ def test_markdown_has_every_section_from_the_spec_in_order(): headings = [line for line in text.splitlines() if line.startswith("## ")] assert headings == [ "## TL;DR", + "## Captive portal check", "## Connection & identity", "## VPN / proxy assessment", "## ASN & BGP intelligence", @@ -800,7 +802,7 @@ def test_diagnostics_translates_module_warnings_into_russian(): def test_an_all_modules_failed_run_still_renders_every_section(): text = render_markdown(dead_report()) headings = [line for line in text.splitlines() if line.startswith("## ")] - assert len(headings) == 15 + assert len(headings) == 16 assert text.count("Not available") == len(SECTION_ORDER) @@ -897,6 +899,7 @@ def test_render_markdown_default_call_is_byte_identical_to_before_the_lang_featu headings = [line for line in text.splitlines() if line.startswith("## ")] assert headings == [ "## TL;DR", + "## Captive portal check", "## Connection & identity", "## VPN / proxy assessment", "## ASN & BGP intelligence", diff --git a/tests/test_interpret.py b/tests/test_interpret.py index fd3e9f8..d013f73 100644 --- a/tests/test_interpret.py +++ b/tests/test_interpret.py @@ -387,6 +387,24 @@ def test_speed_findings_report_an_exhausted_cascade(): assert findings[0].severity == "info" +def test_speed_findings_flags_a_speedtest_server_in_another_country(): + speed = SpeedResult(method="ookla_bin", download_mbps=100.0, upload_mbps=20.0, server_country="Germany") + findings = speed_findings(speed, BufferbloatBands(), client_country="Russia") + assert [f.id for f in findings] == ["speed.server_other_country"] + assert findings[0].severity == "info" + + +def test_speed_findings_silent_when_server_country_matches_client(): + speed = SpeedResult(method="ookla_bin", download_mbps=100.0, upload_mbps=20.0, server_country="Germany") + assert speed_findings(speed, BufferbloatBands(), client_country="Germany") == [] + + +def test_speed_findings_silent_when_either_country_is_unknown(): + speed = SpeedResult(method="ookla_bin", download_mbps=100.0, upload_mbps=20.0, server_country=None) + assert speed_findings(speed, BufferbloatBands(), client_country="Russia") == [] + assert speed_findings(speed, BufferbloatBands(), client_country=None) == [] + + def test_overall_verdict_is_healthy_with_no_findings(): status, score, summary = overall_verdict([]) assert status == "ok" diff --git a/tests/test_speed_fastcom.py b/tests/test_speed_fastcom.py index 6680aae..fe57022 100644 --- a/tests/test_speed_fastcom.py +++ b/tests/test_speed_fastcom.py @@ -38,6 +38,7 @@ async def test_tier_fastcom_happy_path(httpx_mock): assert result.upload_mbps is None assert result.server == "server1.example.com" assert result.netflix_oca_onnet is True + assert result.server_country == "US" async def test_tier_fastcom_no_targets(httpx_mock):