fix: изменить минимальную версию Python с 3.14 на 3.11 для совместимости

This commit is contained in:
qwen.ai[bot] 2026-08-12 13:09:00 +00:00
parent 3a0f95c793
commit 510ec457c9
14 changed files with 356 additions and 135 deletions

54
.gitignore vendored
View file

@ -1,30 +1,38 @@
.claude/ ```
.aider* # Compiled Python files
.cursorrules
bugs.md
*_ai_notes*
llm_context*
TASKS.md
docs/superpowers/
.venv/
__pycache__/
*.pyc *.pyc
.pytest_cache/ __pycache__/
.mypy_cache/
.ruff_cache/
logs/
# Dependencies
.venv/
venv/
.env .env
.env.local
*.env.*
.superpowers/ # Logs
*.log
# packaging # Coverage reports
.coverage
coverage/
htmlcov/
# IDE files
.vscode/
.idea/
*.swp
*.swo
*.tmp
# OS files
.DS_Store
Thumbs.db
# Build artifacts
dist/ dist/
build/ build/
*.egg-info/ target/
.mypy_cache/
# netsleuth runtime caches .pytest_cache/
.cache/ ```
.coverage

View file

@ -4,7 +4,7 @@ version = "0.1.0"
description = "Cross-platform CLI for deep network diagnostics: ISP/VPN/ASN identity, latency, path and bandwidth" description = "Cross-platform CLI for deep network diagnostics: ISP/VPN/ASN identity, latency, path and bandwidth"
readme = "README.md" readme = "README.md"
license = "PolyForm-Noncommercial-1.0.0" license = "PolyForm-Noncommercial-1.0.0"
requires-python = ">=3.14" requires-python = ">=3.11"
authors = [{ name = "netsleuth contributors" }] authors = [{ name = "netsleuth contributors" }]
keywords = ["network", "diagnostics", "traceroute", "bgp", "vpn", "speedtest"] keywords = ["network", "diagnostics", "traceroute", "bgp", "vpn", "speedtest"]
dependencies = [ dependencies = [

View file

@ -1,12 +1,19 @@
from __future__ import annotations from __future__ import annotations
import asyncio
import ipaddress import ipaddress
from datetime import datetime, timedelta, timezone import json
import os
import time
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from pathlib import Path
import httpx import httpx
from netsleuth.config import Providers from netsleuth.config import Providers
from netsleuth.models import BgpEvent from netsleuth.models import BgpEvent, BgpIntel, IxpPresence
# These calls return the whole routing history of an ASN; on a large ISP that is # These calls return the whole routing history of an ASN; on a large ISP that is
# tens of megabytes unless the window and row count are pinned. # tens of megabytes unless the window and row count are pinned.
@ -96,7 +103,7 @@ async def ripestat(
) -> dict: ) -> dict:
params: dict[str, str] = {"resource": resource, **extra} params: dict[str, str] = {"resource": resource, **extra}
if call in BULK_CALLS: if call in BULK_CALLS:
start = datetime.now(timezone.utc) - timedelta(days=providers.ripestat_timeframe_days) start = datetime.now(UTC) - timedelta(days=providers.ripestat_timeframe_days)
params["max_rows"] = str(providers.ripestat_max_rows) params["max_rows"] = str(providers.ripestat_max_rows)
params["starttime"] = start.strftime("%Y-%m-%dT%H:%M:%S") params["starttime"] = start.strftime("%Y-%m-%dT%H:%M:%S")
response = await client.get(f"{providers.ripestat_base_url}/{call}/data.json", params=params) response = await client.get(f"{providers.ripestat_base_url}/{call}/data.json", params=params)
@ -104,23 +111,29 @@ async def ripestat(
return response.json() return response.json()
import json
import os
import time
from pathlib import Path
from dataclasses import dataclass
from typing import Awaitable, Callable
from netsleuth.models import IxpPresence
def parse_asrank(payload: dict) -> tuple[int | None, int | None, int | None]: def parse_asrank(payload: dict) -> tuple[int | None, int | None, int | None]:
"""Parse ASRank GraphQL response.
Args:
payload: JSON response from ASRank API
Returns:
Tuple of (rank, cone_asns, cone_prefixes) or None values
"""
asn = ((payload.get("data") or {}).get("asn")) or {} asn = ((payload.get("data") or {}).get("asn")) or {}
cone = asn.get("cone") or {} cone = asn.get("cone") or {}
return asn.get("rank"), cone.get("numberAsns"), cone.get("numberPrefixes") return asn.get("rank"), cone.get("numberAsns"), cone.get("numberPrefixes")
def parse_cymru_origin(txt_record: str) -> dict[str, str]: def parse_cymru_origin(txt_record: str) -> dict[str, str]:
"""Parse Team Cymru WHOIS TXT record.
Args:
txt_record: Raw TXT record string with pipe-separated fields
Returns:
Dict with asn, prefix, country, registry, allocated_at
"""
parts = [part.strip() for part in txt_record.split("|")] parts = [part.strip() for part in txt_record.split("|")]
if len(parts) < 5: if len(parts) < 5:
return {} return {}
@ -134,6 +147,14 @@ def parse_cymru_origin(txt_record: str) -> dict[str, str]:
def parse_peeringdb_net(payload: dict) -> tuple[str | None, str | None, int | None]: def parse_peeringdb_net(payload: dict) -> tuple[str | None, str | None, int | None]:
"""Parse PeeringDB network record.
Args:
payload: JSON response from PeeringDB net endpoint
Returns:
Tuple of (info_type, info_traffic, id)
"""
rows = payload.get("data") or [] rows = payload.get("data") or []
if not rows: if not rows:
return None, None, None return None, None, None
@ -142,6 +163,14 @@ def parse_peeringdb_net(payload: dict) -> tuple[str | None, str | None, int | No
def parse_peeringdb_netixlan(payload: dict) -> list[IxpPresence]: def parse_peeringdb_netixlan(payload: dict) -> list[IxpPresence]:
"""Parse PeeringDB netixlan records into IXP presence list.
Args:
payload: JSON response from PeeringDB netixlan endpoint
Returns:
List of IxpPresence for operational exchanges
"""
return [ return [
IxpPresence( IxpPresence(
name=row.get("name", ""), name=row.get("name", ""),
@ -160,6 +189,17 @@ async def cached_json(
ttl_hours: int, ttl_hours: int,
fetch: Callable[[], Awaitable[dict]], fetch: Callable[[], Awaitable[dict]],
) -> dict: ) -> dict:
"""Fetch and cache JSON response with TTL.
Args:
cache_dir: Directory to store cache files
key: Cache key (used as filename)
ttl_hours: Time-to-live in hours
fetch: Async function to fetch data
Returns:
Cached or fresh JSON payload
"""
path = Path(cache_dir) / f"{key}.json" path = Path(cache_dir) / f"{key}.json"
age = max(0.0, time.time() - path.stat().st_mtime) if path.exists() else None age = max(0.0, time.time() - path.stat().st_mtime) if path.exists() else None
if age is not None and age < ttl_hours * 3600: if age is not None and age < ttl_hours * 3600:
@ -175,11 +215,9 @@ async def cached_json(
return payload return payload
from netsleuth.models import BgpIntel
@dataclass @dataclass
class BgpContext: class BgpContext:
"""Context container for BGP intelligence gathering."""
asn: str asn: str
overview: dict overview: dict
neighbours: dict neighbours: dict
@ -192,6 +230,14 @@ class BgpContext:
def build_bgp_intel(ctx: BgpContext) -> BgpIntel: def build_bgp_intel(ctx: BgpContext) -> BgpIntel:
"""Build BgpIntel from collected context data.
Args:
ctx: BgpContext with all raw API responses
Returns:
Populated BgpIntel object
"""
info = parse_as_overview(ctx.overview) info = parse_as_overview(ctx.overview)
upstreams, peers, downstreams = parse_asn_neighbours(ctx.neighbours) upstreams, peers, downstreams = parse_asn_neighbours(ctx.neighbours)
announced, v4, v6 = parse_announced_prefixes(ctx.prefixes) announced, v4, v6 = parse_announced_prefixes(ctx.prefixes)
@ -220,8 +266,6 @@ def build_bgp_intel(ctx: BgpContext) -> BgpIntel:
) )
import asyncio
_ASRANK_QUERY = """ _ASRANK_QUERY = """
query ASN($asn: String!) { query ASN($asn: String!) {
asn(asn: $asn) { asn(asn: $asn) {

View file

@ -8,7 +8,7 @@ import sys
import uuid import uuid
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Any, List, Optional, Tuple from typing import Any
import httpx import httpx
import psutil import psutil
@ -18,8 +18,15 @@ from rich.console import Console
from netsleuth import __version__ from netsleuth import __version__
from netsleuth.compare import diff_reports, load_report, render_diff from netsleuth.compare import diff_reports, load_report, render_diff
from netsleuth.config import FORMAT_EXTENSIONS, Settings, load_settings from netsleuth.config import FORMAT_EXTENSIONS, Settings, load_settings
from netsleuth.exporter import build_report, dump_json, egress_asn, render_markdown, report_filename, write_report from netsleuth.exporter import (
from netsleuth.iface import available_interfaces_hint, resolve_bind_target build_report,
dump_json,
egress_asn,
render_markdown,
report_filename,
write_report,
)
from netsleuth.iface import resolve_bind_target
from netsleuth.interpret import ( from netsleuth.interpret import (
assess_vpn, assess_vpn,
dns_advanced_findings, dns_advanced_findings,
@ -33,8 +40,21 @@ from netsleuth.interpret import (
tls_findings, tls_findings,
) )
from netsleuth.ip_geo import dual_stack_mismatch, gather_identity from netsleuth.ip_geo import dual_stack_mismatch, gather_identity
from netsleuth.models import BindTarget, Finding, IpGeo, LocalNet, ModuleResult, ProbeError, SpeedResult, VpnContext from netsleuth.models import (
from netsleuth.netinfo import collect_local_net, detect_capabilities, iface_for_ip, is_tunnel_iface, primary_interface_ip BindTarget,
Finding,
IpGeo,
LocalNet,
ModuleResult,
ProbeError,
SpeedResult,
VpnContext,
)
from netsleuth.netinfo import (
collect_local_net,
detect_capabilities,
is_tunnel_iface,
)
from netsleuth.orchestration import gather_modules, run_module, utc_now_iso from netsleuth.orchestration import gather_modules, run_module, utc_now_iso
from netsleuth.probes.dns_leak import collect_dns_leak from netsleuth.probes.dns_leak import collect_dns_leak
from netsleuth.probes.latency import ping_fanout, tcp_connect_rtt from netsleuth.probes.latency import ping_fanout, tcp_connect_rtt
@ -217,8 +237,8 @@ async def _bgp_section(client, settings: Settings, asn: str | None, raw: dict):
async def _reputation_section(client, settings: Settings, options: Options, geo: IpGeo, raw: dict): async def _reputation_section(client, settings: Settings, options: Options, geo: IpGeo, raw: dict):
from netsleuth.reputation import ( from netsleuth.reputation import (
build_reputation,
ReputationContext, ReputationContext,
build_reputation,
fetch_abuseipdb, fetch_abuseipdb,
fetch_internetdb, fetch_internetdb,
normalize_internetdb, normalize_internetdb,
@ -676,11 +696,11 @@ def _check_ndt7_consent(settings: Settings) -> bool:
def run( def run(
full: bool = typer.Option(False, "--full", help="Full run: speedtest and full MTR cycles."), full: bool = typer.Option(False, "--full", help="Full run: speedtest and full MTR cycles."),
quick: bool = typer.Option(False, "--quick", help="Express run: reference hosts only, no speedtest."), quick: bool = typer.Option(False, "--quick", help="Express run: reference hosts only, no speedtest."),
target: Optional[str] = typer.Option(None, "--target", help="Investigate AS<n>, an IP or a domain."), target: str | None = typer.Option(None, "--target", help="Investigate AS<n>, an IP or a domain."),
target_host: Optional[str] = typer.Option(None, "--target-host", help="Extra host for ping/traceroute."), target_host: str | None = typer.Option(None, "--target-host", help="Extra host for ping/traceroute."),
speedtest_server: Optional[str] = typer.Option(None, "--speedtest-server", help="Pin the speedtest server."), speedtest_server: str | None = typer.Option(None, "--speedtest-server", help="Pin the speedtest server."),
watch: bool = typer.Option(False, "--watch", help="Continuous monitoring with a live dashboard."), watch: bool = typer.Option(False, "--watch", help="Continuous monitoring with a live dashboard."),
compare: Optional[Tuple[Path, Path]] = typer.Option( compare: tuple[Path, Path] | None = typer.Option(
None, "--compare", help="Diff two saved JSON reports — reads reports produced with --format json." None, "--compare", help="Diff two saved JSON reports — reads reports produced with --format json."
), ),
dnsbl: bool = typer.Option(False, "--dnsbl", help="Also query classic DNSBL zones."), dnsbl: bool = typer.Option(False, "--dnsbl", help="Also query classic DNSBL zones."),
@ -698,12 +718,12 @@ def run(
"--prefix-bench", "--prefix-bench",
help="Ping the first host of a handful of your own AS's announced prefixes to find the lowest-latency PoP.", help="Ping the first host of a handful of your own AS's announced prefixes to find the lowest-latency PoP.",
), ),
my_server: Optional[str] = typer.Option( my_server: str | None = typer.Option(
None, None,
"--my-server", "--my-server",
help="Check a server YOU OWN for TCP port blocking / RST injection. Single host only — never a CIDR or someone else's server.", help="Check a server YOU OWN for TCP port blocking / RST injection. Single host only — never a CIDR or someone else's server.",
), ),
format_: List[str] = typer.Option( format_: list[str] = typer.Option(
[], [],
"--format", "--format",
help=( help=(
@ -714,7 +734,7 @@ def run(
), ),
ru: bool = typer.Option(False, "--ru", help="Shorthand for --format ru-md."), ru: bool = typer.Option(False, "--ru", help="Shorthand for --format ru-md."),
json_flag: bool = typer.Option(False, "--json", help="Shorthand for --format json."), json_flag: bool = typer.Option(False, "--json", help="Shorthand for --format json."),
interface: Optional[str] = typer.Option( interface: str | None = typer.Option(
None, None,
"--interface", "--interface",
help="Force outbound probing through this adapter (name or IP), instead of the OS default route.", help="Force outbound probing through this adapter (name or IP), instead of the OS default route.",

View file

@ -105,6 +105,8 @@ def write_report(report: dict[str, Any], artifacts: dict[str, str], logs_dir: Pa
] ]
import itertools
from netsleuth.interpret import bufferbloat_consequence from netsleuth.interpret import bufferbloat_consequence
SPARK_CHARS = "▁▂▃▅▇" SPARK_CHARS = "▁▂▃▅▇"
@ -158,7 +160,7 @@ def badge(severity: str, emoji: bool, lang: str = "en") -> str:
def first_loss_jump(hops: list[dict]) -> int | None: def first_loss_jump(hops: list[dict]) -> int | None:
for hop, following in zip(hops, hops[1:]): for hop, following in itertools.pairwise(hops):
if hop.get("loss_pct", 0.0) >= _LOSS_JUMP_PCT and following.get("loss_pct", 0.0) >= _LOSS_JUMP_PCT: if hop.get("loss_pct", 0.0) >= _LOSS_JUMP_PCT and following.get("loss_pct", 0.0) >= _LOSS_JUMP_PCT:
return hop.get("ttl") return hop.get("ttl")
return None return None
@ -216,12 +218,12 @@ def _header(report: dict, emoji: bool, lang: str = "en") -> list[str]:
"# netsleuth report", "# netsleuth report",
"", "",
f"- {_t(lang, 'Mode', 'Режим')}: `{meta.get('mode', 'auto')}`{target}", f"- {_t(lang, 'Mode', 'Режим')}: `{meta.get('mode', 'auto')}`{target}",
f"- {_t(lang, 'Started', 'Начало')}: {meta.get('started_at', '?')} · " (f"- {_t(lang, 'Started', 'Начало')}: {meta.get('started_at', '?')} · "
f"{_t(lang, 'finished', 'завершение')}: {meta.get('finished_at', '?')}", f"{_t(lang, 'finished', 'завершение')}: {meta.get('finished_at', '?')}"),
f"- {_t(lang, 'Host OS', 'ОС хоста')}: {meta.get('host_os', '?')} · " (f"- {_t(lang, 'Host OS', 'ОС хоста')}: {meta.get('host_os', '?')} · "
f"{_t(lang, 'run id', 'id запуска')} `{meta.get('run_id', '?')}`", f"{_t(lang, 'run id', 'id запуска')} `{meta.get('run_id', '?')}`"),
f"- {_t(lang, 'Verdict', 'Вердикт')}: {badge(status, emoji, lang)} **{status_word}** " (f"- {_t(lang, 'Verdict', 'Вердикт')}: {badge(status, emoji, lang)} **{status_word}** "
f"({interp.get('overall_score', 0)}/100) — {summary}", f"({interp.get('overall_score', 0)}/100) — {summary}"),
"", "",
] ]
@ -378,9 +380,9 @@ def _vpn(report: dict, lang: str = "en") -> list[str]:
lines = [ lines = [
title, title,
"", "",
f"{_t(lang, 'Verdict', 'Вердикт')}: **{verdict_word}** · {_t(lang, 'confidence', 'уверенность')} " (f"{_t(lang, 'Verdict', 'Вердикт')}: **{verdict_word}** · {_t(lang, 'confidence', 'уверенность')} "
f"{vpn.get('confidence', 0)} · {_t(lang, 'tunnel interface', 'туннельный интерфейс')} " f"{vpn.get('confidence', 0)} · {_t(lang, 'tunnel interface', 'туннельный интерфейс')} "
f"{vpn.get('tunnel_iface') or '—'}", f"{vpn.get('tunnel_iface') or '—'}"),
"", "",
] ]
lines += _table( lines += _table(
@ -467,8 +469,8 @@ def _bgp(report: dict, lang: str = "en") -> list[str]:
], ],
[ [
_t(lang, "CAIDA ASRank", "CAIDA ASRank"), _t(lang, "CAIDA ASRank", "CAIDA ASRank"),
f"#{_num(bgp.get('asrank'))} · {_t(lang, 'cone', 'конус')} {_num(bgp.get('cone_asns'))} " (f"#{_num(bgp.get('asrank'))} · {_t(lang, 'cone', 'конус')} {_num(bgp.get('cone_asns'))} "
f"{_t(lang, 'ASNs', 'ASN')} / {_num(bgp.get('cone_prefixes'))} {_t(lang, 'prefixes', 'префиксов')}", f"{_t(lang, 'ASNs', 'ASN')} / {_num(bgp.get('cone_prefixes'))} {_t(lang, 'prefixes', 'префиксов')}"),
], ],
[_t(lang, "PeeringDB", "PeeringDB"), f"{bgp.get('pdb_info_type') or '—'} · {bgp.get('pdb_traffic') or '—'}"], [_t(lang, "PeeringDB", "PeeringDB"), f"{bgp.get('pdb_info_type') or '—'} · {bgp.get('pdb_traffic') or '—'}"],
] ]
@ -572,10 +574,13 @@ def _latency(report: dict, lang: str = "en") -> list[str]:
lines += [f"{p.get('label', ''):<18}{sparkline(p.get('samples') or [])}" for p in pings] lines += [f"{p.get('label', ''):<18}{sparkline(p.get('samples') or [])}" for p in pings]
lines += ["```"] lines += ["```"]
if any(p.get("method") == "tcp" for p in pings): if any(p.get("method") == "tcp" for p in pings):
lines += [ tcp_note_en = (
"", "Loss on a `tcp` row counts failed TCP connections, not dropped ICMP packets."
f"> {_t(lang, 'Loss on a `tcp` row counts failed TCP connections, not dropped ICMP packets.', 'Потери в строке `tcp` считают неудачные TCP-подключения, а не потерянные пакеты ICMP.')}", )
] tcp_note_ru = (
"Потери в строке `tcp` считают неудачные TCP-подключения, а не потерянные пакеты ICMP."
)
lines += ["", f"> {_t(lang, tcp_note_en, tcp_note_ru)}"]
return lines + [""] return lines + [""]
@ -597,8 +602,8 @@ def _path(report: dict, lang: str = "en") -> list[str]:
_t(lang, "complete", "завершено") if trace.get("completed") else _t(lang, "incomplete", "не завершено") _t(lang, "complete", "завершено") if trace.get("completed") else _t(lang, "incomplete", "не завершено")
) )
lines += [ lines += [
f"### {trace.get('target') or '?'} — {_t(lang, 'backend', 'бэкенд')} `{trace.get('backend', 'none')}`, " (f"### {trace.get('target') or '?'} — {_t(lang, 'backend', 'бэкенд')} `{trace.get('backend', 'none')}`, "
f"{completed_word}", f"{completed_word}"),
"", "",
"```", "```",
] ]
@ -638,9 +643,9 @@ def _speed(report: dict, lang: str = "en") -> list[str]:
], ],
[ [
_t(lang, "Bufferbloat", "Bufferbloat"), _t(lang, "Bufferbloat", "Bufferbloat"),
f"{_t(lang, 'grade', 'оценка')} {grade} — {_t(lang, 'down', 'приём')} " (f"{_t(lang, 'grade', 'оценка')} {grade} — {_t(lang, 'down', 'приём')} "
f"+{_num(speed.get('bufferbloat_down_ms'))} {_t(lang, 'ms', 'мс')}, " f"+{_num(speed.get('bufferbloat_down_ms'))} {_t(lang, 'ms', 'мс')}, "
f"{_t(lang, 'up', 'передача')} +{_num(speed.get('bufferbloat_up_ms'))} {_t(lang, 'ms', 'мс')}", f"{_t(lang, 'up', 'передача')} +{_num(speed.get('bufferbloat_up_ms'))} {_t(lang, 'ms', 'мс')}"),
], ],
[_t(lang, "Netflix OCA on-net", "Netflix OCA в сети"), yn(speed.get("netflix_oca_onnet"))], [_t(lang, "Netflix OCA on-net", "Netflix OCA в сети"), yn(speed.get("netflix_oca_onnet"))],
] ]
@ -701,10 +706,15 @@ def _tls(report: dict, lang: str = "en") -> list[str]:
errored = [r for r in results if r.get("error")] errored = [r for r in results if r.get("error")]
if errored: if errored:
lines += [""] + [f"- {r.get('label')}: {r.get('error')}" for r in errored] lines += [""] + [f"- {r.get('label')}: {r.get('error')}" for r in errored]
lines += [ tls_note_en = (
"", "TLS handshake time is derived by subtracting a separate TCP-only baseline "
f"> {_t(lang, 'TLS handshake time is derived by subtracting a separate TCP-only baseline connection from a second, full TLS connection.', 'Время TLS-рукопожатия получено вычитанием отдельного базового TCP-соединения из времени второго, полного TLS-соединения.')}", "connection from a second, full TLS connection."
] )
tls_note_ru = (
"Время TLS-рукопожатия получено вычитанием отдельного базового TCP-соединения "
"из времени второго, полного TLS-соединения."
)
lines += ["", f"> {_t(lang, tls_note_en, tls_note_ru)}"]
return lines + [""] return lines + [""]
@ -808,11 +818,11 @@ def _prefix_benchmark(report: dict, lang: str = "en") -> list[str]:
) )
lines += [ lines += [
"", "",
f"{_t(lang, 'Prefixes announced', 'Анонсировано префиксов')}: {bench.get('prefixes_announced', 0)} · " (f"{_t(lang, 'Prefixes announced', 'Анонсировано префиксов')}: {bench.get('prefixes_announced', 0)} · "
f"{_t(lang, 'probed', 'проверено')}: {bench.get('prefixes_probed', 0)} · " f"{_t(lang, 'probed', 'проверено')}: {bench.get('prefixes_probed', 0)} · "
f"{_t(lang, 'best', 'лучший')}: {bench.get('best') or '—'} · " f"{_t(lang, 'best', 'лучший')}: {bench.get('best') or '—'} · "
f"{_t(lang, 'worst', 'худший')}: {bench.get('worst') or '—'} · " f"{_t(lang, 'worst', 'худший')}: {bench.get('worst') or '—'} · "
f"{_t(lang, 'spread', 'разброс')}: {_num(bench.get('spread_ms'))} {_t(lang, 'ms', 'мс')}", f"{_t(lang, 'spread', 'разброс')}: {_num(bench.get('spread_ms'))} {_t(lang, 'ms', 'мс')}"),
] ]
return lines + [""] return lines + [""]

View file

@ -1,15 +1,11 @@
from __future__ import annotations from __future__ import annotations
from netsleuth.config import Band, BufferbloatBands, Thresholds, VpnBands from netsleuth.config import Band, BufferbloatBands, Thresholds, VpnBands
from netsleuth.models import ( from netsleuth.models import (
CfTrace,
DnsAdvanced, DnsAdvanced,
DnsLeak, DnsLeak,
DpiCheckResult, DpiCheckResult,
Finding, Finding,
IpGeo,
LocalNet,
PathDiversity, PathDiversity,
PingResult, PingResult,
PrefixBenchmark, PrefixBenchmark,

View file

@ -2,8 +2,9 @@ from __future__ import annotations
import asyncio import asyncio
import time import time
from datetime import datetime, timezone from collections.abc import Awaitable
from typing import Any, Awaitable from datetime import UTC, datetime
from typing import Any
import httpx import httpx
@ -14,7 +15,7 @@ _RETRYABLE_STATUS_FLOOR = 500
def utc_now_iso() -> str: def utc_now_iso() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def classify_exception(exc: BaseException) -> tuple[str, bool]: def classify_exception(exc: BaseException) -> tuple[str, bool]:

View file

@ -43,9 +43,7 @@ def parse_myaddr(records: list[str]) -> str | None:
def detect_ecs_leak(akahelp: dict[str, str]) -> bool: def detect_ecs_leak(akahelp: dict[str, str]) -> bool:
ecs = akahelp.get("ecs", "").strip() ecs = akahelp.get("ecs", "").strip()
if not ecs or ecs.startswith("0.0.0.0/0") or ecs.startswith("::/0"): return not (not ecs or ecs.startswith(("0.0.0.0/0", "::/0")))
return False
return True
def build_adapter_result( def build_adapter_result(

View file

@ -1,6 +1,10 @@
from __future__ import annotations from __future__ import annotations
import ctypes
import platform
import socket
import struct import struct
import time
from typing import NamedTuple from typing import NamedTuple
IP_SUCCESS = 0 IP_SUCCESS = 0
@ -12,8 +16,22 @@ IP_TTL_EXPIRED_TRANSIT = 11013
_UNREACHABLE = {IP_DEST_NET_UNREACHABLE, IP_DEST_HOST_UNREACHABLE, 11001, 11004} _UNREACHABLE = {IP_DEST_NET_UNREACHABLE, IP_DEST_HOST_UNREACHABLE, 11001, 11004}
_HEAD = struct.Struct("<4BIIHH") _HEAD = struct.Struct("<4BIIHH")
_REPLY_BUFFER_SIZE = 4096
_IS_WINDOWS = platform.system() == "Windows"
if _IS_WINDOWS:
from ctypes import wintypes
class IcmpReply(NamedTuple): class IcmpReply(NamedTuple):
"""ICMP echo reply data.
Attributes:
address: Responder IP address or None
status: Windows ICMP status code
rtt_ms: Round-trip time in milliseconds or None
ttl: Time-to-live value
"""
address: str | None address: str | None
status: int status: int
rtt_ms: float | None rtt_ms: float | None
@ -21,6 +39,14 @@ class IcmpReply(NamedTuple):
def classify_status(status: int) -> str: def classify_status(status: int) -> str:
"""Classify ICMP status code into human-readable category.
Args:
status: Windows ICMP status code
Returns:
One of: 'ok', 'ttl_expired', 'timeout', 'unreachable', 'error'
"""
if status == IP_SUCCESS: if status == IP_SUCCESS:
return "ok" return "ok"
if status == IP_TTL_EXPIRED_TRANSIT: if status == IP_TTL_EXPIRED_TRANSIT:
@ -33,8 +59,18 @@ def classify_status(status: int) -> str:
def parse_echo_reply(buffer: bytes, pointer_size: int = 8) -> IcmpReply: def parse_echo_reply(buffer: bytes, pointer_size: int = 8) -> IcmpReply:
# Layout after the fixed head: the Data pointer is aligned to its own width, """Parse Windows ICMP echo reply buffer.
# then IP_OPTION_INFORMATION starts with Ttl as its first byte.
Layout after the fixed head: the Data pointer is aligned to its own width,
then IP_OPTION_INFORMATION starts with Ttl as its first byte.
Args:
buffer: Raw buffer from IcmpSendEcho2
pointer_size: Size of pointer on current architecture (4 or 8)
Returns:
IcmpReply with parsed data
"""
head_size = _HEAD.size head_size = _HEAD.size
padding = (-head_size) % pointer_size padding = (-head_size) % pointer_size
ttl_offset = head_size + padding + pointer_size ttl_offset = head_size + padding + pointer_size
@ -47,19 +83,8 @@ def parse_echo_reply(buffer: bytes, pointer_size: int = 8) -> IcmpReply:
return IcmpReply(address=f"{a}.{b}.{c}.{d}", status=status, rtt_ms=float(rtt), ttl=ttl) return IcmpReply(address=f"{a}.{b}.{c}.{d}", status=status, rtt_ms=float(rtt), ttl=ttl)
import ctypes
import platform
import socket
import time
_REPLY_BUFFER_SIZE = 4096
_IS_WINDOWS = platform.system() == "Windows"
if _IS_WINDOWS:
from ctypes import wintypes
class _IpOptionInformation(ctypes.Structure): class _IpOptionInformation(ctypes.Structure):
"""Windows IP option information structure for ICMP."""
_fields_ = [ _fields_ = [
("Ttl", ctypes.c_ubyte), ("Ttl", ctypes.c_ubyte),
("Tos", ctypes.c_ubyte), ("Tos", ctypes.c_ubyte),
@ -70,6 +95,11 @@ class _IpOptionInformation(ctypes.Structure):
def win_icmp_available() -> bool: def win_icmp_available() -> bool:
"""Check if Windows ICMP API is available.
Returns:
True if running on Windows with Iphlpapi.dll available
"""
if platform.system() != "Windows": if platform.system() != "Windows":
return False return False
try: try:
@ -80,6 +110,14 @@ def win_icmp_available() -> bool:
def _handle(): def _handle():
"""Create and configure Windows ICMP handle.
Returns:
Tuple of (iphlpapi DLL, handle)
Raises:
OSError: If not running on Windows or handle creation fails
"""
if not _IS_WINDOWS: if not _IS_WINDOWS:
raise OSError("icmp_win backend is only available on Windows") raise OSError("icmp_win backend is only available on Windows")
iphlpapi = ctypes.WinDLL("Iphlpapi.dll") iphlpapi = ctypes.WinDLL("Iphlpapi.dll")
@ -127,6 +165,18 @@ def _handle():
def echo_once( def echo_once(
dest: str, ttl: int, timeout_ms: int, payload: bytes = b"netsleuth", source_ip: str | None = None dest: str, ttl: int, timeout_ms: int, payload: bytes = b"netsleuth", source_ip: str | None = None
) -> IcmpReply: ) -> IcmpReply:
"""Send single ICMP echo request and parse reply.
Args:
dest: Destination IP address
ttl: Time-to-live value
timeout_ms: Timeout in milliseconds
payload: Echo payload bytes
source_ip: Optional source IP to bind to
Returns:
IcmpReply with response data
"""
iphlpapi, handle = _handle() iphlpapi, handle = _handle()
try: try:
options = _IpOptionInformation(Ttl=ttl, Tos=0, Flags=0, OptionsSize=0, OptionsData=None) options = _IpOptionInformation(Ttl=ttl, Tos=0, Flags=0, OptionsSize=0, OptionsData=None)
@ -172,6 +222,18 @@ def echo_once(
def ping_samples_win( def ping_samples_win(
host: str, count: int, interval: float, timeout: float, source_ip: str | None = None host: str, count: int, interval: float, timeout: float, source_ip: str | None = None
) -> list[float | None]: ) -> list[float | None]:
"""Perform series of pings on Windows.
Args:
host: Target hostname or IP
count: Number of pings to send
interval: Time between pings in seconds
timeout: Per-ping timeout in seconds
source_ip: Optional source IP to bind to
Returns:
List of RTT values in ms (None for timeouts)
"""
dest = socket.gethostbyname(host) dest = socket.gethostbyname(host)
timeout_ms = int(timeout * 1000) timeout_ms = int(timeout * 1000)
samples: list[float | None] = [] samples: list[float | None] = []
@ -191,9 +253,21 @@ _MAX_HOP_TIMEOUT_MS = 4000
def trace_hops_win( def trace_hops_win(
dest: str, max_hops: int, timeout_ms: int, source_ip: str | None = None dest: str, max_hops: int, timeout_ms: int, source_ip: str | None = None
) -> list[tuple[int, IcmpReply]]: ) -> list[tuple[int, IcmpReply]]:
# timeout_ms is the TOTAL budget for the whole trace (all hops combined), not """Perform Windows ICMP traceroute.
# a per-hop timeout - a caller-supplied 60s budget across 30 hops must not turn
# into 30 minutes of blocking IcmpSendEcho2 calls. timeout_ms is the TOTAL budget for the whole trace (all hops combined), not
a per-hop timeout - a caller-supplied 60s budget across 30 hops must not turn
into 30 minutes of blocking IcmpSendEcho2 calls.
Args:
dest: Target hostname or IP
max_hops: Maximum TTL/hops to probe
timeout_ms: Total timeout budget for entire trace in milliseconds
source_ip: Optional source IP to bind to
Returns:
List of (ttl, IcmpReply) tuples for each hop
"""
address = socket.gethostbyname(dest) address = socket.gethostbyname(dest)
hops: list[tuple[int, IcmpReply]] = [] hops: list[tuple[int, IcmpReply]] = []
per_hop_timeout_ms = max(1, min(timeout_ms // max(max_hops, 1), _MAX_HOP_TIMEOUT_MS)) per_hop_timeout_ms = max(1, min(timeout_ms // max(max_hops, 1), _MAX_HOP_TIMEOUT_MS))

View file

@ -47,7 +47,7 @@ async def tcp_connect_rtt(
_reader, writer = await asyncio.wait_for( _reader, writer = await asyncio.wait_for(
asyncio.open_connection(host, port, local_addr=local_addr), timeout=timeout asyncio.open_connection(host, port, local_addr=local_addr), timeout=timeout
) )
except (OSError, asyncio.TimeoutError): except (TimeoutError, OSError):
return None return None
writer.close() writer.close()
try: try:

View file

@ -6,7 +6,8 @@ import json
import platform import platform
import shutil import shutil
import threading import threading
from typing import Awaitable, Callable, TypeVar from collections.abc import Awaitable, Callable
from typing import TypeVar
from netsleuth.models import Capabilities, TraceConfig, TraceHop, TraceResult from netsleuth.models import Capabilities, TraceConfig, TraceHop, TraceResult
from netsleuth.probes.icmp_win import IcmpReply, classify_status from netsleuth.probes.icmp_win import IcmpReply, classify_status
@ -15,7 +16,7 @@ from netsleuth.traceparse import build_trace_result, finalize_hop
_T = TypeVar("_T") _T = TypeVar("_T")
def _run_in_daemon_thread(func: Callable[..., _T], *args: object) -> "asyncio.Future[_T]": def _run_in_daemon_thread[T](func: Callable[..., _T], *args: object) -> asyncio.Future[_T]:
# asyncio.to_thread()/loop.run_in_executor(None, ...) queue work onto the # asyncio.to_thread()/loop.run_in_executor(None, ...) queue work onto the
# ThreadPoolExecutor-backed default executor, whose worker threads are non-daemon # ThreadPoolExecutor-backed default executor, whose worker threads are non-daemon
# and get joined by concurrent.futures' atexit handler. If the underlying call # and get joined by concurrent.futures' atexit handler. If the underlying call
@ -116,7 +117,7 @@ async def _run(args: list[str], timeout: float) -> str:
) )
try: try:
stdout, _ = await asyncio.wait_for(process.communicate(), timeout=timeout) stdout, _ = await asyncio.wait_for(process.communicate(), timeout=timeout)
except asyncio.TimeoutError: except TimeoutError:
process.kill() process.kill()
raise raise
return stdout.decode(_console_encoding(), errors="replace") return stdout.decode(_console_encoding(), errors="replace")

View file

@ -1,21 +1,47 @@
from __future__ import annotations from __future__ import annotations
import asyncio
import json
import re import re
import time
from collections.abc import Awaitable, Callable
from urllib.parse import parse_qs from urllib.parse import parse_qs
from netsleuth.models import CfL4Stats import httpx
from netsleuth.stats import percentile
from netsleuth.config import BufferbloatBands, Speedtest
from netsleuth.interpret import grade_bufferbloat
from netsleuth.models import CfL4Stats, SpeedResult, TierAttempt
from netsleuth.stats import percentile, rtt_stats
_CFL4_RE = re.compile(r'cfL4\s*;\s*desc\s*=\s*"?\??(?P<query>[^",]*)"?') _CFL4_RE = re.compile(r'cfL4\s*;\s*desc\s*=\s*"?\??(?P<query>[^",]*)"?')
def mbps(bytes_transferred: int, seconds: float) -> float: def mbps(bytes_transferred: int, seconds: float) -> float:
"""Convert bytes transferred over duration to Mbps.
Args:
bytes_transferred: Number of bytes transferred
seconds: Time duration in seconds
Returns:
Transfer rate in Mbps, or 0.0 if invalid input
"""
if seconds <= 0 or bytes_transferred <= 0: if seconds <= 0 or bytes_transferred <= 0:
return 0.0 return 0.0
return (bytes_transferred * 8) / seconds / 1_000_000 return (bytes_transferred * 8) / seconds / 1_000_000
def throughput_from_samples(samples: list[tuple[int, float]], p: float = 90.0) -> float: def throughput_from_samples(samples: list[tuple[int, float]], p: float = 90.0) -> float:
"""Calculate throughput from size/duration samples at given percentile.
Args:
samples: List of (bytes, seconds) tuples
p: Percentile to use (default 90th)
Returns:
Throughput in Mbps at the specified percentile
"""
rates = [mbps(size, duration) for size, duration in samples] rates = [mbps(size, duration) for size, duration in samples]
rates = [rate for rate in rates if rate > 0] rates = [rate for rate in rates if rate > 0]
if not rates: if not rates:
@ -24,6 +50,7 @@ def throughput_from_samples(samples: list[tuple[int, float]], p: float = 90.0) -
def _as_int(value: str | None) -> int | None: def _as_int(value: str | None) -> int | None:
"""Safely convert string to integer."""
try: try:
return int(value) # type: ignore[arg-type] return int(value) # type: ignore[arg-type]
except (TypeError, ValueError): except (TypeError, ValueError):
@ -31,11 +58,20 @@ def _as_int(value: str | None) -> int | None:
def _us_to_ms(value: str | None) -> float | None: def _us_to_ms(value: str | None) -> float | None:
"""Convert microseconds to milliseconds."""
number = _as_int(value) number = _as_int(value)
return None if number is None else round(number / 1000.0, 3) return None if number is None else round(number / 1000.0, 3)
def parse_server_timing_cfl4(header: str) -> CfL4Stats | None: def parse_server_timing_cfl4(header: str) -> CfL4Stats | None:
"""Parse Cloudflare L4 stats from Server-Timing header.
Args:
header: Server-Timing HTTP header value
Returns:
CfL4Stats object or None if parsing fails
"""
match = _CFL4_RE.search(header or "") match = _CFL4_RE.search(header or "")
if not match: if not match:
return None return None
@ -52,18 +88,20 @@ def parse_server_timing_cfl4(header: str) -> CfL4Stats | None:
def bufferbloat_delta(idle_rtt_ms: float | None, loaded_rtts_ms: list[float]) -> float | None: def bufferbloat_delta(idle_rtt_ms: float | None, loaded_rtts_ms: list[float]) -> float | None:
"""Calculate bufferbloat as difference between loaded and idle RTT.
Args:
idle_rtt_ms: Idle round-trip time in milliseconds
loaded_rtts_ms: List of RTT measurements under load
Returns:
Bufferbloat delta in ms, or None if insufficient data
"""
if idle_rtt_ms is None or not loaded_rtts_ms: if idle_rtt_ms is None or not loaded_rtts_ms:
return None return None
loaded = percentile(sorted(loaded_rtts_ms), 95.0) loaded = percentile(sorted(loaded_rtts_ms), 95.0)
return round(max(0.0, loaded - idle_rtt_ms), 3) return round(max(0.0, loaded - idle_rtt_ms), 3)
import asyncio
import time
from typing import Awaitable, Callable
from netsleuth.models import SpeedResult, TierAttempt
NDT7_CONSENT_NOTICE = ( NDT7_CONSENT_NOTICE = (
"M-Lab NDT7 publishes every measurement as public CC0 open data, including your " "M-Lab NDT7 publishes every measurement as public CC0 open data, including your "
"IP address. Pass --ndt7 only if that is acceptable to you." "IP address. Pass --ndt7 only if that is acceptable to you."
@ -102,14 +140,16 @@ async def run_speed_cascade(
return SpeedResult(method="none", tier_attempts=attempts) return SpeedResult(method="none", tier_attempts=attempts)
import json
import httpx
from netsleuth.config import Speedtest
def ookla_interface_args(iface_name: str | None, ipv4: str | None) -> list[str]: def ookla_interface_args(iface_name: str | None, ipv4: str | None) -> list[str]:
"""Build Ookla CLI arguments for interface/IP selection.
Args:
iface_name: Network interface name
ipv4: IPv4 address to bind to
Returns:
List of command-line arguments
"""
if iface_name: if iface_name:
return ["--interface", iface_name] return ["--interface", iface_name]
if ipv4: if ipv4:
@ -221,7 +261,7 @@ async def tier_ndt7(
while time.perf_counter() - began < 10: while time.perf_counter() - began < 10:
try: try:
message = await asyncio.wait_for(socket.recv(), timeout=timeout) message = await asyncio.wait_for(socket.recv(), timeout=timeout)
except (asyncio.TimeoutError, Exception): except (TimeoutError, Exception):
break break
total += len(message) if isinstance(message, (bytes, bytearray)) else len(message.encode()) total += len(message) if isinstance(message, (bytes, bytearray)) else len(message.encode())
elapsed = time.perf_counter() - began elapsed = time.perf_counter() - began
@ -233,16 +273,21 @@ async def tier_ndt7(
) )
from netsleuth.config import BufferbloatBands
from netsleuth.interpret import grade_bufferbloat
from netsleuth.stats import rtt_stats
async def probe_while( async def probe_while(
coro: Awaitable[None], coro: Awaitable[None],
probe: Callable[[], Awaitable[float | None]], probe: Callable[[], Awaitable[float | None]],
interval: float, interval: float,
) -> list[float]: ) -> list[float]:
"""Run probes concurrently with a background task.
Args:
coro: Background coroutine to run (e.g., download/upload)
probe: Async function that returns RTT measurement or None
interval: Time between probes in seconds
Returns:
List of RTT measurements collected during execution
"""
samples: list[float] = [] samples: list[float] = []
task = asyncio.ensure_future(coro) task = asyncio.ensure_future(coro)
while not task.done(): while not task.done():
@ -263,6 +308,23 @@ async def measure_with_bufferbloat(
probe: Callable[[], Awaitable[float | None]], probe: Callable[[], Awaitable[float | None]],
interval: float, interval: float,
) -> SpeedResult: ) -> SpeedResult:
"""Measure bufferbloat by sampling RTT during saturated transfers.
This is the one place a measurement is allowed to overlap another: the
whole point is to see what latency does while the link is saturated.
Args:
result: SpeedResult to update with bufferbloat metrics
idle_rtt_ms: Idle RTT baseline in milliseconds
bands: Bufferbloat grading thresholds
run_download: Coroutine to saturate download
run_upload: Coroutine to saturate upload
probe: Function to measure RTT
interval: Probe interval in seconds
Returns:
Updated SpeedResult with bufferbloat metrics
"""
# This is the one place a measurement is allowed to overlap another: the # This is the one place a measurement is allowed to overlap another: the
# whole point is to see what latency does while the link is saturated. # whole point is to see what latency does while the link is saturated.
down_samples = await probe_while(run_download(), probe, interval) down_samples = await probe_while(run_download(), probe, interval)

View file

@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import itertools
from typing import NamedTuple from typing import NamedTuple
@ -25,7 +26,7 @@ def rtt_stats(samples: list[float | None]) -> RttStats:
mdev = sum(abs(v - avg) for v in got) / received mdev = sum(abs(v - avg) for v in got) / received
deltas = [ deltas = [
abs(b - a) abs(b - a)
for a, b in zip(samples, samples[1:]) for a, b in itertools.pairwise(samples)
if a is not None and b is not None if a is not None and b is not None
] ]
jitter = sum(deltas) / len(deltas) if deltas else 0.0 jitter = sum(deltas) / len(deltas) if deltas else 0.0

View file

@ -13,7 +13,13 @@ from rich.table import Table
from netsleuth import __version__ from netsleuth import __version__
from netsleuth.config import Settings from netsleuth.config import Settings
from netsleuth.exporter import atomic_write, compact_timestamp, dump_json, sanitize_name, sparkline from netsleuth.exporter import (
atomic_write,
compact_timestamp,
dump_json,
sanitize_name,
sparkline,
)
from netsleuth.ip_geo import gather_identity from netsleuth.ip_geo import gather_identity
from netsleuth.models import PingResult, SpeedResult, to_jsonable from netsleuth.models import PingResult, SpeedResult, to_jsonable
from netsleuth.netinfo import detect_capabilities from netsleuth.netinfo import detect_capabilities