diff --git a/src/netcheck/ip_geo.py b/src/netcheck/ip_geo.py new file mode 100644 index 0000000..1038e60 --- /dev/null +++ b/src/netcheck/ip_geo.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import ipaddress +import re + +from netcheck.models import CfTrace, IpGeo + +_AS_PREFIX_RE = re.compile(r"^AS(\d+)\s*(?P.*)$", re.IGNORECASE) + + +def _as_number(value: object) -> str | None: + if value is None or value == "": + return None + text = str(value).strip() + match = _AS_PREFIX_RE.match(text) + if match: + return f"AS{match.group(1)}" + if text.isdigit(): + return f"AS{text}" + return None + + +def _as_label(value: object) -> str | None: + match = _AS_PREFIX_RE.match(str(value or "")) + name = (match.group("name") if match else "").strip() + return name or None + + +def _ip_version(ip: str | None) -> int | None: + if not ip: + return None + try: + return ipaddress.ip_address(ip).version + except ValueError: + return None + + +def classify_ip_type(mobile: bool, proxy: bool, hosting: bool, known: bool) -> str: + if not known: + return "unknown" + if mobile: + return "mobile" + if hosting or proxy: + return "hosting" + return "residential" + + +def provider_flags(payload: dict) -> dict[str, bool]: + if payload.get("status") != "success": + return {} + return { + "mobile": bool(payload.get("mobile")), + "proxy": bool(payload.get("proxy")), + "hosting": bool(payload.get("hosting")), + } + + +def normalize_ip_api(payload: dict) -> IpGeo: + if payload.get("status") != "success": + return IpGeo(ip=payload.get("query"), ip_version=_ip_version(payload.get("query"))) + ip = payload.get("query") + return IpGeo( + ip=ip, + ip_version=_ip_version(ip), + reverse_dns=payload.get("reverse") or None, + asn=_as_number(payload.get("as")), + as_name=_as_label(payload.get("as")), + org=payload.get("org") or payload.get("isp") or None, + country=payload.get("country"), + country_code=payload.get("countryCode"), + city=payload.get("city"), + lat=payload.get("lat"), + lon=payload.get("lon"), + timezone=payload.get("timezone"), + ip_type=classify_ip_type( + bool(payload.get("mobile")), bool(payload.get("proxy")), bool(payload.get("hosting")), True + ), + sources={"provider": "ip-api"}, + ) + + +def normalize_freeipapi(payload: dict) -> IpGeo: + ip = payload.get("ipAddress") + return IpGeo( + ip=ip, + ip_version=payload.get("ipVersion") or _ip_version(ip), + asn=_as_number(payload.get("asn")), + org=payload.get("asnOrganization") or None, + country=payload.get("countryName"), + country_code=payload.get("countryCode"), + city=payload.get("cityName"), + lat=payload.get("latitude"), + lon=payload.get("longitude"), + ip_type=classify_ip_type(False, bool(payload.get("isProxy")), False, True), + sources={"provider": "freeipapi"}, + ) + + +def normalize_ipinfo(payload: dict) -> IpGeo: + lat = lon = None + loc = payload.get("loc") + if isinstance(loc, str) and "," in loc: + raw_lat, _, raw_lon = loc.partition(",") + try: + lat, lon = float(raw_lat), float(raw_lon) + except ValueError: + lat = lon = None + ip = payload.get("ip") + return IpGeo( + ip=ip, + ip_version=_ip_version(ip), + reverse_dns=payload.get("hostname") or None, + asn=_as_number(payload.get("org")), + as_name=_as_label(payload.get("org")), + org=_as_label(payload.get("org")), + country_code=payload.get("country"), + city=payload.get("city"), + lat=lat, + lon=lon, + timezone=payload.get("timezone"), + sources={"provider": "ipinfo"}, + ) + + +def normalize_ipwhois(payload: dict) -> IpGeo: + if not payload.get("success", True): + return IpGeo(ip=payload.get("ip")) + connection = payload.get("connection") or {} + timezone = payload.get("timezone") or {} + ip = payload.get("ip") + return IpGeo( + ip=ip, + ip_version=_ip_version(ip), + asn=_as_number(connection.get("asn")), + as_name=connection.get("isp") or None, + org=connection.get("org") or None, + country=payload.get("country"), + country_code=payload.get("country_code"), + city=payload.get("city"), + lat=payload.get("latitude"), + lon=payload.get("longitude"), + timezone=timezone.get("id") if isinstance(timezone, dict) else timezone, + sources={"provider": "ipwho.is"}, + ) + + +def normalize_ripestat_network_info(payload: dict) -> IpGeo: + data = payload.get("data") or {} + asns = data.get("asns") or [] + return IpGeo(asn=_as_number(asns[0]) if asns else None, sources={"asn": "ripestat"}) + + +def parse_cf_trace(text: str) -> CfTrace: + raw: dict[str, str] = {} + for line in text.splitlines(): + if "=" not in line: + continue + key, _, value = line.partition("=") + raw[key.strip()] = value.strip() + return CfTrace( + ip=raw.get("ip"), + colo=raw.get("colo"), + loc=raw.get("loc"), + warp=raw.get("warp"), + gateway=raw.get("gateway"), + rbi=raw.get("rbi"), + raw=raw, + ) diff --git a/tests/fixtures/api/cf_trace.txt b/tests/fixtures/api/cf_trace.txt new file mode 100644 index 0000000..9e3a02c --- /dev/null +++ b/tests/fixtures/api/cf_trace.txt @@ -0,0 +1,16 @@ +fl=123f45 +h=www.cloudflare.com +ip=203.0.113.44 +ts=1786000000.123 +visit_scheme=https +uag=netcheck/0.1.0 +colo=AMS +sliver=none +http=http/2 +loc=NL +tls=TLSv1.3 +sni=plaintext +warp=off +gateway=off +rbi=off +kex=X25519 diff --git a/tests/fixtures/api/freeipapi.json b/tests/fixtures/api/freeipapi.json new file mode 100644 index 0000000..d1508c9 --- /dev/null +++ b/tests/fixtures/api/freeipapi.json @@ -0,0 +1,16 @@ +{ + "ipVersion": 4, + "ipAddress": "203.0.113.44", + "latitude": 52.3759, + "longitude": 4.8975, + "countryName": "Netherlands", + "countryCode": "NL", + "timeZone": "+02:00", + "zipCode": "1012", + "cityName": "Amsterdam", + "regionName": "North Holland", + "isProxy": false, + "continent": "Europe", + "asn": "64500", + "asnOrganization": "Example Telecom BV" +} diff --git a/tests/fixtures/api/ip_api.json b/tests/fixtures/api/ip_api.json new file mode 100644 index 0000000..794c519 --- /dev/null +++ b/tests/fixtures/api/ip_api.json @@ -0,0 +1,21 @@ +{ + "status": "success", + "country": "Netherlands", + "countryCode": "NL", + "region": "NH", + "regionName": "North Holland", + "city": "Amsterdam", + "zip": "1012", + "lat": 52.3759, + "lon": 4.8975, + "timezone": "Europe/Amsterdam", + "isp": "Example Telecom", + "org": "Example Telecom BV", + "as": "AS64500 Example Telecom", + "asname": "EXAMPLE-AS", + "reverse": "host-203-0-113-44.example.net", + "mobile": false, + "proxy": false, + "hosting": true, + "query": "203.0.113.44" +} diff --git a/tests/fixtures/api/ip_api_fail.json b/tests/fixtures/api/ip_api_fail.json new file mode 100644 index 0000000..868c46e --- /dev/null +++ b/tests/fixtures/api/ip_api_fail.json @@ -0,0 +1,5 @@ +{ + "status": "fail", + "message": "private range", + "query": "192.168.1.34" +} diff --git a/tests/fixtures/api/ipinfo.json b/tests/fixtures/api/ipinfo.json new file mode 100644 index 0000000..bd948c1 --- /dev/null +++ b/tests/fixtures/api/ipinfo.json @@ -0,0 +1,11 @@ +{ + "ip": "203.0.113.44", + "hostname": "host-203-0-113-44.example.net", + "city": "Amsterdam", + "region": "North Holland", + "country": "NL", + "loc": "52.3759,4.8975", + "org": "AS64500 Example Telecom BV", + "postal": "1012", + "timezone": "Europe/Amsterdam" +} diff --git a/tests/fixtures/api/ipwhois.json b/tests/fixtures/api/ipwhois.json new file mode 100644 index 0000000..a3e4478 --- /dev/null +++ b/tests/fixtures/api/ipwhois.json @@ -0,0 +1,19 @@ +{ + "ip": "203.0.113.44", + "success": true, + "type": "IPv4", + "continent": "Europe", + "country": "Netherlands", + "country_code": "NL", + "region": "North Holland", + "city": "Amsterdam", + "latitude": 52.3759, + "longitude": 4.8975, + "connection": { + "asn": 64500, + "org": "Example Telecom BV", + "isp": "Example Telecom", + "domain": "example.net" + }, + "timezone": { "id": "Europe/Amsterdam", "utc": "+02:00" } +} diff --git a/tests/fixtures/api/ripestat_network_info.json b/tests/fixtures/api/ripestat_network_info.json new file mode 100644 index 0000000..bbff513 --- /dev/null +++ b/tests/fixtures/api/ripestat_network_info.json @@ -0,0 +1,8 @@ +{ + "status": "ok", + "data_call_name": "network-info", + "data": { + "asns": ["64500"], + "prefix": "203.0.113.0/24" + } +} diff --git a/tests/test_ip_geo_normalize.py b/tests/test_ip_geo_normalize.py new file mode 100644 index 0000000..5202909 --- /dev/null +++ b/tests/test_ip_geo_normalize.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import pytest + +from netcheck.ip_geo import ( + classify_ip_type, + normalize_freeipapi, + normalize_ip_api, + normalize_ipinfo, + normalize_ipwhois, + normalize_ripestat_network_info, + parse_cf_trace, + provider_flags, +) + + +def test_ip_api_normalizes_to_the_common_shape(api_fixture): + geo = normalize_ip_api(api_fixture("ip_api.json")) + assert geo.ip == "203.0.113.44" + assert geo.ip_version == 4 + assert geo.asn == "AS64500" + assert geo.as_name == "Example Telecom" + assert geo.org == "Example Telecom BV" + assert geo.country == "Netherlands" + assert geo.country_code == "NL" + assert geo.city == "Amsterdam" + assert geo.lat == 52.3759 + assert geo.lon == 4.8975 + assert geo.timezone == "Europe/Amsterdam" + assert geo.reverse_dns == "host-203-0-113-44.example.net" + assert geo.ip_type == "hosting" + + +def test_ip_api_failure_payload_yields_an_empty_geo_not_an_exception(api_fixture): + geo = normalize_ip_api(api_fixture("ip_api_fail.json")) + assert geo.asn is None + assert geo.country is None + assert geo.ip_type == "unknown" + + +def test_freeipapi_normalizes_and_prefixes_the_bare_asn(api_fixture): + geo = normalize_freeipapi(api_fixture("freeipapi.json")) + assert geo.ip == "203.0.113.44" + assert geo.ip_version == 4 + assert geo.asn == "AS64500" + assert geo.org == "Example Telecom BV" + assert geo.country_code == "NL" + assert geo.city == "Amsterdam" + assert geo.ip_type == "residential" + + +def test_ipinfo_splits_the_org_field_into_asn_and_name(api_fixture): + geo = normalize_ipinfo(api_fixture("ipinfo.json")) + assert geo.asn == "AS64500" + assert geo.as_name == "Example Telecom BV" + assert geo.lat == 52.3759 + assert geo.lon == 4.8975 + assert geo.country_code == "NL" + assert geo.reverse_dns == "host-203-0-113-44.example.net" + + +def test_ipinfo_without_a_loc_field_does_not_crash(): + geo = normalize_ipinfo({"ip": "203.0.113.44", "org": "AS64500 Example"}) + assert geo.lat is None + assert geo.lon is None + + +def test_ipwhois_reads_the_nested_connection_and_timezone_objects(api_fixture): + geo = normalize_ipwhois(api_fixture("ipwhois.json")) + assert geo.asn == "AS64500" + assert geo.as_name == "Example Telecom" + assert geo.org == "Example Telecom BV" + assert geo.timezone == "Europe/Amsterdam" + assert geo.ip_version == 4 + assert geo.country_code == "NL" + + +def test_ripestat_network_info_gives_authoritative_asn_only(api_fixture): + geo = normalize_ripestat_network_info(api_fixture("ripestat_network_info.json")) + assert geo.asn == "AS64500" + assert geo.city is None + assert geo.sources == {"asn": "ripestat"} + + +def test_cf_trace_parses_every_key_and_the_vpn_relevant_flags(fixtures_dir): + cf = parse_cf_trace((fixtures_dir / "api" / "cf_trace.txt").read_text(encoding="utf-8")) + assert cf.ip == "203.0.113.44" + assert cf.colo == "AMS" + assert cf.loc == "NL" + assert cf.warp == "off" + assert cf.gateway == "off" + assert cf.rbi == "off" + assert cf.raw["tls"] == "TLSv1.3" + + +def test_cf_trace_tolerates_blank_lines_and_missing_keys(): + cf = parse_cf_trace("ip=1.2.3.4\n\nnot-a-pair\ncolo=AMS\n") + assert cf.ip == "1.2.3.4" + assert cf.colo == "AMS" + assert cf.warp is None + + +def test_provider_flags_reads_the_ip_api_booleans(api_fixture): + assert provider_flags(api_fixture("ip_api.json")) == { + "mobile": False, + "proxy": False, + "hosting": True, + } + + +def test_provider_flags_of_a_failed_payload_is_empty(api_fixture): + assert provider_flags(api_fixture("ip_api_fail.json")) == {} + + +@pytest.mark.parametrize( + ("mobile", "proxy", "hosting", "known", "expected"), + [ + (False, False, False, False, "unknown"), + (False, False, False, True, "residential"), + (True, False, False, True, "mobile"), + (True, False, True, True, "mobile"), + (False, True, False, True, "hosting"), + (False, False, True, True, "hosting"), + ], +) +def test_ip_type_classification(mobile, proxy, hosting, known, expected): + assert classify_ip_type(mobile, proxy, hosting, known) == expected