ip_geo: per-provider normalizers to a common IpGeo shape

src/netcheck/ip_geo.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import ipaddress
+import re
+
+from netcheck.models import CfTrace, IpGeo
+
+_AS_PREFIX_RE = re.compile(r"^AS(\d+)\s*(?P<name>.*)$", re.IGNORECASE)
+
+
+def _as_number(value: object) -> str | None:
+    if value is None or value == "":
+        return None
+    text = str(value).strip()
+    match = _AS_PREFIX_RE.match(text)
+    if match:
+        return f"AS{match.group(1)}"
+    if text.isdigit():
+        return f"AS{text}"
+    return None
+
+
+def _as_label(value: object) -> str | None:
+    match = _AS_PREFIX_RE.match(str(value or ""))
+    name = (match.group("name") if match else "").strip()
+    return name or None
+
+
+def _ip_version(ip: str | None) -> int | None:
+    if not ip:
+        return None
+    try:
+        return ipaddress.ip_address(ip).version
+    except ValueError:
+        return None
+
+
+def classify_ip_type(mobile: bool, proxy: bool, hosting: bool, known: bool) -> str:
+    if not known:
+        return "unknown"
+    if mobile:
+        return "mobile"
+    if hosting or proxy:
+        return "hosting"
+    return "residential"
+
+
+def provider_flags(payload: dict) -> dict[str, bool]:
+    if payload.get("status") != "success":
+        return {}
+    return {
+        "mobile": bool(payload.get("mobile")),
+        "proxy": bool(payload.get("proxy")),
+        "hosting": bool(payload.get("hosting")),
+    }
+
+
+def normalize_ip_api(payload: dict) -> IpGeo:
+    if payload.get("status") != "success":
+        return IpGeo(ip=payload.get("query"), ip_version=_ip_version(payload.get("query")))
+    ip = payload.get("query")
+    return IpGeo(
+        ip=ip,
+        ip_version=_ip_version(ip),
+        reverse_dns=payload.get("reverse") or None,
+        asn=_as_number(payload.get("as")),
+        as_name=_as_label(payload.get("as")),
+        org=payload.get("org") or payload.get("isp") or None,
+        country=payload.get("country"),
+        country_code=payload.get("countryCode"),
+        city=payload.get("city"),
+        lat=payload.get("lat"),
+        lon=payload.get("lon"),
+        timezone=payload.get("timezone"),
+        ip_type=classify_ip_type(
+            bool(payload.get("mobile")), bool(payload.get("proxy")), bool(payload.get("hosting")), True
+        ),
+        sources={"provider": "ip-api"},
+    )
+
+
+def normalize_freeipapi(payload: dict) -> IpGeo:
+    ip = payload.get("ipAddress")
+    return IpGeo(
+        ip=ip,
+        ip_version=payload.get("ipVersion") or _ip_version(ip),
+        asn=_as_number(payload.get("asn")),
+        org=payload.get("asnOrganization") or None,
+        country=payload.get("countryName"),
+        country_code=payload.get("countryCode"),
+        city=payload.get("cityName"),
+        lat=payload.get("latitude"),
+        lon=payload.get("longitude"),
+        ip_type=classify_ip_type(False, bool(payload.get("isProxy")), False, True),
+        sources={"provider": "freeipapi"},
+    )
+
+
+def normalize_ipinfo(payload: dict) -> IpGeo:
+    lat = lon = None
+    loc = payload.get("loc")
+    if isinstance(loc, str) and "," in loc:
+        raw_lat, _, raw_lon = loc.partition(",")
+        try:
+            lat, lon = float(raw_lat), float(raw_lon)
+        except ValueError:
+            lat = lon = None
+    ip = payload.get("ip")
+    return IpGeo(
+        ip=ip,
+        ip_version=_ip_version(ip),
+        reverse_dns=payload.get("hostname") or None,
+        asn=_as_number(payload.get("org")),
+        as_name=_as_label(payload.get("org")),
+        org=_as_label(payload.get("org")),
+        country_code=payload.get("country"),
+        city=payload.get("city"),
+        lat=lat,
+        lon=lon,
+        timezone=payload.get("timezone"),
+        sources={"provider": "ipinfo"},
+    )
+
+
+def normalize_ipwhois(payload: dict) -> IpGeo:
+    if not payload.get("success", True):
+        return IpGeo(ip=payload.get("ip"))
+    connection = payload.get("connection") or {}
+    timezone = payload.get("timezone") or {}
+    ip = payload.get("ip")
+    return IpGeo(
+        ip=ip,
+        ip_version=_ip_version(ip),
+        asn=_as_number(connection.get("asn")),
+        as_name=connection.get("isp") or None,
+        org=connection.get("org") or None,
+        country=payload.get("country"),
+        country_code=payload.get("country_code"),
+        city=payload.get("city"),
+        lat=payload.get("latitude"),
+        lon=payload.get("longitude"),
+        timezone=timezone.get("id") if isinstance(timezone, dict) else timezone,
+        sources={"provider": "ipwho.is"},
+    )
+
+
+def normalize_ripestat_network_info(payload: dict) -> IpGeo:
+    data = payload.get("data") or {}
+    asns = data.get("asns") or []
+    return IpGeo(asn=_as_number(asns[0]) if asns else None, sources={"asn": "ripestat"})
+
+
+def parse_cf_trace(text: str) -> CfTrace:
+    raw: dict[str, str] = {}
+    for line in text.splitlines():
+        if "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        raw[key.strip()] = value.strip()
+    return CfTrace(
+        ip=raw.get("ip"),
+        colo=raw.get("colo"),
+        loc=raw.get("loc"),
+        warp=raw.get("warp"),
+        gateway=raw.get("gateway"),
+        rbi=raw.get("rbi"),
+        raw=raw,
+    )

tests/fixtures/api/cf_trace.txt

@@ -0,0 +1,16 @@
+fl=123f45
+h=www.cloudflare.com
+ip=203.0.113.44
+ts=1786000000.123
+visit_scheme=https
+uag=netcheck/0.1.0
+colo=AMS
+sliver=none
+http=http/2
+loc=NL
+tls=TLSv1.3
+sni=plaintext
+warp=off
+gateway=off
+rbi=off
+kex=X25519

tests/fixtures/api/freeipapi.json

@@ -0,0 +1,16 @@
+{
+  "ipVersion": 4,
+  "ipAddress": "203.0.113.44",
+  "latitude": 52.3759,
+  "longitude": 4.8975,
+  "countryName": "Netherlands",
+  "countryCode": "NL",
+  "timeZone": "+02:00",
+  "zipCode": "1012",
+  "cityName": "Amsterdam",
+  "regionName": "North Holland",
+  "isProxy": false,
+  "continent": "Europe",
+  "asn": "64500",
+  "asnOrganization": "Example Telecom BV"
+}

tests/fixtures/api/ip_api.json

@@ -0,0 +1,21 @@
+{
+  "status": "success",
+  "country": "Netherlands",
+  "countryCode": "NL",
+  "region": "NH",
+  "regionName": "North Holland",
+  "city": "Amsterdam",
+  "zip": "1012",
+  "lat": 52.3759,
+  "lon": 4.8975,
+  "timezone": "Europe/Amsterdam",
+  "isp": "Example Telecom",
+  "org": "Example Telecom BV",
+  "as": "AS64500 Example Telecom",
+  "asname": "EXAMPLE-AS",
+  "reverse": "host-203-0-113-44.example.net",
+  "mobile": false,
+  "proxy": false,
+  "hosting": true,
+  "query": "203.0.113.44"
+}

tests/fixtures/api/ip_api_fail.json

@@ -0,0 +1,5 @@
+{
+  "status": "fail",
+  "message": "private range",
+  "query": "192.168.1.34"
+}

tests/fixtures/api/ipinfo.json

@@ -0,0 +1,11 @@
+{
+  "ip": "203.0.113.44",
+  "hostname": "host-203-0-113-44.example.net",
+  "city": "Amsterdam",
+  "region": "North Holland",
+  "country": "NL",
+  "loc": "52.3759,4.8975",
+  "org": "AS64500 Example Telecom BV",
+  "postal": "1012",
+  "timezone": "Europe/Amsterdam"
+}

tests/fixtures/api/ipwhois.json

@@ -0,0 +1,19 @@
+{
+  "ip": "203.0.113.44",
+  "success": true,
+  "type": "IPv4",
+  "continent": "Europe",
+  "country": "Netherlands",
+  "country_code": "NL",
+  "region": "North Holland",
+  "city": "Amsterdam",
+  "latitude": 52.3759,
+  "longitude": 4.8975,
+  "connection": {
+    "asn": 64500,
+    "org": "Example Telecom BV",
+    "isp": "Example Telecom",
+    "domain": "example.net"
+  },
+  "timezone": { "id": "Europe/Amsterdam", "utc": "+02:00" }
+}

tests/fixtures/api/ripestat_network_info.json

@@ -0,0 +1,8 @@
+{
+  "status": "ok",
+  "data_call_name": "network-info",
+  "data": {
+    "asns": ["64500"],
+    "prefix": "203.0.113.0/24"
+  }
+}

tests/test_ip_geo_normalize.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import pytest
+
+from netcheck.ip_geo import (
+    classify_ip_type,
+    normalize_freeipapi,
+    normalize_ip_api,
+    normalize_ipinfo,
+    normalize_ipwhois,
+    normalize_ripestat_network_info,
+    parse_cf_trace,
+    provider_flags,
+)
+
+
+def test_ip_api_normalizes_to_the_common_shape(api_fixture):
+    geo = normalize_ip_api(api_fixture("ip_api.json"))
+    assert geo.ip == "203.0.113.44"
+    assert geo.ip_version == 4
+    assert geo.asn == "AS64500"
+    assert geo.as_name == "Example Telecom"
+    assert geo.org == "Example Telecom BV"
+    assert geo.country == "Netherlands"
+    assert geo.country_code == "NL"
+    assert geo.city == "Amsterdam"
+    assert geo.lat == 52.3759
+    assert geo.lon == 4.8975
+    assert geo.timezone == "Europe/Amsterdam"
+    assert geo.reverse_dns == "host-203-0-113-44.example.net"
+    assert geo.ip_type == "hosting"
+
+
+def test_ip_api_failure_payload_yields_an_empty_geo_not_an_exception(api_fixture):
+    geo = normalize_ip_api(api_fixture("ip_api_fail.json"))
+    assert geo.asn is None
+    assert geo.country is None
+    assert geo.ip_type == "unknown"
+
+
+def test_freeipapi_normalizes_and_prefixes_the_bare_asn(api_fixture):
+    geo = normalize_freeipapi(api_fixture("freeipapi.json"))
+    assert geo.ip == "203.0.113.44"
+    assert geo.ip_version == 4
+    assert geo.asn == "AS64500"
+    assert geo.org == "Example Telecom BV"
+    assert geo.country_code == "NL"
+    assert geo.city == "Amsterdam"
+    assert geo.ip_type == "residential"
+
+
+def test_ipinfo_splits_the_org_field_into_asn_and_name(api_fixture):
+    geo = normalize_ipinfo(api_fixture("ipinfo.json"))
+    assert geo.asn == "AS64500"
+    assert geo.as_name == "Example Telecom BV"
+    assert geo.lat == 52.3759
+    assert geo.lon == 4.8975
+    assert geo.country_code == "NL"
+    assert geo.reverse_dns == "host-203-0-113-44.example.net"
+
+
+def test_ipinfo_without_a_loc_field_does_not_crash():
+    geo = normalize_ipinfo({"ip": "203.0.113.44", "org": "AS64500 Example"})
+    assert geo.lat is None
+    assert geo.lon is None
+
+
+def test_ipwhois_reads_the_nested_connection_and_timezone_objects(api_fixture):
+    geo = normalize_ipwhois(api_fixture("ipwhois.json"))
+    assert geo.asn == "AS64500"
+    assert geo.as_name == "Example Telecom"
+    assert geo.org == "Example Telecom BV"
+    assert geo.timezone == "Europe/Amsterdam"
+    assert geo.ip_version == 4
+    assert geo.country_code == "NL"
+
+
+def test_ripestat_network_info_gives_authoritative_asn_only(api_fixture):
+    geo = normalize_ripestat_network_info(api_fixture("ripestat_network_info.json"))
+    assert geo.asn == "AS64500"
+    assert geo.city is None
+    assert geo.sources == {"asn": "ripestat"}
+
+
+def test_cf_trace_parses_every_key_and_the_vpn_relevant_flags(fixtures_dir):
+    cf = parse_cf_trace((fixtures_dir / "api" / "cf_trace.txt").read_text(encoding="utf-8"))
+    assert cf.ip == "203.0.113.44"
+    assert cf.colo == "AMS"
+    assert cf.loc == "NL"
+    assert cf.warp == "off"
+    assert cf.gateway == "off"
+    assert cf.rbi == "off"
+    assert cf.raw["tls"] == "TLSv1.3"
+
+
+def test_cf_trace_tolerates_blank_lines_and_missing_keys():
+    cf = parse_cf_trace("ip=1.2.3.4\n\nnot-a-pair\ncolo=AMS\n")
+    assert cf.ip == "1.2.3.4"
+    assert cf.colo == "AMS"
+    assert cf.warp is None
+
+
+def test_provider_flags_reads_the_ip_api_booleans(api_fixture):
+    assert provider_flags(api_fixture("ip_api.json")) == {
+        "mobile": False,
+        "proxy": False,
+        "hosting": True,
+    }
+
+
+def test_provider_flags_of_a_failed_payload_is_empty(api_fixture):
+    assert provider_flags(api_fixture("ip_api_fail.json")) == {}
+
+
+@pytest.mark.parametrize(
+    ("mobile", "proxy", "hosting", "known", "expected"),
+    [
+        (False, False, False, False, "unknown"),
+        (False, False, False, True, "residential"),
+        (True, False, False, True, "mobile"),
+        (True, False, True, True, "mobile"),
+        (False, True, False, True, "hosting"),
+        (False, False, True, True, "hosting"),
+    ],
+)
+def test_ip_type_classification(mobile, proxy, hosting, known, expected):
+    assert classify_ip_type(mobile, proxy, hosting, known) == expected