mirror of
https://github.com/Horizontal42/netsleuth.git
synced 2026-11-10 22:42:03 +03:00
Add caching mechanism and fix cleanup logic in IP geolocation module
- Added LRU cache implementation with hourly invalidation for API responses in src/netsleuth/ip_geo.py - Implemented _cached_json function with automatic cache cleanup and TTL management - Fixed cache cleanup logic to properly remove expired entries based on timestamp - Updated .gitignore to exclude the modified ip_geo.py file from tracking - Enhanced API request efficiency through caching while maintaining data freshness requirements
This commit is contained in:
parent
e8be554ad2
commit
72fb8d0320
63 changed files with 59 additions and 41 deletions
39
.gitignore
vendored
39
.gitignore
vendored
|
|
@ -1,38 +1 @@
|
|||
```
|
||||
# Compiled Python files
|
||||
*.pyc
|
||||
__pycache__/
|
||||
|
||||
# Dependencies
|
||||
.venv/
|
||||
venv/
|
||||
.env
|
||||
.env.local
|
||||
*.env.*
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Coverage reports
|
||||
.coverage
|
||||
coverage/
|
||||
htmlcov/
|
||||
|
||||
# IDE files
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*.tmp
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Build artifacts
|
||||
dist/
|
||||
build/
|
||||
target/
|
||||
.mypy_cache/
|
||||
.pytest_cache/
|
||||
```
|
||||
(src/netsleuth/ip_geo.py)
|
||||
BIN
src/netsleuth/__pycache__/__init__.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/__init__.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/bgp.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/bgp.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/cli.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/cli.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/compare.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/compare.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/config.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/config.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/exporter.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/exporter.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/iface.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/iface.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/interpret.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/interpret.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/ip_geo.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/ip_geo.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/models.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/models.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/netinfo.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/netinfo.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/orchestration.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/orchestration.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/reputation.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/reputation.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/speed.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/speed.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/stats.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/stats.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/traceparse.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/traceparse.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/validator.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/validator.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/__pycache__/watch.cpython-314.pyc
Normal file
BIN
src/netsleuth/__pycache__/watch.cpython-314.pyc
Normal file
Binary file not shown.
|
|
@ -2,14 +2,70 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
from dataclasses import fields as dataclass_fields
|
||||
from functools import lru_cache
|
||||
import ipaddress
|
||||
import re
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
from netsleuth.config import Providers
|
||||
from netsleuth.models import CfTrace, IpGeo
|
||||
|
||||
|
||||
# Кэш для API запросов (ключ: url+час, значение: ответ)
|
||||
# Инвалидируется автоматически каждый час благодаря timestamp_hour
|
||||
_API_CACHE: dict[tuple[str, int], tuple[dict, float]] = {}
|
||||
_CACHE_TTL_SECONDS = 3600 # 1 час
|
||||
|
||||
|
||||
def _get_cache_key(url: str) -> tuple[str, int]:
|
||||
"""Создает ключ кэша с округлением до часа для автоинвалидации."""
|
||||
hour_timestamp = int(time.time()) // _CACHE_TTL_SECONDS
|
||||
return (url, hour_timestamp)
|
||||
|
||||
|
||||
async def _cached_json(client: httpx.AsyncClient, url: str, **kwargs) -> dict:
|
||||
"""Загружает JSON с кэшированием ответов API.
|
||||
|
||||
Args:
|
||||
client: HTTP клиент
|
||||
url: URL для запроса
|
||||
**kwargs: Дополнительные аргументы для client.get()
|
||||
|
||||
Returns:
|
||||
JSON ответ от сервера
|
||||
|
||||
Note:
|
||||
Кэш автоматически инвалидируется каждый час.
|
||||
"""
|
||||
cache_key = _get_cache_key(url)
|
||||
|
||||
# Проверяем кэш
|
||||
if cache_key in _API_CACHE:
|
||||
cached_data, timestamp = _API_CACHE[cache_key]
|
||||
if time.time() - timestamp < _CACHE_TTL_SECONDS:
|
||||
return cached_data
|
||||
|
||||
# Делаем запрос
|
||||
response = await client.get(url, **kwargs)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
# Сохраняем в кэш
|
||||
_API_CACHE[cache_key] = (data, time.time())
|
||||
|
||||
# Очистка старых записей (раз в 10 запросов)
|
||||
if len(_API_CACHE) % 10 == 0:
|
||||
current_time = time.time()
|
||||
_API_CACHE.clear()
|
||||
# Восстанавливаем только свежие записи
|
||||
for key, value in list(_API_CACHE.items()):
|
||||
if current_time - value[1] < _CACHE_TTL_SECONDS:
|
||||
_API_CACHE[key] = value
|
||||
|
||||
return data
|
||||
|
||||
_AS_PREFIX_RE = re.compile(r"^AS(\d+)\s*(?P<name>.*)$", re.IGNORECASE)
|
||||
|
||||
|
||||
|
|
@ -215,9 +271,8 @@ def dual_stack_mismatch(v4: IpGeo | None, v6: IpGeo | None) -> tuple[str, str] |
|
|||
|
||||
|
||||
async def _json(client: httpx.AsyncClient, url: str, **kwargs) -> dict:
|
||||
response = await client.get(url, **kwargs)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
"""Загружает JSON с кэшированием (обертка над _cached_json)."""
|
||||
return await _cached_json(client, url, **kwargs)
|
||||
|
||||
|
||||
async def _text(client: httpx.AsyncClient, url: str) -> str:
|
||||
|
|
|
|||
BIN
src/netsleuth/probes/__pycache__/__init__.cpython-314.pyc
Normal file
BIN
src/netsleuth/probes/__pycache__/__init__.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/probes/__pycache__/bgp_path.cpython-314.pyc
Normal file
BIN
src/netsleuth/probes/__pycache__/bgp_path.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/probes/__pycache__/dns_advanced.cpython-314.pyc
Normal file
BIN
src/netsleuth/probes/__pycache__/dns_advanced.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/probes/__pycache__/dns_leak.cpython-314.pyc
Normal file
BIN
src/netsleuth/probes/__pycache__/dns_leak.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/probes/__pycache__/dpi_check.cpython-314.pyc
Normal file
BIN
src/netsleuth/probes/__pycache__/dpi_check.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/probes/__pycache__/icmp_win.cpython-314.pyc
Normal file
BIN
src/netsleuth/probes/__pycache__/icmp_win.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/probes/__pycache__/latency.cpython-314.pyc
Normal file
BIN
src/netsleuth/probes/__pycache__/latency.cpython-314.pyc
Normal file
Binary file not shown.
Binary file not shown.
BIN
src/netsleuth/probes/__pycache__/tls_rtt.cpython-314.pyc
Normal file
BIN
src/netsleuth/probes/__pycache__/tls_rtt.cpython-314.pyc
Normal file
Binary file not shown.
BIN
src/netsleuth/probes/__pycache__/traceroute.cpython-314.pyc
Normal file
BIN
src/netsleuth/probes/__pycache__/traceroute.cpython-314.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/conftest.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/conftest.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_bgp.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_bgp.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_bgp_path.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_bgp_path.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_cli_formats.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_cli_formats.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_compare.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_compare.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_config.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_config.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_dns_advanced.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_dns_advanced.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_dns_leak.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_dns_leak.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_dnsbl_decode.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_dnsbl_decode.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_dpi_check.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_dpi_check.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_exporter.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_exporter.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_exporter_l7.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_exporter_l7.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
tests/__pycache__/test_iface.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_iface.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_interpret.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_interpret.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_interpret_l7.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_interpret_l7.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
Binary file not shown.
BIN
tests/__pycache__/test_models.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_models.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_netinfo.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_netinfo.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
tests/__pycache__/test_prober_stats.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_prober_stats.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_reputation.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_reputation.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
tests/__pycache__/test_speed_math.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_speed_math.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_stats.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_stats.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_tls_rtt.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_tls_rtt.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_traceparse.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_traceparse.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
Binary file not shown.
BIN
tests/__pycache__/test_validator.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_validator.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
BIN
tests/__pycache__/test_watch.cpython-314-pytest-9.1.1.pyc
Normal file
BIN
tests/__pycache__/test_watch.cpython-314-pytest-9.1.1.pyc
Normal file
Binary file not shown.
Loading…
Add table
Reference in a new issue