From f77fafd24046365c5202d6933a9812890f3c3dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=9F=D0=B8=D0=BC?= =?UTF-8?q?=D0=B5=D0=BD=D0=BE=D0=B2?= Date: Wed, 19 Aug 2026 03:53:25 +0300 Subject: [PATCH] Fix adversarial code review defects --- sort_ym/analyze.py | 19 ++++++++++++------- sort_ym/apply.py | 18 +++++++++++++----- sort_ym/auth.py | 5 ++++- sort_ym/cli.py | 3 ++- sort_ym/config.py | 45 ++++++++++++++++++++++++--------------------- sort_ym/digest.py | 4 +++- sort_ym/fetch.py | 7 +++++-- sort_ym/genres.py | 3 ++- sort_ym/language.py | 23 ++++++++++++++++++----- sort_ym/report.py | 2 ++ sort_ym/source.py | 2 +- sort_ym/ymclient.py | 17 ++++++++++++----- 12 files changed, 98 insertions(+), 50 deletions(-) diff --git a/sort_ym/analyze.py b/sort_ym/analyze.py index 0f83dad..3b8b432 100644 --- a/sort_ym/analyze.py +++ b/sort_ym/analyze.py @@ -5,7 +5,7 @@ import json import urllib.request from collections import Counter from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Callable @@ -41,7 +41,7 @@ STANCES = ["sincere", "ironic", "bitter"] # фиксируются схемой: в пилоте без паттерна язык плавал между английским и русским внутри # одного прогона. Паттерн на уровне JSON Schema делает кириллицу и составные значения # механически невозможными на этапе генерации, а не отлавливаемыми постфактум. -THEME_PATTERN = "^[a-z][a-z0-9_]{2,29}$" +THEME_PATTERN = "^[a-z0-9_-]{2,30}$" def _enum(values: list[str]) -> dict: @@ -193,9 +193,13 @@ def call_ollama(settings: OllamaSettings, prompt: str) -> dict: data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), headers={"Content-Type": "application/json"}, ) - with urllib.request.urlopen(request, timeout=settings.timeout) as response: - body = json.loads(response.read().decode("utf-8")) - return json.loads(body["message"]["content"]) + try: + with urllib.request.urlopen(request, timeout=settings.timeout) as response: + body = json.loads(response.read().decode("utf-8")) + return json.loads(body["message"]["content"]) + except urllib.error.HTTPError as e: + error_body = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"HTTP Error {e.code}: {e.reason}\n{error_body}") from e def _atomic_write_json(path: Path, data: dict) -> None: @@ -241,14 +245,15 @@ def analyze_tracks( print(f"Разбор текстов: {len(pending)} треков, модель {settings.model}") failed = 0 + current_themes = top_themes(cache) for i, (tid, text, text_hash) in enumerate(pending, 1): info = meta.get(tid, {"title": tid, "artists": []}) - prompt = build_prompt(info["title"], info["artists"], text, top_themes(cache)) + prompt = build_prompt(info["title"], info["artists"], text, current_themes) stamp = { "model": settings.model, "prompt_version": settings.prompt_version, "lyrics_hash": text_hash, - "analyzed_at": datetime.now().isoformat(timespec="seconds"), + "analyzed_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), } try: data = call(settings, prompt) diff --git a/sort_ym/apply.py b/sort_ym/apply.py index eb28c03..a21556e 100644 --- a/sort_ym/apply.py +++ b/sort_ym/apply.py @@ -27,7 +27,7 @@ def _existing_track_ids(client: Client, playlist: Playlist, max_attempts: int = for attempt in range(max_attempts): full = with_retries(lambda: client.users_playlists(playlist.kind)) if full is not None: - return {t.track_id for t in full.tracks} + return {str(t.track_id).split(":")[0] for t in full.tracks} if attempt < max_attempts - 1: wait = 2.0 * (2**attempt) print(f" пустой ответ при чтении плейлиста «{playlist.title}», повтор через {wait:.0f}с...") @@ -91,7 +91,7 @@ def _insert_track_verified( fresh = with_retries(lambda: client.users_playlists(playlist.kind)) if fresh is not None: revision = fresh.revision or revision - if track_key in {t.track_id for t in fresh.tracks}: + if track_key in {str(t.track_id).split(":")[0] for t in fresh.tracks}: return revision if last_attempt: raise @@ -195,7 +195,7 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None print(f" не удалось прочитать «{playlist.title}», пропускаю") continue - ids = [t.track_id for t in full.tracks] + ids = [str(t.track_id).split(":")[0] for t in full.tracks] ranges = find_duplicate_ranges(ids) if not ranges: continue @@ -210,8 +210,16 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None revision = full.revision or 1 for frm, to in sorted(ranges, key=lambda r: -r[0]): - updated = client.users_playlists_delete_track(playlist.kind, frm, to, revision=revision) - revision = updated.revision if updated is not None and updated.revision is not None else revision + 1 + try: + updated = with_retries(lambda: client.users_playlists_delete_track(playlist.kind, frm, to, revision=revision)) + if updated is not None and updated.revision is not None: + revision = updated.revision + except NetworkError as e: + print(f" сетевая ошибка при удалении из «{playlist.title}» ({e}), прерываю") + break + except BadRequestError as e: + print(f" ошибка при удалении из «{playlist.title}» ({e}), прерываю") + break time.sleep(delay) print(f"{playlist.title}: удалено {n} дублей") diff --git a/sort_ym/auth.py b/sort_ym/auth.py index 0bb314f..556e9b5 100644 --- a/sort_ym/auth.py +++ b/sort_ym/auth.py @@ -15,7 +15,10 @@ def load_token(token_file: Path) -> str | None: def save_token(token_file: Path, token: str) -> None: token_file.parent.mkdir(parents=True, exist_ok=True) - token_file.write_text(token, encoding="utf-8") + import os + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + with open(os.open(token_file, flags, 0o600), "w", encoding="utf-8") as f: + f.write(token) def _on_code(code: DeviceCode) -> None: diff --git a/sort_ym/cli.py b/sort_ym/cli.py index fcfcbd1..31d3177 100644 --- a/sort_ym/cli.py +++ b/sort_ym/cli.py @@ -203,7 +203,8 @@ def cmd_apply(args: argparse.Namespace) -> None: catalog = genres.load_or_fetch_catalog(client, cfg.cache_dir) artist_genres = fetch.load_artist_genres(cfg.cache_dir) - lang_cache = language.load_lang_cache(cfg.cache_dir) + ids_needing_lang = [str(t["id"]) for t in tracks_cache.values() if t["lyrics_available"]] + lang_cache = language.fetch_api_languages(client, ids_needing_lang, cfg.cache_dir, cfg.lyrics_request_delay) rows = report.build_rows(tracks_cache, lang_cache, catalog, artist_genres, cfg.small_group_min) apply_mod.apply_classification(client, rows, cfg.apply_request_delay, limit=args.limit) diff --git a/sort_ym/config.py b/sort_ym/config.py index fbda3c5..e94d95f 100644 --- a/sort_ym/config.py +++ b/sort_ym/config.py @@ -4,7 +4,7 @@ import tomllib from dataclasses import dataclass from pathlib import Path -ROOT = Path(__file__).resolve().parent.parent +ROOT = Path.cwd() @dataclass(frozen=True) @@ -29,24 +29,27 @@ class Config: def load_config(path: Path | None = None) -> Config: path = path or ROOT / "config.toml" - with path.open("rb") as f: - data = tomllib.load(f) + try: + with path.open("rb") as f: + data = tomllib.load(f) - return Config( - fetch_batch_delay=data["throttle"]["fetch_batch_delay"], - lyrics_request_delay=data["throttle"]["lyrics_request_delay"], - apply_request_delay=data["throttle"]["apply_request_delay"], - request_timeout=data["throttle"]["request_timeout"], - batch_size=data["fetch"]["batch_size"], - small_group_min=data["classify"]["small_group_min"], - digest_top_artists=data["digest"]["top_artists"], - digest_top_albums=data["digest"]["top_albums"], - ollama_host=data["ollama"]["host"], - ollama_model=data["ollama"]["model"], - ollama_prompt_version=data["ollama"]["prompt_version"], - ollama_timeout=data["ollama"]["timeout"], - ollama_keep_alive=data["ollama"]["keep_alive"], - token_file=ROOT / data["paths"]["token_file"], - cache_dir=ROOT / data["paths"]["cache_dir"], - out_dir=ROOT / data["paths"]["out_dir"], - ) + return Config( + fetch_batch_delay=data["throttle"]["fetch_batch_delay"], + lyrics_request_delay=data["throttle"]["lyrics_request_delay"], + apply_request_delay=data["throttle"]["apply_request_delay"], + request_timeout=data["throttle"]["request_timeout"], + batch_size=data["fetch"]["batch_size"], + small_group_min=data["classify"]["small_group_min"], + digest_top_artists=data["digest"]["top_artists"], + digest_top_albums=data["digest"]["top_albums"], + ollama_host=data["ollama"]["host"], + ollama_model=data["ollama"]["model"], + ollama_prompt_version=data["ollama"]["prompt_version"], + ollama_timeout=data["ollama"]["timeout"], + ollama_keep_alive=data["ollama"]["keep_alive"], + token_file=ROOT / data["paths"]["token_file"], + cache_dir=ROOT / data["paths"]["cache_dir"], + out_dir=ROOT / data["paths"]["out_dir"], + ) + except (KeyError, tomllib.TOMLDecodeError) as e: + raise SystemExit(f"Ошибка в конфигурационном файле {path}: {type(e).__name__}: {e}") from e diff --git a/sort_ym/digest.py b/sort_ym/digest.py index 4fe9607..bcaa017 100644 --- a/sort_ym/digest.py +++ b/sort_ym/digest.py @@ -492,5 +492,7 @@ def render_lyrics_digest(tracks_cache: dict[str, dict], analysis: dict[str, dict def write_digest(text: str, out_dir: Path, filename: str = DIGEST_FILE) -> Path: out_dir.mkdir(parents=True, exist_ok=True) out_file = out_dir / filename - out_file.write_text(text, encoding="utf-8") + tmp = out_file.with_suffix(".tmp") + tmp.write_text(text, encoding="utf-8") + tmp.replace(out_file) return out_file diff --git a/sort_ym/fetch.py b/sort_ym/fetch.py index 27186ed..1343381 100644 --- a/sort_ym/fetch.py +++ b/sort_ym/fetch.py @@ -100,8 +100,11 @@ def fetch_liked_tracks( # пересчитанному из albums[0] вернувшегося Track - для трека, доступного на нескольких # альбомах, "основной" альбом в ответе может отличаться от того, под которым трек лайкнут. # Иначе такой трек тут же попадает под stale-очистку ниже как "больше не лайкнутый". - for original_id, track in zip(batch, tracks): - cache[original_id] = serialize_track(track, added_at=added_at_by_id.get(original_id.split(":")[0])) + by_id = {str(t.id): t for t in tracks} + for original_id in batch: + track = by_id.get(original_id.split(":")[0]) + if track is not None: + cache[original_id] = serialize_track(track, added_at=added_at_by_id.get(original_id.split(":")[0])) _atomic_write_json(cache_file, cache) print(f" загружено {len(cache)}/{len(all_ids)}") time.sleep(batch_delay) diff --git a/sort_ym/genres.py b/sort_ym/genres.py index c2eae38..a032e8e 100644 --- a/sort_ym/genres.py +++ b/sort_ym/genres.py @@ -6,6 +6,7 @@ from pathlib import Path from yandex_music import Client, Genre +from .fetch import _atomic_write_json from .ymclient import with_retries GENRE_CATALOG_FILE = "genre_catalog.json" @@ -114,7 +115,7 @@ def load_or_fetch_catalog(client: Client, cache_dir: Path) -> dict[str, dict]: roots = with_retries(lambda: client.genres()) flat = flatten_catalog(roots) cache_file = cache_dir / GENRE_CATALOG_FILE - cache_file.write_text(json.dumps(flat, ensure_ascii=False, indent=2), encoding="utf-8") + _atomic_write_json(cache_file, flat) return flat diff --git a/sort_ym/language.py b/sort_ym/language.py index bd82659..012d57a 100644 --- a/sort_ym/language.py +++ b/sort_ym/language.py @@ -45,6 +45,8 @@ def alphabet_heuristic(title: str, artists: list[str]) -> str | None: text = " ".join([title, *artists]) cyrillic, latin = _script_counts(text) if cyrillic == 0 and latin == 0: + if any(ch.isalpha() for ch in text): + return "INT" return None if cyrillic > latin: return "RU" @@ -99,14 +101,14 @@ def load_lang_cache(cache_dir: Path) -> dict[str, str | None]: return {} -def _fetch_one_language(client: Client, track_id: str) -> str | None: +def _fetch_supplement(client: Client, track_id: str) -> tuple[str | None, str | None]: try: supplement = with_retries(lambda: client.track_supplement(track_id)) except NotFoundError: - return None + return None, None if supplement is None or supplement.lyrics is None: - return None - return supplement.lyrics.text_language + return None, None + return supplement.lyrics.text_language, supplement.lyrics.full_lyrics def fetch_api_languages( @@ -119,17 +121,28 @@ def fetch_api_languages( cache_file = cache_dir / LYRICS_LANG_CACHE_FILE cache = load_lang_cache(cache_dir) + text_cache_file = cache_dir / "lyrics_text.json" + if text_cache_file.exists(): + text_cache = json.loads(text_cache_file.read_text(encoding="utf-8")) + else: + text_cache = {} + missing = [tid for tid in track_ids if tid not in cache] if not missing: return cache print(f"Запрос языка текста песни для {len(missing)} треков...") for i, tid in enumerate(missing, 1): - cache[tid] = _fetch_one_language(client, tid) + lang, text = _fetch_supplement(client, tid) + cache[tid] = lang + if text is not None: + text_cache[tid] = text if i % 20 == 0: _atomic_write_json(cache_file, cache) + _atomic_write_json(text_cache_file, text_cache) print(f" обработано {i}/{len(missing)}") time.sleep(delay) _atomic_write_json(cache_file, cache) + _atomic_write_json(text_cache_file, text_cache) return cache diff --git a/sort_ym/report.py b/sort_ym/report.py index 704b76c..d3c56e7 100644 --- a/sort_ym/report.py +++ b/sort_ym/report.py @@ -103,6 +103,8 @@ def build_rows( for t, fine, lang in prelim: if group_sizes[(fine, lang)] < small_group_min: bucket = genres.coarse_of(fine) + if bucket == fine: + bucket = "other" else: bucket = fine playlist = classify.playlist_name(bucket, lang) diff --git a/sort_ym/source.py b/sort_ym/source.py index d3e5140..e934824 100644 --- a/sort_ym/source.py +++ b/sort_ym/source.py @@ -18,7 +18,7 @@ from yandex_music.exceptions import NotFoundError, UnauthorizedError from . import fetch from .ymclient import chunked, with_retries -HOST_RE = re.compile(r"^(?:www\.)?music\.yandex\.[a-z]{2,3}$", re.IGNORECASE) +HOST_RE = re.compile(r"^(?:www\.|m\.)?music\.yandex\.[a-z\.]+$", re.IGNORECASE) # "lk." (личный кабинет) - префикс у ссылок на персональные плейлисты (например, "Мне нравится"), # в отличие от чистого uuid у редакционных плейлистов - подтверждено вживую через client.playlist(): # без префикса API отвечает NotFoundError, префикс - часть самого uuid, а не что-то, что надо отрезать. diff --git a/sort_ym/ymclient.py b/sort_ym/ymclient.py index 2dbfc07..0c30fa5 100644 --- a/sort_ym/ymclient.py +++ b/sort_ym/ymclient.py @@ -1,7 +1,8 @@ from __future__ import annotations import time -from typing import Callable, Iterator, List, TypeVar +from collections.abc import Callable, Iterator, Sequence +from typing import TypeVar from yandex_music import Client from yandex_music.exceptions import BadRequestError, NetworkError, NotFoundError @@ -16,7 +17,9 @@ def make_client(token: str, request_timeout: float = 20) -> Client: return client -def chunked(items: List[T], size: int) -> Iterator[List[T]]: +def chunked(items: Sequence[T], size: int) -> Iterator[Sequence[T]]: + if size <= 0: + raise ValueError("size must be > 0") for i in range(0, len(items), size): yield items[i : i + size] @@ -28,17 +31,21 @@ def with_retries(fn: Callable[[], T], max_attempts: int = 4, base_delay: float = (данные точно не найдены / запрос точно невалиден), а не сбои сети, хотя формально оба тоже унаследованы от NetworkError. Повтор такого запроса даст тот же результат. """ - last_exc: NetworkError | None = None + if max_attempts <= 0: + max_attempts = 1 + last_exc: Exception | None = None for attempt in range(max_attempts): try: return fn() except (NotFoundError, BadRequestError): raise - except NetworkError as e: + except Exception as e: + if isinstance(e, (TypeError, ValueError, AttributeError, NameError)): + raise last_exc = e if attempt < max_attempts - 1: wait = base_delay * (2**attempt) - print(f" сетевая ошибка ({e}), повтор через {wait:.0f}с...") + print(f" сетевая ошибка ({type(e).__name__}: {e}), повтор через {wait:.0f}с...") time.sleep(wait) assert last_exc is not None