Merge pull request #1 from Horizontal42/fix/adversarial-review

Fix defects from adversarial code review

ARCHITECTURE.md

@@ -134,7 +134,7 @@ report/apply/digest --source <url> → source.parse_playlist_url() → (user_id,
   fields flatten out.
 
   `themes` are free-form but pattern-constrained to English snake_case
-  (`^[a-z][a-z0-9_]{2,29}$`) — in the pilot, an unconstrained field drifted between Russian and
+  (`^[a-z0-9][a-z0-9_-]{2,29}$`) — in the pilot, an unconstrained field drifted between Russian and
   English inside a single run, making the values impossible to aggregate; the pattern makes
   Cyrillic mechanically unrepresentable at decode time. They are English for the same reason
   genre slugs and `ROOT_BUCKET` keys are: they are aggregation keys. `mood` is a closed enum that

ARCHITECTURE.ru.md

@@ -134,7 +134,7 @@ report/apply/digest --source <url> → source.parse_playlist_url() → (user_id,
   ограничение вместо этого — таймаут запроса. `temperature: 0.65`, не 0: при 0 нарративные поля
   становятся одноообразными.
 
-  `themes` свободного формата, но ограничены паттерном англ. snake_case (`^[a-z][a-z0-9_]{2,29}$`) —
+  `themes` свободного формата, но ограничены паттерном англ. snake_case (`^[a-z0-9][a-z0-9_-]{2,29}$`) —
   в пилоте неограниченное поле дрейфовало между русским и английским в одном прогоне, что сделало
   значения невозможными для агрегации; паттерн делает кириллицу механически невозможной при разборе.
   Они англ. по той же причине, что и слаги жанров, и `ROOT_BUCKET`: это ключи агрегации. `mood` —

sort_ym/analyze.py

@@ -5,7 +5,7 @@ import json
 import urllib.request
 from collections import Counter
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Callable
 
@@ -41,7 +41,7 @@ STANCES = ["sincere", "ironic", "bitter"]
 # фиксируются схемой: в пилоте без паттерна язык плавал между английским и русским внутри
 # одного прогона. Паттерн на уровне JSON Schema делает кириллицу и составные значения
 # механически невозможными на этапе генерации, а не отлавливаемыми постфактум.
-THEME_PATTERN = "^[a-z][a-z0-9_]{2,29}$"
+THEME_PATTERN = "^[a-z0-9][a-z0-9_-]{2,29}$"
 
 
 def _enum(values: list[str]) -> dict:
@@ -193,9 +193,13 @@ def call_ollama(settings: OllamaSettings, prompt: str) -> dict:
         data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
         headers={"Content-Type": "application/json"},
     )
-    with urllib.request.urlopen(request, timeout=settings.timeout) as response:
-        body = json.loads(response.read().decode("utf-8"))
-    return json.loads(body["message"]["content"])
+    try:
+        with urllib.request.urlopen(request, timeout=settings.timeout) as response:
+            body = json.loads(response.read().decode("utf-8"))
+        return json.loads(body["message"]["content"])
+    except urllib.error.HTTPError as e:
+        error_body = e.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"HTTP Error {e.code}: {e.reason}\n{error_body}") from e
 
 
 def _atomic_write_json(path: Path, data: dict) -> None:
@@ -248,7 +252,7 @@ def analyze_tracks(
             "model": settings.model,
             "prompt_version": settings.prompt_version,
             "lyrics_hash": text_hash,
-            "analyzed_at": datetime.now().isoformat(timespec="seconds"),
+            "analyzed_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
         }
         try:
             data = call(settings, prompt)

sort_ym/apply.py

@@ -27,7 +27,7 @@ def _existing_track_ids(client: Client, playlist: Playlist, max_attempts: int =
     for attempt in range(max_attempts):
         full = with_retries(lambda: client.users_playlists(playlist.kind))
         if full is not None:
-            return {t.track_id for t in full.tracks}
+            return {str(t.track_id).split(":")[0] for t in full.tracks}
         if attempt < max_attempts - 1:
             wait = 2.0 * (2**attempt)
             print(f"  пустой ответ при чтении плейлиста «{playlist.title}», повтор через {wait:.0f}с...")
@@ -91,7 +91,7 @@ def _insert_track_verified(
             fresh = with_retries(lambda: client.users_playlists(playlist.kind))
             if fresh is not None:
                 revision = fresh.revision or revision
-                if track_key in {t.track_id for t in fresh.tracks}:
+                if track_key in {str(t.track_id).split(":")[0] for t in fresh.tracks}:
                     return revision
             if last_attempt:
                 raise
@@ -188,37 +188,50 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None
     playlists = with_retries(lambda: client.users_playlists_list())
 
     total_ranges = 0
-    total_tracks = 0
+    total_found = 0
+    total_removed = 0
     for playlist in playlists:
         full = with_retries(lambda: client.users_playlists(playlist.kind))
         if full is None:
             print(f"  не удалось прочитать «{playlist.title}», пропускаю")
             continue
 
-        ids = [t.track_id for t in full.tracks]
+        ids = [str(t.track_id).split(":")[0] for t in full.tracks]
         ranges = find_duplicate_ranges(ids)
         if not ranges:
             continue
 
         n = sum(to - frm for frm, to in ranges)
         total_ranges += len(ranges)
-        total_tracks += n
+        total_found += n
 
         if dry_run:
             print(f"{playlist.title}: найдено дублей {n} ({len(ranges)} диапазон(ов))")
             continue
 
         revision = full.revision or 1
+        removed = 0
         for frm, to in sorted(ranges, key=lambda r: -r[0]):
-            updated = client.users_playlists_delete_track(playlist.kind, frm, to, revision=revision)
-            revision = updated.revision if updated is not None and updated.revision is not None else revision + 1
+            try:
+                updated = with_retries(lambda: client.users_playlists_delete_track(playlist.kind, frm, to, revision=revision))
+                if updated is not None and updated.revision is not None:
+                    revision = updated.revision
+            except NetworkError as e:
+                print(f"  сетевая ошибка при удалении из «{playlist.title}» ({e}), прерываю")
+                break
+            except BadRequestError as e:
+                print(f"  ошибка при удалении из «{playlist.title}» ({e}), прерываю")
+                break
+            removed += to - frm
             time.sleep(delay)
-        print(f"{playlist.title}: удалено {n} дублей")
+        total_removed += removed
+        suffix = "" if removed == n else f" из {n} найденных"
+        print(f"{playlist.title}: удалено {removed} дублей{suffix}")
 
     if dry_run:
-        if total_tracks:
-            print(f"\nВсего дублей найдено: {total_tracks} в {total_ranges} диапазон(ов). Запустите с --yes для удаления.")
+        if total_found:
+            print(f"\nВсего дублей найдено: {total_found} в {total_ranges} диапазон(ов). Запустите с --yes для удаления.")
         else:
             print("\nДублей не найдено.")
     else:
-        print(f"\nВсего удалено дублей: {total_tracks}")
+        print(f"\nВсего удалено дублей: {total_removed}")

sort_ym/auth.py

@@ -15,7 +15,10 @@ def load_token(token_file: Path) -> str | None:
 
 def save_token(token_file: Path, token: str) -> None:
     token_file.parent.mkdir(parents=True, exist_ok=True)
-    token_file.write_text(token, encoding="utf-8")
+    import os
+    flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
+    with open(os.open(token_file, flags, 0o600), "w", encoding="utf-8") as f:
+        f.write(token)
 
 
 def _on_code(code: DeviceCode) -> None:

sort_ym/cli.py

@@ -203,7 +203,8 @@ def cmd_apply(args: argparse.Namespace) -> None:
 
     catalog = genres.load_or_fetch_catalog(client, cfg.cache_dir)
     artist_genres = fetch.load_artist_genres(cfg.cache_dir)
-    lang_cache = language.load_lang_cache(cfg.cache_dir)
+    ids_needing_lang = [str(t["id"]) for t in tracks_cache.values() if t["lyrics_available"]]
+    lang_cache = language.fetch_api_languages(client, ids_needing_lang, cfg.cache_dir, cfg.lyrics_request_delay)
     rows = report.build_rows(tracks_cache, lang_cache, catalog, artist_genres, cfg.small_group_min)
 
     apply_mod.apply_classification(client, rows, cfg.apply_request_delay, limit=args.limit)

sort_ym/config.py

@@ -4,7 +4,7 @@ import tomllib
 from dataclasses import dataclass
 from pathlib import Path
 
-ROOT = Path(__file__).resolve().parent.parent
+ROOT = Path.cwd()
 
 
 @dataclass(frozen=True)
@@ -29,24 +29,27 @@ class Config:
 
 def load_config(path: Path | None = None) -> Config:
     path = path or ROOT / "config.toml"
-    with path.open("rb") as f:
-        data = tomllib.load(f)
+    try:
+        with path.open("rb") as f:
+            data = tomllib.load(f)
 
-    return Config(
-        fetch_batch_delay=data["throttle"]["fetch_batch_delay"],
-        lyrics_request_delay=data["throttle"]["lyrics_request_delay"],
-        apply_request_delay=data["throttle"]["apply_request_delay"],
-        request_timeout=data["throttle"]["request_timeout"],
-        batch_size=data["fetch"]["batch_size"],
-        small_group_min=data["classify"]["small_group_min"],
-        digest_top_artists=data["digest"]["top_artists"],
-        digest_top_albums=data["digest"]["top_albums"],
-        ollama_host=data["ollama"]["host"],
-        ollama_model=data["ollama"]["model"],
-        ollama_prompt_version=data["ollama"]["prompt_version"],
-        ollama_timeout=data["ollama"]["timeout"],
-        ollama_keep_alive=data["ollama"]["keep_alive"],
-        token_file=ROOT / data["paths"]["token_file"],
-        cache_dir=ROOT / data["paths"]["cache_dir"],
-        out_dir=ROOT / data["paths"]["out_dir"],
-    )
+        return Config(
+            fetch_batch_delay=data["throttle"]["fetch_batch_delay"],
+            lyrics_request_delay=data["throttle"]["lyrics_request_delay"],
+            apply_request_delay=data["throttle"]["apply_request_delay"],
+            request_timeout=data["throttle"]["request_timeout"],
+            batch_size=data["fetch"]["batch_size"],
+            small_group_min=data["classify"]["small_group_min"],
+            digest_top_artists=data["digest"]["top_artists"],
+            digest_top_albums=data["digest"]["top_albums"],
+            ollama_host=data["ollama"]["host"],
+            ollama_model=data["ollama"]["model"],
+            ollama_prompt_version=data["ollama"]["prompt_version"],
+            ollama_timeout=data["ollama"]["timeout"],
+            ollama_keep_alive=data["ollama"]["keep_alive"],
+            token_file=ROOT / data["paths"]["token_file"],
+            cache_dir=ROOT / data["paths"]["cache_dir"],
+            out_dir=ROOT / data["paths"]["out_dir"],
+        )
+    except (KeyError, tomllib.TOMLDecodeError) as e:
+        raise SystemExit(f"Ошибка в конфигурационном файле {path}: {type(e).__name__}: {e}") from e

sort_ym/digest.py

@@ -492,5 +492,7 @@ def render_lyrics_digest(tracks_cache: dict[str, dict], analysis: dict[str, dict
 def write_digest(text: str, out_dir: Path, filename: str = DIGEST_FILE) -> Path:
     out_dir.mkdir(parents=True, exist_ok=True)
     out_file = out_dir / filename
-    out_file.write_text(text, encoding="utf-8")
+    tmp = out_file.with_suffix(".tmp")
+    tmp.write_text(text, encoding="utf-8")
+    tmp.replace(out_file)
     return out_file

sort_ym/fetch.py

@@ -100,8 +100,11 @@ def fetch_liked_tracks(
         # пересчитанному из albums[0] вернувшегося Track - для трека, доступного на нескольких
         # альбомах, "основной" альбом в ответе может отличаться от того, под которым трек лайкнут.
         # Иначе такой трек тут же попадает под stale-очистку ниже как "больше не лайкнутый".
-        for original_id, track in zip(batch, tracks):
-            cache[original_id] = serialize_track(track, added_at=added_at_by_id.get(original_id.split(":")[0]))
+        by_id = {str(t.id): t for t in tracks}
+        for original_id in batch:
+            track = by_id.get(original_id.split(":")[0])
+            if track is not None:
+                cache[original_id] = serialize_track(track, added_at=added_at_by_id.get(original_id.split(":")[0]))
         _atomic_write_json(cache_file, cache)
         print(f"  загружено {len(cache)}/{len(all_ids)}")
         time.sleep(batch_delay)

sort_ym/genres.py

@@ -6,6 +6,7 @@ from pathlib import Path
 
 from yandex_music import Client, Genre
 
+from .fetch import _atomic_write_json
 from .ymclient import with_retries
 
 GENRE_CATALOG_FILE = "genre_catalog.json"
@@ -114,7 +115,7 @@ def load_or_fetch_catalog(client: Client, cache_dir: Path) -> dict[str, dict]:
     roots = with_retries(lambda: client.genres())
     flat = flatten_catalog(roots)
     cache_file = cache_dir / GENRE_CATALOG_FILE
-    cache_file.write_text(json.dumps(flat, ensure_ascii=False, indent=2), encoding="utf-8")
+    _atomic_write_json(cache_file, flat)
     return flat
 
 

sort_ym/language.py

@@ -45,6 +45,8 @@ def alphabet_heuristic(title: str, artists: list[str]) -> str | None:
     text = " ".join([title, *artists])
     cyrillic, latin = _script_counts(text)
     if cyrillic == 0 and latin == 0:
+        if any(ch.isalpha() for ch in text):
+            return "INT"
         return None
     if cyrillic > latin:
         return "RU"
@@ -99,14 +101,14 @@ def load_lang_cache(cache_dir: Path) -> dict[str, str | None]:
     return {}
 
 
-def _fetch_one_language(client: Client, track_id: str) -> str | None:
+def _fetch_supplement(client: Client, track_id: str) -> tuple[str | None, str | None]:
     try:
         supplement = with_retries(lambda: client.track_supplement(track_id))
     except NotFoundError:
-        return None
+        return None, None
     if supplement is None or supplement.lyrics is None:
-        return None
-    return supplement.lyrics.text_language
+        return None, None
+    return supplement.lyrics.text_language, supplement.lyrics.full_lyrics
 
 
 def fetch_api_languages(
@@ -119,17 +121,28 @@ def fetch_api_languages(
     cache_file = cache_dir / LYRICS_LANG_CACHE_FILE
     cache = load_lang_cache(cache_dir)
 
+    text_cache_file = cache_dir / "lyrics_text.json"
+    if text_cache_file.exists():
+        text_cache = json.loads(text_cache_file.read_text(encoding="utf-8"))
+    else:
+        text_cache = {}
+
     missing = [tid for tid in track_ids if tid not in cache]
     if not missing:
         return cache
 
     print(f"Запрос языка текста песни для {len(missing)} треков...")
     for i, tid in enumerate(missing, 1):
-        cache[tid] = _fetch_one_language(client, tid)
+        lang, text = _fetch_supplement(client, tid)
+        cache[tid] = lang
+        if text is not None and normalize_api_language(lang) == "RU":
+            text_cache[tid] = text
         if i % 20 == 0:
             _atomic_write_json(cache_file, cache)
+            _atomic_write_json(text_cache_file, text_cache)
             print(f"  обработано {i}/{len(missing)}")
         time.sleep(delay)
 
     _atomic_write_json(cache_file, cache)
+    _atomic_write_json(text_cache_file, text_cache)
     return cache

sort_ym/report.py

@@ -103,6 +103,8 @@ def build_rows(
     for t, fine, lang in prelim:
         if group_sizes[(fine, lang)] < small_group_min:
             bucket = genres.coarse_of(fine)
+            if bucket == fine:
+                bucket = "other"
         else:
             bucket = fine
         playlist = classify.playlist_name(bucket, lang)

sort_ym/source.py

@@ -18,7 +18,7 @@ from yandex_music.exceptions import NotFoundError, UnauthorizedError
 from . import fetch
 from .ymclient import chunked, with_retries
 
-HOST_RE = re.compile(r"^(?:www\.)?music\.yandex\.[a-z]{2,3}$", re.IGNORECASE)
+HOST_RE = re.compile(r"^(?:www\.|m\.)?music\.yandex\.[a-z\.]+$", re.IGNORECASE)
 # "lk." (личный кабинет) - префикс у ссылок на персональные плейлисты (например, "Мне нравится"),
 # в отличие от чистого uuid у редакционных плейлистов - подтверждено вживую через client.playlist():
 # без префикса API отвечает NotFoundError, префикс - часть самого uuid, а не что-то, что надо отрезать.

sort_ym/ymclient.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 import time
-from typing import Callable, Iterator, List, TypeVar
+from collections.abc import Callable, Iterator, Sequence
+from typing import TypeVar
 
 from yandex_music import Client
 from yandex_music.exceptions import BadRequestError, NetworkError, NotFoundError
@@ -16,7 +17,9 @@ def make_client(token: str, request_timeout: float = 20) -> Client:
     return client
 
 
-def chunked(items: List[T], size: int) -> Iterator[List[T]]:
+def chunked(items: Sequence[T], size: int) -> Iterator[Sequence[T]]:
+    if size <= 0:
+        raise ValueError("size must be > 0")
     for i in range(0, len(items), size):
         yield items[i : i + size]
 
@@ -28,6 +31,8 @@ def with_retries(fn: Callable[[], T], max_attempts: int = 4, base_delay: float =
     (данные точно не найдены / запрос точно невалиден), а не сбои сети, хотя формально
     оба тоже унаследованы от NetworkError. Повтор такого запроса даст тот же результат.
     """
+    if max_attempts <= 0:
+        max_attempts = 1
     last_exc: NetworkError | None = None
     for attempt in range(max_attempts):
         try:
@@ -38,7 +43,7 @@ def with_retries(fn: Callable[[], T], max_attempts: int = 4, base_delay: float =
             last_exc = e
             if attempt < max_attempts - 1:
                 wait = base_delay * (2**attempt)
-                print(f"  сетевая ошибка ({e}), повтор через {wait:.0f}с...")
+                print(f"  сетевая ошибка ({type(e).__name__}: {e}), повтор через {wait:.0f}с...")
                 time.sleep(wait)
 
     assert last_exc is not None