From 970fd551e51b0fd2345a24e3ac21e66c02024955 Mon Sep 17 00:00:00 2001 From: Andrew Pimenov Date: Wed, 19 Aug 2026 04:14:29 +0300 Subject: [PATCH] tighten retry scope, RU-only lyrics text cache, fix theme pattern and dedupe log with_retries caught bare Exception with a type whitelist, masking real bugs as network errors under retry/backoff - narrowed back to NetworkError. fetch_api_languages was writing full lyrics text for every language into lyrics_text.json via its track_supplement side channel, breaking the documented RU-only invariant and letting analyze feed foreign-language lyrics through the Russian-only prompt - now gated on normalize_api_language. dedupe_playlists logged the found duplicate count as "removed" even when a mid-loop network/API error cut deletion short. THEME_PATTERN's relaxed regex allowed 2-char themes, contradicting its own test; current_themes was hoisted out of the analyze loop so later prompts never saw themes found earlier in the same run. Both reverted to intended behavior, docs updated to match. --- ARCHITECTURE.md | 2 +- ARCHITECTURE.ru.md | 2 +- sort_ym/analyze.py | 5 ++--- sort_ym/apply.py | 17 +++++++++++------ sort_ym/language.py | 2 +- sort_ym/ymclient.py | 6 ++---- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index cfb6ff7..64f44c6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -134,7 +134,7 @@ report/apply/digest --source → source.parse_playlist_url() → (user_id, fields flatten out. `themes` are free-form but pattern-constrained to English snake_case - (`^[a-z][a-z0-9_]{2,29}$`) — in the pilot, an unconstrained field drifted between Russian and + (`^[a-z0-9][a-z0-9_-]{2,29}$`) — in the pilot, an unconstrained field drifted between Russian and English inside a single run, making the values impossible to aggregate; the pattern makes Cyrillic mechanically unrepresentable at decode time. They are English for the same reason genre slugs and `ROOT_BUCKET` keys are: they are aggregation keys. `mood` is a closed enum that diff --git a/ARCHITECTURE.ru.md b/ARCHITECTURE.ru.md index 788f647..33bcfe2 100644 --- a/ARCHITECTURE.ru.md +++ b/ARCHITECTURE.ru.md @@ -134,7 +134,7 @@ report/apply/digest --source → source.parse_playlist_url() → (user_id, ограничение вместо этого — таймаут запроса. `temperature: 0.65`, не 0: при 0 нарративные поля становятся одноообразными. - `themes` свободного формата, но ограничены паттерном англ. snake_case (`^[a-z][a-z0-9_]{2,29}$`) — + `themes` свободного формата, но ограничены паттерном англ. snake_case (`^[a-z0-9][a-z0-9_-]{2,29}$`) — в пилоте неограниченное поле дрейфовало между русским и английским в одном прогоне, что сделало значения невозможными для агрегации; паттерн делает кириллицу механически невозможной при разборе. Они англ. по той же причине, что и слаги жанров, и `ROOT_BUCKET`: это ключи агрегации. `mood` — diff --git a/sort_ym/analyze.py b/sort_ym/analyze.py index 3b8b432..509e45c 100644 --- a/sort_ym/analyze.py +++ b/sort_ym/analyze.py @@ -41,7 +41,7 @@ STANCES = ["sincere", "ironic", "bitter"] # фиксируются схемой: в пилоте без паттерна язык плавал между английским и русским внутри # одного прогона. Паттерн на уровне JSON Schema делает кириллицу и составные значения # механически невозможными на этапе генерации, а не отлавливаемыми постфактум. -THEME_PATTERN = "^[a-z0-9_-]{2,30}$" +THEME_PATTERN = "^[a-z0-9][a-z0-9_-]{2,29}$" def _enum(values: list[str]) -> dict: @@ -245,10 +245,9 @@ def analyze_tracks( print(f"Разбор текстов: {len(pending)} треков, модель {settings.model}") failed = 0 - current_themes = top_themes(cache) for i, (tid, text, text_hash) in enumerate(pending, 1): info = meta.get(tid, {"title": tid, "artists": []}) - prompt = build_prompt(info["title"], info["artists"], text, current_themes) + prompt = build_prompt(info["title"], info["artists"], text, top_themes(cache)) stamp = { "model": settings.model, "prompt_version": settings.prompt_version, diff --git a/sort_ym/apply.py b/sort_ym/apply.py index a21556e..4397289 100644 --- a/sort_ym/apply.py +++ b/sort_ym/apply.py @@ -188,7 +188,8 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None playlists = with_retries(lambda: client.users_playlists_list()) total_ranges = 0 - total_tracks = 0 + total_found = 0 + total_removed = 0 for playlist in playlists: full = with_retries(lambda: client.users_playlists(playlist.kind)) if full is None: @@ -202,13 +203,14 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None n = sum(to - frm for frm, to in ranges) total_ranges += len(ranges) - total_tracks += n + total_found += n if dry_run: print(f"{playlist.title}: найдено дублей {n} ({len(ranges)} диапазон(ов))") continue revision = full.revision or 1 + removed = 0 for frm, to in sorted(ranges, key=lambda r: -r[0]): try: updated = with_retries(lambda: client.users_playlists_delete_track(playlist.kind, frm, to, revision=revision)) @@ -220,13 +222,16 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None except BadRequestError as e: print(f" ошибка при удалении из «{playlist.title}» ({e}), прерываю") break + removed += to - frm time.sleep(delay) - print(f"{playlist.title}: удалено {n} дублей") + total_removed += removed + suffix = "" if removed == n else f" из {n} найденных" + print(f"{playlist.title}: удалено {removed} дублей{suffix}") if dry_run: - if total_tracks: - print(f"\nВсего дублей найдено: {total_tracks} в {total_ranges} диапазон(ов). Запустите с --yes для удаления.") + if total_found: + print(f"\nВсего дублей найдено: {total_found} в {total_ranges} диапазон(ов). Запустите с --yes для удаления.") else: print("\nДублей не найдено.") else: - print(f"\nВсего удалено дублей: {total_tracks}") + print(f"\nВсего удалено дублей: {total_removed}") diff --git a/sort_ym/language.py b/sort_ym/language.py index 012d57a..601b424 100644 --- a/sort_ym/language.py +++ b/sort_ym/language.py @@ -135,7 +135,7 @@ def fetch_api_languages( for i, tid in enumerate(missing, 1): lang, text = _fetch_supplement(client, tid) cache[tid] = lang - if text is not None: + if text is not None and normalize_api_language(lang) == "RU": text_cache[tid] = text if i % 20 == 0: _atomic_write_json(cache_file, cache) diff --git a/sort_ym/ymclient.py b/sort_ym/ymclient.py index 0c30fa5..18bafc4 100644 --- a/sort_ym/ymclient.py +++ b/sort_ym/ymclient.py @@ -33,15 +33,13 @@ def with_retries(fn: Callable[[], T], max_attempts: int = 4, base_delay: float = """ if max_attempts <= 0: max_attempts = 1 - last_exc: Exception | None = None + last_exc: NetworkError | None = None for attempt in range(max_attempts): try: return fn() except (NotFoundError, BadRequestError): raise - except Exception as e: - if isinstance(e, (TypeError, ValueError, AttributeError, NameError)): - raise + except NetworkError as e: last_exc = e if attempt < max_attempts - 1: wait = base_delay * (2**attempt)