tighten retry scope, RU-only lyrics text cache, fix theme pattern and dedupe log

with_retries caught bare Exception with a type whitelist, masking real bugs
as network errors under retry/backoff - narrowed back to NetworkError.

fetch_api_languages was writing full lyrics text for every language into
lyrics_text.json via its track_supplement side channel, breaking the
documented RU-only invariant and letting analyze feed foreign-language
lyrics through the Russian-only prompt - now gated on normalize_api_language.

dedupe_playlists logged the found duplicate count as "removed" even when
a mid-loop network/API error cut deletion short.

THEME_PATTERN's relaxed regex allowed 2-char themes, contradicting its own
test; current_themes was hoisted out of the analyze loop so later prompts
never saw themes found earlier in the same run. Both reverted to intended
behavior, docs updated to match.
This commit is contained in:
Andrew Pimenov 2026-08-19 04:14:29 +03:00
parent f77fafd240
commit 970fd551e5
6 changed files with 18 additions and 16 deletions

View file

@ -134,7 +134,7 @@ report/apply/digest --source <url> → source.parse_playlist_url() → (user_id,
fields flatten out. fields flatten out.
`themes` are free-form but pattern-constrained to English snake_case `themes` are free-form but pattern-constrained to English snake_case
(`^[a-z][a-z0-9_]{2,29}$`) — in the pilot, an unconstrained field drifted between Russian and (`^[a-z0-9][a-z0-9_-]{2,29}$`) — in the pilot, an unconstrained field drifted between Russian and
English inside a single run, making the values impossible to aggregate; the pattern makes English inside a single run, making the values impossible to aggregate; the pattern makes
Cyrillic mechanically unrepresentable at decode time. They are English for the same reason Cyrillic mechanically unrepresentable at decode time. They are English for the same reason
genre slugs and `ROOT_BUCKET` keys are: they are aggregation keys. `mood` is a closed enum that genre slugs and `ROOT_BUCKET` keys are: they are aggregation keys. `mood` is a closed enum that

View file

@ -134,7 +134,7 @@ report/apply/digest --source <url> → source.parse_playlist_url() → (user_id,
ограничение вместо этого — таймаут запроса. `temperature: 0.65`, не 0: при 0 нарративные поля ограничение вместо этого — таймаут запроса. `temperature: 0.65`, не 0: при 0 нарративные поля
становятся одноообразными. становятся одноообразными.
`themes` свободного формата, но ограничены паттерном англ. snake_case (`^[a-z][a-z0-9_]{2,29}$`) — `themes` свободного формата, но ограничены паттерном англ. snake_case (`^[a-z0-9][a-z0-9_-]{2,29}$`) —
в пилоте неограниченное поле дрейфовало между русским и английским в одном прогоне, что сделало в пилоте неограниченное поле дрейфовало между русским и английским в одном прогоне, что сделало
значения невозможными для агрегации; паттерн делает кириллицу механически невозможной при разборе. значения невозможными для агрегации; паттерн делает кириллицу механически невозможной при разборе.
Они англ. по той же причине, что и слаги жанров, и `ROOT_BUCKET`: это ключи агрегации. `mood` — Они англ. по той же причине, что и слаги жанров, и `ROOT_BUCKET`: это ключи агрегации. `mood` —

View file

@ -41,7 +41,7 @@ STANCES = ["sincere", "ironic", "bitter"]
# фиксируются схемой: в пилоте без паттерна язык плавал между английским и русским внутри # фиксируются схемой: в пилоте без паттерна язык плавал между английским и русским внутри
# одного прогона. Паттерн на уровне JSON Schema делает кириллицу и составные значения # одного прогона. Паттерн на уровне JSON Schema делает кириллицу и составные значения
# механически невозможными на этапе генерации, а не отлавливаемыми постфактум. # механически невозможными на этапе генерации, а не отлавливаемыми постфактум.
THEME_PATTERN = "^[a-z0-9_-]{2,30}$" THEME_PATTERN = "^[a-z0-9][a-z0-9_-]{2,29}$"
def _enum(values: list[str]) -> dict: def _enum(values: list[str]) -> dict:
@ -245,10 +245,9 @@ def analyze_tracks(
print(f"Разбор текстов: {len(pending)} треков, модель {settings.model}") print(f"Разбор текстов: {len(pending)} треков, модель {settings.model}")
failed = 0 failed = 0
current_themes = top_themes(cache)
for i, (tid, text, text_hash) in enumerate(pending, 1): for i, (tid, text, text_hash) in enumerate(pending, 1):
info = meta.get(tid, {"title": tid, "artists": []}) info = meta.get(tid, {"title": tid, "artists": []})
prompt = build_prompt(info["title"], info["artists"], text, current_themes) prompt = build_prompt(info["title"], info["artists"], text, top_themes(cache))
stamp = { stamp = {
"model": settings.model, "model": settings.model,
"prompt_version": settings.prompt_version, "prompt_version": settings.prompt_version,

View file

@ -188,7 +188,8 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None
playlists = with_retries(lambda: client.users_playlists_list()) playlists = with_retries(lambda: client.users_playlists_list())
total_ranges = 0 total_ranges = 0
total_tracks = 0 total_found = 0
total_removed = 0
for playlist in playlists: for playlist in playlists:
full = with_retries(lambda: client.users_playlists(playlist.kind)) full = with_retries(lambda: client.users_playlists(playlist.kind))
if full is None: if full is None:
@ -202,13 +203,14 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None
n = sum(to - frm for frm, to in ranges) n = sum(to - frm for frm, to in ranges)
total_ranges += len(ranges) total_ranges += len(ranges)
total_tracks += n total_found += n
if dry_run: if dry_run:
print(f"{playlist.title}: найдено дублей {n} ({len(ranges)} диапазон(ов))") print(f"{playlist.title}: найдено дублей {n} ({len(ranges)} диапазон(ов))")
continue continue
revision = full.revision or 1 revision = full.revision or 1
removed = 0
for frm, to in sorted(ranges, key=lambda r: -r[0]): for frm, to in sorted(ranges, key=lambda r: -r[0]):
try: try:
updated = with_retries(lambda: client.users_playlists_delete_track(playlist.kind, frm, to, revision=revision)) updated = with_retries(lambda: client.users_playlists_delete_track(playlist.kind, frm, to, revision=revision))
@ -220,13 +222,16 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None
except BadRequestError as e: except BadRequestError as e:
print(f" ошибка при удалении из «{playlist.title}» ({e}), прерываю") print(f" ошибка при удалении из «{playlist.title}» ({e}), прерываю")
break break
removed += to - frm
time.sleep(delay) time.sleep(delay)
print(f"{playlist.title}: удалено {n} дублей") total_removed += removed
suffix = "" if removed == n else f" из {n} найденных"
print(f"{playlist.title}: удалено {removed} дублей{suffix}")
if dry_run: if dry_run:
if total_tracks: if total_found:
print(f"\nВсего дублей найдено: {total_tracks} в {total_ranges} диапазон(ов). Запустите с --yes для удаления.") print(f"\nВсего дублей найдено: {total_found} в {total_ranges} диапазон(ов). Запустите с --yes для удаления.")
else: else:
print("\nДублей не найдено.") print("\nДублей не найдено.")
else: else:
print(f"\nВсего удалено дублей: {total_tracks}") print(f"\nВсего удалено дублей: {total_removed}")

View file

@ -135,7 +135,7 @@ def fetch_api_languages(
for i, tid in enumerate(missing, 1): for i, tid in enumerate(missing, 1):
lang, text = _fetch_supplement(client, tid) lang, text = _fetch_supplement(client, tid)
cache[tid] = lang cache[tid] = lang
if text is not None: if text is not None and normalize_api_language(lang) == "RU":
text_cache[tid] = text text_cache[tid] = text
if i % 20 == 0: if i % 20 == 0:
_atomic_write_json(cache_file, cache) _atomic_write_json(cache_file, cache)

View file

@ -33,15 +33,13 @@ def with_retries(fn: Callable[[], T], max_attempts: int = 4, base_delay: float =
""" """
if max_attempts <= 0: if max_attempts <= 0:
max_attempts = 1 max_attempts = 1
last_exc: Exception | None = None last_exc: NetworkError | None = None
for attempt in range(max_attempts): for attempt in range(max_attempts):
try: try:
return fn() return fn()
except (NotFoundError, BadRequestError): except (NotFoundError, BadRequestError):
raise raise
except Exception as e: except NetworkError as e:
if isinstance(e, (TypeError, ValueError, AttributeError, NameError)):
raise
last_exc = e last_exc = e
if attempt < max_attempts - 1: if attempt < max_attempts - 1:
wait = base_delay * (2**attempt) wait = base_delay * (2**attempt)