tighten retry scope, RU-only lyrics text cache, fix theme pattern and dedupe log

with_retries caught bare Exception with a type whitelist, masking real bugs
as network errors under retry/backoff - narrowed back to NetworkError.

fetch_api_languages was writing full lyrics text for every language into
lyrics_text.json via its track_supplement side channel, breaking the
documented RU-only invariant and letting analyze feed foreign-language
lyrics through the Russian-only prompt - now gated on normalize_api_language.

dedupe_playlists logged the found duplicate count as "removed" even when
a mid-loop network/API error cut deletion short.

THEME_PATTERN's relaxed regex allowed 2-char themes, contradicting its own
test; current_themes was hoisted out of the analyze loop so later prompts
never saw themes found earlier in the same run. Both reverted to intended
behavior, docs updated to match.
This commit is contained in:
Andrew Pimenov 2026-08-19 04:14:29 +03:00
parent f77fafd240
commit 970fd551e5
6 changed files with 18 additions and 16 deletions

View file

@ -134,7 +134,7 @@ report/apply/digest --source <url> → source.parse_playlist_url() → (user_id,
fields flatten out.
`themes` are free-form but pattern-constrained to English snake_case
(`^[a-z][a-z0-9_]{2,29}$`) — in the pilot, an unconstrained field drifted between Russian and
(`^[a-z0-9][a-z0-9_-]{2,29}$`) — in the pilot, an unconstrained field drifted between Russian and
English inside a single run, making the values impossible to aggregate; the pattern makes
Cyrillic mechanically unrepresentable at decode time. They are English for the same reason
genre slugs and `ROOT_BUCKET` keys are: they are aggregation keys. `mood` is a closed enum that

View file

@ -134,7 +134,7 @@ report/apply/digest --source <url> → source.parse_playlist_url() → (user_id,
ограничение вместо этого — таймаут запроса. `temperature: 0.65`, не 0: при 0 нарративные поля
становятся одноообразными.
`themes` свободного формата, но ограничены паттерном англ. snake_case (`^[a-z][a-z0-9_]{2,29}$`) —
`themes` свободного формата, но ограничены паттерном англ. snake_case (`^[a-z0-9][a-z0-9_-]{2,29}$`) —
в пилоте неограниченное поле дрейфовало между русским и английским в одном прогоне, что сделало
значения невозможными для агрегации; паттерн делает кириллицу механически невозможной при разборе.
Они англ. по той же причине, что и слаги жанров, и `ROOT_BUCKET`: это ключи агрегации. `mood` —

View file

@ -41,7 +41,7 @@ STANCES = ["sincere", "ironic", "bitter"]
# фиксируются схемой: в пилоте без паттерна язык плавал между английским и русским внутри
# одного прогона. Паттерн на уровне JSON Schema делает кириллицу и составные значения
# механически невозможными на этапе генерации, а не отлавливаемыми постфактум.
THEME_PATTERN = "^[a-z0-9_-]{2,30}$"
THEME_PATTERN = "^[a-z0-9][a-z0-9_-]{2,29}$"
def _enum(values: list[str]) -> dict:
@ -245,10 +245,9 @@ def analyze_tracks(
print(f"Разбор текстов: {len(pending)} треков, модель {settings.model}")
failed = 0
current_themes = top_themes(cache)
for i, (tid, text, text_hash) in enumerate(pending, 1):
info = meta.get(tid, {"title": tid, "artists": []})
prompt = build_prompt(info["title"], info["artists"], text, current_themes)
prompt = build_prompt(info["title"], info["artists"], text, top_themes(cache))
stamp = {
"model": settings.model,
"prompt_version": settings.prompt_version,

View file

@ -188,7 +188,8 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None
playlists = with_retries(lambda: client.users_playlists_list())
total_ranges = 0
total_tracks = 0
total_found = 0
total_removed = 0
for playlist in playlists:
full = with_retries(lambda: client.users_playlists(playlist.kind))
if full is None:
@ -202,13 +203,14 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None
n = sum(to - frm for frm, to in ranges)
total_ranges += len(ranges)
total_tracks += n
total_found += n
if dry_run:
print(f"{playlist.title}: найдено дублей {n} ({len(ranges)} диапазон(ов))")
continue
revision = full.revision or 1
removed = 0
for frm, to in sorted(ranges, key=lambda r: -r[0]):
try:
updated = with_retries(lambda: client.users_playlists_delete_track(playlist.kind, frm, to, revision=revision))
@ -220,13 +222,16 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None
except BadRequestError as e:
print(f" ошибка при удалении из «{playlist.title}» ({e}), прерываю")
break
removed += to - frm
time.sleep(delay)
print(f"{playlist.title}: удалено {n} дублей")
total_removed += removed
suffix = "" if removed == n else f" из {n} найденных"
print(f"{playlist.title}: удалено {removed} дублей{suffix}")
if dry_run:
if total_tracks:
print(f"\nВсего дублей найдено: {total_tracks} в {total_ranges} диапазон(ов). Запустите с --yes для удаления.")
if total_found:
print(f"\nВсего дублей найдено: {total_found} в {total_ranges} диапазон(ов). Запустите с --yes для удаления.")
else:
print("\nДублей не найдено.")
else:
print(f"\nВсего удалено дублей: {total_tracks}")
print(f"\nВсего удалено дублей: {total_removed}")

View file

@ -135,7 +135,7 @@ def fetch_api_languages(
for i, tid in enumerate(missing, 1):
lang, text = _fetch_supplement(client, tid)
cache[tid] = lang
if text is not None:
if text is not None and normalize_api_language(lang) == "RU":
text_cache[tid] = text
if i % 20 == 0:
_atomic_write_json(cache_file, cache)

View file

@ -33,15 +33,13 @@ def with_retries(fn: Callable[[], T], max_attempts: int = 4, base_delay: float =
"""
if max_attempts <= 0:
max_attempts = 1
last_exc: Exception | None = None
last_exc: NetworkError | None = None
for attempt in range(max_attempts):
try:
return fn()
except (NotFoundError, BadRequestError):
raise
except Exception as e:
if isinstance(e, (TypeError, ValueError, AttributeError, NameError)):
raise
except NetworkError as e:
last_exc = e
if attempt < max_attempts - 1:
wait = base_delay * (2**attempt)