mirror of
https://github.com/Horizontal42/Sort_YM.git
synced 2026-11-10 22:41:59 +03:00
tighten retry scope, RU-only lyrics text cache, fix theme pattern and dedupe log
with_retries caught bare Exception with a type whitelist, masking real bugs as network errors under retry/backoff - narrowed back to NetworkError. fetch_api_languages was writing full lyrics text for every language into lyrics_text.json via its track_supplement side channel, breaking the documented RU-only invariant and letting analyze feed foreign-language lyrics through the Russian-only prompt - now gated on normalize_api_language. dedupe_playlists logged the found duplicate count as "removed" even when a mid-loop network/API error cut deletion short. THEME_PATTERN's relaxed regex allowed 2-char themes, contradicting its own test; current_themes was hoisted out of the analyze loop so later prompts never saw themes found earlier in the same run. Both reverted to intended behavior, docs updated to match.
This commit is contained in:
parent
f77fafd240
commit
970fd551e5
6 changed files with 18 additions and 16 deletions
|
|
@ -134,7 +134,7 @@ report/apply/digest --source <url> → source.parse_playlist_url() → (user_id,
|
||||||
fields flatten out.
|
fields flatten out.
|
||||||
|
|
||||||
`themes` are free-form but pattern-constrained to English snake_case
|
`themes` are free-form but pattern-constrained to English snake_case
|
||||||
(`^[a-z][a-z0-9_]{2,29}$`) — in the pilot, an unconstrained field drifted between Russian and
|
(`^[a-z0-9][a-z0-9_-]{2,29}$`) — in the pilot, an unconstrained field drifted between Russian and
|
||||||
English inside a single run, making the values impossible to aggregate; the pattern makes
|
English inside a single run, making the values impossible to aggregate; the pattern makes
|
||||||
Cyrillic mechanically unrepresentable at decode time. They are English for the same reason
|
Cyrillic mechanically unrepresentable at decode time. They are English for the same reason
|
||||||
genre slugs and `ROOT_BUCKET` keys are: they are aggregation keys. `mood` is a closed enum that
|
genre slugs and `ROOT_BUCKET` keys are: they are aggregation keys. `mood` is a closed enum that
|
||||||
|
|
|
||||||
|
|
@ -134,7 +134,7 @@ report/apply/digest --source <url> → source.parse_playlist_url() → (user_id,
|
||||||
ограничение вместо этого — таймаут запроса. `temperature: 0.65`, не 0: при 0 нарративные поля
|
ограничение вместо этого — таймаут запроса. `temperature: 0.65`, не 0: при 0 нарративные поля
|
||||||
становятся одноообразными.
|
становятся одноообразными.
|
||||||
|
|
||||||
`themes` свободного формата, но ограничены паттерном англ. snake_case (`^[a-z][a-z0-9_]{2,29}$`) —
|
`themes` свободного формата, но ограничены паттерном англ. snake_case (`^[a-z0-9][a-z0-9_-]{2,29}$`) —
|
||||||
в пилоте неограниченное поле дрейфовало между русским и английским в одном прогоне, что сделало
|
в пилоте неограниченное поле дрейфовало между русским и английским в одном прогоне, что сделало
|
||||||
значения невозможными для агрегации; паттерн делает кириллицу механически невозможной при разборе.
|
значения невозможными для агрегации; паттерн делает кириллицу механически невозможной при разборе.
|
||||||
Они англ. по той же причине, что и слаги жанров, и `ROOT_BUCKET`: это ключи агрегации. `mood` —
|
Они англ. по той же причине, что и слаги жанров, и `ROOT_BUCKET`: это ключи агрегации. `mood` —
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ STANCES = ["sincere", "ironic", "bitter"]
|
||||||
# фиксируются схемой: в пилоте без паттерна язык плавал между английским и русским внутри
|
# фиксируются схемой: в пилоте без паттерна язык плавал между английским и русским внутри
|
||||||
# одного прогона. Паттерн на уровне JSON Schema делает кириллицу и составные значения
|
# одного прогона. Паттерн на уровне JSON Schema делает кириллицу и составные значения
|
||||||
# механически невозможными на этапе генерации, а не отлавливаемыми постфактум.
|
# механически невозможными на этапе генерации, а не отлавливаемыми постфактум.
|
||||||
THEME_PATTERN = "^[a-z0-9_-]{2,30}$"
|
THEME_PATTERN = "^[a-z0-9][a-z0-9_-]{2,29}$"
|
||||||
|
|
||||||
|
|
||||||
def _enum(values: list[str]) -> dict:
|
def _enum(values: list[str]) -> dict:
|
||||||
|
|
@ -245,10 +245,9 @@ def analyze_tracks(
|
||||||
|
|
||||||
print(f"Разбор текстов: {len(pending)} треков, модель {settings.model}")
|
print(f"Разбор текстов: {len(pending)} треков, модель {settings.model}")
|
||||||
failed = 0
|
failed = 0
|
||||||
current_themes = top_themes(cache)
|
|
||||||
for i, (tid, text, text_hash) in enumerate(pending, 1):
|
for i, (tid, text, text_hash) in enumerate(pending, 1):
|
||||||
info = meta.get(tid, {"title": tid, "artists": []})
|
info = meta.get(tid, {"title": tid, "artists": []})
|
||||||
prompt = build_prompt(info["title"], info["artists"], text, current_themes)
|
prompt = build_prompt(info["title"], info["artists"], text, top_themes(cache))
|
||||||
stamp = {
|
stamp = {
|
||||||
"model": settings.model,
|
"model": settings.model,
|
||||||
"prompt_version": settings.prompt_version,
|
"prompt_version": settings.prompt_version,
|
||||||
|
|
|
||||||
|
|
@ -188,7 +188,8 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None
|
||||||
playlists = with_retries(lambda: client.users_playlists_list())
|
playlists = with_retries(lambda: client.users_playlists_list())
|
||||||
|
|
||||||
total_ranges = 0
|
total_ranges = 0
|
||||||
total_tracks = 0
|
total_found = 0
|
||||||
|
total_removed = 0
|
||||||
for playlist in playlists:
|
for playlist in playlists:
|
||||||
full = with_retries(lambda: client.users_playlists(playlist.kind))
|
full = with_retries(lambda: client.users_playlists(playlist.kind))
|
||||||
if full is None:
|
if full is None:
|
||||||
|
|
@ -202,13 +203,14 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None
|
||||||
|
|
||||||
n = sum(to - frm for frm, to in ranges)
|
n = sum(to - frm for frm, to in ranges)
|
||||||
total_ranges += len(ranges)
|
total_ranges += len(ranges)
|
||||||
total_tracks += n
|
total_found += n
|
||||||
|
|
||||||
if dry_run:
|
if dry_run:
|
||||||
print(f"{playlist.title}: найдено дублей {n} ({len(ranges)} диапазон(ов))")
|
print(f"{playlist.title}: найдено дублей {n} ({len(ranges)} диапазон(ов))")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
revision = full.revision or 1
|
revision = full.revision or 1
|
||||||
|
removed = 0
|
||||||
for frm, to in sorted(ranges, key=lambda r: -r[0]):
|
for frm, to in sorted(ranges, key=lambda r: -r[0]):
|
||||||
try:
|
try:
|
||||||
updated = with_retries(lambda: client.users_playlists_delete_track(playlist.kind, frm, to, revision=revision))
|
updated = with_retries(lambda: client.users_playlists_delete_track(playlist.kind, frm, to, revision=revision))
|
||||||
|
|
@ -220,13 +222,16 @@ def dedupe_playlists(client: Client, delay: float, dry_run: bool = True) -> None
|
||||||
except BadRequestError as e:
|
except BadRequestError as e:
|
||||||
print(f" ошибка при удалении из «{playlist.title}» ({e}), прерываю")
|
print(f" ошибка при удалении из «{playlist.title}» ({e}), прерываю")
|
||||||
break
|
break
|
||||||
|
removed += to - frm
|
||||||
time.sleep(delay)
|
time.sleep(delay)
|
||||||
print(f"{playlist.title}: удалено {n} дублей")
|
total_removed += removed
|
||||||
|
suffix = "" if removed == n else f" из {n} найденных"
|
||||||
|
print(f"{playlist.title}: удалено {removed} дублей{suffix}")
|
||||||
|
|
||||||
if dry_run:
|
if dry_run:
|
||||||
if total_tracks:
|
if total_found:
|
||||||
print(f"\nВсего дублей найдено: {total_tracks} в {total_ranges} диапазон(ов). Запустите с --yes для удаления.")
|
print(f"\nВсего дублей найдено: {total_found} в {total_ranges} диапазон(ов). Запустите с --yes для удаления.")
|
||||||
else:
|
else:
|
||||||
print("\nДублей не найдено.")
|
print("\nДублей не найдено.")
|
||||||
else:
|
else:
|
||||||
print(f"\nВсего удалено дублей: {total_tracks}")
|
print(f"\nВсего удалено дублей: {total_removed}")
|
||||||
|
|
|
||||||
|
|
@ -135,7 +135,7 @@ def fetch_api_languages(
|
||||||
for i, tid in enumerate(missing, 1):
|
for i, tid in enumerate(missing, 1):
|
||||||
lang, text = _fetch_supplement(client, tid)
|
lang, text = _fetch_supplement(client, tid)
|
||||||
cache[tid] = lang
|
cache[tid] = lang
|
||||||
if text is not None:
|
if text is not None and normalize_api_language(lang) == "RU":
|
||||||
text_cache[tid] = text
|
text_cache[tid] = text
|
||||||
if i % 20 == 0:
|
if i % 20 == 0:
|
||||||
_atomic_write_json(cache_file, cache)
|
_atomic_write_json(cache_file, cache)
|
||||||
|
|
|
||||||
|
|
@ -33,15 +33,13 @@ def with_retries(fn: Callable[[], T], max_attempts: int = 4, base_delay: float =
|
||||||
"""
|
"""
|
||||||
if max_attempts <= 0:
|
if max_attempts <= 0:
|
||||||
max_attempts = 1
|
max_attempts = 1
|
||||||
last_exc: Exception | None = None
|
last_exc: NetworkError | None = None
|
||||||
for attempt in range(max_attempts):
|
for attempt in range(max_attempts):
|
||||||
try:
|
try:
|
||||||
return fn()
|
return fn()
|
||||||
except (NotFoundError, BadRequestError):
|
except (NotFoundError, BadRequestError):
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except NetworkError as e:
|
||||||
if isinstance(e, (TypeError, ValueError, AttributeError, NameError)):
|
|
||||||
raise
|
|
||||||
last_exc = e
|
last_exc = e
|
||||||
if attempt < max_attempts - 1:
|
if attempt < max_attempts - 1:
|
||||||
wait = base_delay * (2**attempt)
|
wait = base_delay * (2**attempt)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue