feat(titles): normalize video titles for display (feed, search, downloads)
Noisy YouTube titles are cleaned for display + storage, raw kept in videos.original_title: - app/titles.normalize_title: strip emoji/symbols (keep accents), remove trailing SEO hashtag clusters (keep numeric #3 episode markers), context-aware de-shout (mostly-ALL-CAPS titles -> Title Case with an acronym whitelist + function-word lowercasing; otherwise only long all-caps words), collapse repeated punctuation - applied at enrichment (sync/videos.py) and in the download worker (ad-hoc yt-dlp titles); catalog downloads inherit the normalized title automatically - migration 0039: add original_title, preserve raw, rewrite title (generated search_vector regenerates); reversible via original_title Backfill on localdev: 122115/273417 titles normalized in ~2 min. Verified in the feed + on real messy samples (emoji/de-shout/hashtags), accents + acronyms (PS5/AI/USA/PC) preserved.
This commit is contained in:
parent
b200f61642
commit
d55799cbc1
5 changed files with 140 additions and 4 deletions
49
backend/alembic/versions/0039_video_title_normalize.py
Normal file
49
backend/alembic/versions/0039_video_title_normalize.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""normalize video titles for display; keep the raw one in original_title
|
||||
|
||||
Revision ID: 0039_title_normalize
|
||||
Revises: 0038_asset_gc_notified
|
||||
Create Date: 2026-07-03
|
||||
|
||||
Adds videos.original_title (the raw YouTube title) and rewrites videos.title to a normalized,
|
||||
display-friendly form (emoji stripped, ALL-CAPS de-shouted, trailing SEO hashtags removed —
|
||||
see app.titles). Reversible: original_title preserves the source, and title can be re-derived.
|
||||
The generated search_vector column regenerates automatically as each title is updated.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from app.titles import normalize_title
|
||||
|
||||
revision: str = "0039_title_normalize"
|
||||
down_revision: Union[str, None] = "0038_asset_gc_notified"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
_BATCH = 2000
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("videos", sa.Column("original_title", sa.Text(), nullable=True))
|
||||
conn = op.get_bind()
|
||||
# Preserve the raw title first (fast, set-based).
|
||||
conn.execute(sa.text("UPDATE videos SET original_title = title WHERE title IS NOT NULL"))
|
||||
# Then rewrite title to the normalized form, batched (each update regenerates its FTS vector).
|
||||
rows = conn.execute(
|
||||
sa.text("SELECT id, original_title FROM videos WHERE original_title IS NOT NULL")
|
||||
).fetchall()
|
||||
changes = []
|
||||
for vid, raw in rows:
|
||||
norm = normalize_title(raw)
|
||||
if norm != raw:
|
||||
changes.append({"i": vid, "t": norm})
|
||||
stmt = sa.text("UPDATE videos SET title = :t WHERE id = :i")
|
||||
for start in range(0, len(changes), _BATCH):
|
||||
conn.execute(stmt, changes[start : start + _BATCH])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
conn.execute(sa.text("UPDATE videos SET title = original_title WHERE original_title IS NOT NULL"))
|
||||
op.drop_column("videos", "original_title")
|
||||
Loading…
Add table
Add a link
Reference in a new issue