merge: M3 auto-tagging

README.md

@@ -15,6 +15,9 @@ once and stored locally, so filtering/searching/sorting are instant and don't bu
 > - **M2** (ingest core): subscription import, free RSS detection, recent-first + deep backfill
 >   from the uploads playlist, enrichment (duration/stats/category, Shorts & livestream
 >   classification), a shared daily quota guard, and a background scheduler.
+> - **M3** (auto-tagging): system tags for channel language (offline detection) and topic
+>   (from YouTube topics + dominant category), regenerated automatically; user tags are
+>   never overwritten.
 
 ## Requirements
 

backend/alembic/versions/0003_tags.py

@@ -0,0 +1,89 @@
+"""tags and channel_tags
+
+Revision ID: 0003_tags
+Revises: 0002_ingest_core
+Create Date: 2026-06-11
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "0003_tags"
+down_revision: Union[str, None] = "0002_ingest_core"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "tags",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column(
+            "user_id",
+            sa.Integer(),
+            sa.ForeignKey("users.id", ondelete="CASCADE"),
+            nullable=True,
+        ),
+        sa.Column("name", sa.String(length=64), nullable=False),
+        sa.Column("color", sa.String(length=16), nullable=True),
+        sa.Column("category", sa.String(length=16), nullable=False, server_default="other"),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.func.now(),
+            nullable=False,
+        ),
+    )
+    op.create_index("ix_tags_user_id", "tags", ["user_id"])
+    # System tag names unique among system tags; user tag names unique per user.
+    op.create_index(
+        "uq_tags_system_name",
+        "tags",
+        ["name"],
+        unique=True,
+        postgresql_where=sa.text("user_id IS NULL"),
+    )
+    op.create_index(
+        "uq_tags_user_name",
+        "tags",
+        ["user_id", "name"],
+        unique=True,
+        postgresql_where=sa.text("user_id IS NOT NULL"),
+    )
+
+    op.create_table(
+        "channel_tags",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column(
+            "channel_id",
+            sa.String(length=32),
+            sa.ForeignKey("channels.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "tag_id",
+            sa.Integer(),
+            sa.ForeignKey("tags.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "user_id",
+            sa.Integer(),
+            sa.ForeignKey("users.id", ondelete="CASCADE"),
+            nullable=True,
+        ),
+        sa.Column("source", sa.String(length=8), nullable=False, server_default="auto"),
+        sa.Column("confidence", sa.Float(), nullable=True),
+        sa.UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),
+    )
+    op.create_index("ix_channel_tags_channel_id", "channel_tags", ["channel_id"])
+    op.create_index("ix_channel_tags_tag_id", "channel_tags", ["tag_id"])
+
+
+def downgrade() -> None:
+    op.drop_table("channel_tags")
+    op.drop_index("uq_tags_user_name", table_name="tags")
+    op.drop_index("uq_tags_system_name", table_name="tags")
+    op.drop_index("ix_tags_user_id", table_name="tags")
+    op.drop_table("tags")

backend/app/config.py

@@ -48,6 +48,14 @@ class Settings(BaseSettings):
     # interactive syncs / enrichment of fresh videos.
     backfill_quota_reserve: int = 2000
 
+    # --- Auto-tagging / feed defaults ---
+    # Number of recent video titles sampled per channel for language detection.
+    autotag_title_sample: int = 40
+    autotag_interval_minutes: int = 30
+    # live_status values hidden from the feed by default. Completed-stream VODs
+    # ("was_live") are real watchable content and stay visible.
+    feed_default_hidden_live: str = "live,upcoming"
+
     @property
     def allowed_email_set(self) -> set[str]:
         return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}

backend/app/main.py

@@ -9,7 +9,7 @@ from starlette.middleware.sessions import SessionMiddleware
 
 from app import auth
 from app.config import settings
-from app.routes import health, sync
+from app.routes import health, sync, tags
 from app.scheduler import shutdown_scheduler, start_scheduler
 
 
@@ -43,6 +43,7 @@ if settings.frontend_origin:
 app.include_router(health.router)
 app.include_router(auth.router)
 app.include_router(sync.router)
+app.include_router(tags.router)
 
 STATIC_DIR = Path(__file__).parent / "static"
 app.mount("/assets", StaticFiles(directory=STATIC_DIR / "assets"), name="assets")

backend/app/models.py

@@ -5,6 +5,7 @@ from sqlalchemy import (
     Boolean,
     Date,
     DateTime,
+    Float,
     ForeignKey,
     Integer,
     JSON,
@@ -155,6 +156,51 @@ class Video(Base):
     channel: Mapped["Channel"] = relationship(back_populates="videos")
 
 
+class Tag(Base):
+    """A label. System tags (user_id NULL) are computed automatically and shared by all
+    users; user tags (user_id set) are personal. category groups them in the UI."""
+
+    __tablename__ = "tags"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    user_id: Mapped[int | None] = mapped_column(
+        ForeignKey("users.id", ondelete="CASCADE"), index=True
+    )
+    name: Mapped[str] = mapped_column(String(64))
+    color: Mapped[str | None] = mapped_column(String(16))
+    # language | topic | other
+    category: Mapped[str] = mapped_column(
+        String(16), default="other", server_default="other"
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now()
+    )
+
+
+class ChannelTag(Base):
+    """Links a tag to a channel. source='auto' rows are managed by the auto-tagger and
+    regenerated freely; source='user' rows are never touched by the system."""
+
+    __tablename__ = "channel_tags"
+    __table_args__ = (
+        UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    channel_id: Mapped[str] = mapped_column(
+        ForeignKey("channels.id", ondelete="CASCADE"), index=True
+    )
+    tag_id: Mapped[int] = mapped_column(
+        ForeignKey("tags.id", ondelete="CASCADE"), index=True
+    )
+    # NULL = system/auto link visible to everyone; set = a specific user's link.
+    user_id: Mapped[int | None] = mapped_column(
+        ForeignKey("users.id", ondelete="CASCADE")
+    )
+    source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto")
+    confidence: Mapped[float | None] = mapped_column(Float)
+
+
 class ApiQuotaUsage(Base):
     """Tracks YouTube Data API units spent per Pacific-time day (the quota resets at
     midnight Pacific). The whole app shares one daily budget."""

backend/app/routes/tags.py

@@ -0,0 +1,43 @@
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy import func, or_, select
+from sqlalchemy.orm import Session
+
+from app.auth import current_user
+from app.db import get_db
+from app.models import ChannelTag, Tag, User
+from app.sync.autotag import run_autotag_all
+
+router = APIRouter(prefix="/api/tags", tags=["tags"])
+
+
+@router.get("")
+def list_tags(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
+    rows = db.execute(
+        select(Tag, func.count(ChannelTag.id))
+        .outerjoin(ChannelTag, ChannelTag.tag_id == Tag.id)
+        .where(or_(Tag.user_id.is_(None), Tag.user_id == user.id))
+        .group_by(Tag.id)
+        .order_by(Tag.category, Tag.name)
+    ).all()
+    return [
+        {
+            "id": tag.id,
+            "name": tag.name,
+            "color": tag.color,
+            "category": tag.category,
+            "system": tag.user_id is None,
+            "channel_count": count,
+        }
+        for tag, count in rows
+    ]
+
+
+@router.post("/recompute")
+def recompute(
+    user: User = Depends(current_user),
+    db: Session = Depends(get_db),
+    only_missing: bool = False,
+) -> dict:
+    if user.role != "admin":
+        raise HTTPException(status_code=403, detail="Admin only")
+    return run_autotag_all(db, only_missing=only_missing)

backend/app/scheduler.py

@@ -6,6 +6,7 @@ from apscheduler.schedulers.background import BackgroundScheduler
 
 from app.config import settings
 from app.db import SessionLocal
+from app.sync.autotag import run_autotag_all
 from app.sync.runner import (
     run_deep_backfill,
     run_enrich,
@@ -48,6 +49,10 @@ def _backfill_job() -> None:
     _job("backfill", work)
 
 
+def _autotag_job() -> None:
+    _job("autotag", lambda db: run_autotag_all(db, only_missing=True))
+
+
 def start_scheduler() -> None:
     global _scheduler
     if not settings.scheduler_enabled or _scheduler is not None:
@@ -65,6 +70,12 @@ def start_scheduler() -> None:
         minutes=settings.backfill_interval_minutes,
         id="backfill",
     )
+    scheduler.add_job(
+        _autotag_job,
+        "interval",
+        minutes=settings.autotag_interval_minutes,
+        id="autotag",
+    )
     scheduler.start()
     _scheduler = scheduler
     logger.info("scheduler started")

backend/app/sync/autotag.py

@@ -0,0 +1,279 @@
+"""Compute system (auto) tags for channels: a language tag and one or more topic tags.
+
+Language comes from the channel's declared default language or, failing that, offline
+detection over a sample of recent video titles. Topics are mapped from YouTube's
+topicDetails categories and the channel's dominant video category. System tags are
+regenerated freely; user tags are never touched here.
+"""
+from sqlalchemy import and_, exists, func, select
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.models import Channel, ChannelTag, Tag, Video
+
+# --- language id (offline, deterministic, small) ---
+try:
+    from py3langid.langid import MODEL_FILE, LanguageIdentifier
+
+    _identifier = LanguageIdentifier.from_pickled_model(MODEL_FILE, norm_probs=True)
+
+    def _classify(text: str):
+        return _identifier.classify(text)
+
+    def _restrict(codes: list[str]) -> None:
+        _identifier.set_languages(codes)
+
+except Exception:  # pragma: no cover - fallback to module-level API
+    import py3langid as _langid
+
+    def _classify(text: str):
+        return _langid.classify(text)
+
+    def _restrict(codes: list[str]) -> None:
+        _langid.set_languages(codes)
+
+
+LANG_NAMES = {
+    "en": "English",
+    "hu": "Hungarian",
+    "de": "German",
+    "fr": "French",
+    "es": "Spanish",
+    "it": "Italian",
+    "pt": "Portuguese",
+    "nl": "Dutch",
+    "ru": "Russian",
+    "pl": "Polish",
+    "ro": "Romanian",
+    "cs": "Czech",
+    "sk": "Slovak",
+    "tr": "Turkish",
+    "sv": "Swedish",
+    "da": "Danish",
+    "no": "Norwegian",
+    "fi": "Finnish",
+    "ja": "Japanese",
+    "ko": "Korean",
+    "zh": "Chinese",
+    "hi": "Hindi",
+    "ar": "Arabic",
+    "uk": "Ukrainian",
+    "el": "Greek",
+}
+# Constrain detection to plausible languages so short/branded titles can't trip the
+# detector into obscure false positives (e.g. Latin, Luxembourgish).
+_restrict(list(LANG_NAMES))
+
+LANG_COLOR = "#3b82f6"
+DEFAULT_TOPIC_COLOR = "#6b7280"
+TOPIC_COLORS = {
+    "Gaming": "#8b5cf6",
+    "Music": "#ec4899",
+    "Cooking": "#f59e0b",
+    "Tech": "#06b6d4",
+    "Education": "#10b981",
+    "Sports": "#ef4444",
+    "Film": "#a855f7",
+    "TV": "#a855f7",
+    "Politics": "#64748b",
+    "Health & Fitness": "#22c55e",
+    "Vehicles": "#eab308",
+    "Animals": "#84cc16",
+    "Fashion & Beauty": "#f472b6",
+    "Travel": "#14b8a6",
+    "Comedy": "#fb923c",
+    "Entertainment": "#c084fc",
+    "Business": "#0ea5e9",
+    "Science": "#06b6d4",
+    "News": "#94a3b8",
+}
+
+# Exact topic slug -> friendly tag (for slugs the heuristics below don't catch).
+_TOPIC_SLUGS = {
+    "Knowledge": "Education",
+    "Food": "Cooking",
+    "Technology": "Tech",
+    # "Lifestyle_(sociology)" is YouTube's catch-all (on ~65% of channels) — too generic
+    # to be a useful filter, so it is intentionally not mapped.
+    "Society": "Society",
+    "Politics": "Politics",
+    "Health": "Health & Fitness",
+    "Physical_fitness": "Health & Fitness",
+    "Vehicle": "Vehicles",
+    "Pet": "Animals",
+    "Tourism": "Travel",
+    "Fashion": "Fashion & Beauty",
+    "Physical_attractiveness": "Fashion & Beauty",
+    "Humor": "Comedy",
+    "Entertainment": "Entertainment",
+    "Film": "Film",
+    "Television_program": "TV",
+    "Performing_arts": "Performing Arts",
+    "Business": "Business",
+    "Military": "Military",
+    "Religion": "Religion",
+    "Hobby": "Hobbies",
+}
+
+# Dominant video category id -> friendly tag (overly generic ones are skipped).
+_CATEGORY_NAMES = {
+    1: "Film",
+    2: "Vehicles",
+    10: "Music",
+    15: "Animals",
+    17: "Sports",
+    19: "Travel",
+    20: "Gaming",
+    23: "Comedy",
+    25: "News",
+    26: "Fashion & Beauty",
+    27: "Education",
+    28: "Tech",
+}
+
+
+def map_topic_slug(slug: str) -> str | None:
+    low = slug.lower()
+    if "game" in low:
+        return "Gaming"
+    if "music" in low:
+        return "Music"
+    if any(s in low for s in ("sport", "football", "basketball", "tennis", "boxing")):
+        return "Sports"
+    return _TOPIC_SLUGS.get(slug)
+
+
+def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None, float]:
+    if channel.default_language:
+        return channel.default_language.split("-")[0].lower(), 0.99
+    titles = (
+        db.execute(
+            select(Video.title)
+            .where(Video.channel_id == channel.id, Video.title.is_not(None))
+            .order_by(Video.published_at.desc())
+            .limit(settings.autotag_title_sample)
+        )
+        .scalars()
+        .all()
+    )
+    text = " . ".join(t for t in titles if t).strip()
+    if len(text) < 15:
+        return None, 0.0
+    lang, confidence = _classify(text)
+    return lang, float(confidence)
+
+
+def compute_channel_topics(db: Session, channel: Channel) -> set[str]:
+    topics: set[str] = set()
+    for url in channel.topic_categories or []:
+        slug = url.rstrip("/").split("/")[-1]
+        name = map_topic_slug(slug)
+        if name:
+            topics.add(name)
+    dominant = db.execute(
+        select(Video.category_id)
+        .where(Video.channel_id == channel.id, Video.category_id.is_not(None))
+        .group_by(Video.category_id)
+        .order_by(func.count().desc())
+        .limit(1)
+    ).scalar_one_or_none()
+    if dominant is not None and dominant in _CATEGORY_NAMES:
+        topics.add(_CATEGORY_NAMES[dominant])
+    return topics
+
+
+def ensure_system_tag(db: Session, name: str, category: str, color: str | None) -> Tag:
+    tag = db.execute(
+        select(Tag).where(Tag.user_id.is_(None), Tag.name == name)
+    ).scalar_one_or_none()
+    if tag is None:
+        tag = Tag(user_id=None, name=name, category=category, color=color)
+        db.add(tag)
+        db.flush()
+    return tag
+
+
+def apply_channel_autotags(db: Session, channel: Channel) -> None:
+    desired: dict[int, float | None] = {}
+
+    lang, confidence = detect_channel_language(db, channel)
+    if lang:
+        tag = ensure_system_tag(db, LANG_NAMES.get(lang, lang.upper()), "language", LANG_COLOR)
+        desired[tag.id] = confidence
+
+    for topic in compute_channel_topics(db, channel):
+        tag = ensure_system_tag(
+            db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR)
+        )
+        desired.setdefault(tag.id, None)
+
+    existing = {
+        ct.tag_id: ct
+        for ct in db.execute(
+            select(ChannelTag).where(
+                ChannelTag.channel_id == channel.id,
+                ChannelTag.user_id.is_(None),
+                ChannelTag.source == "auto",
+            )
+        ).scalars()
+    }
+    for tag_id, ct in existing.items():
+        if tag_id not in desired:
+            db.delete(ct)
+    for tag_id, conf in desired.items():
+        ct = existing.get(tag_id)
+        if ct is None:
+            db.add(
+                ChannelTag(
+                    channel_id=channel.id,
+                    tag_id=tag_id,
+                    user_id=None,
+                    source="auto",
+                    confidence=conf,
+                )
+            )
+        else:
+            ct.confidence = conf
+    db.commit()
+
+
+def run_autotag_all(db: Session, only_missing: bool = False) -> dict:
+    query = select(Channel)
+    if only_missing:
+        query = query.where(
+            ~exists().where(
+                and_(
+                    ChannelTag.channel_id == Channel.id,
+                    ChannelTag.source == "auto",
+                    ChannelTag.user_id.is_(None),
+                )
+            )
+        )
+    channels = db.execute(query).scalars().all()
+    tagged = 0
+    for channel in channels:
+        try:
+            apply_channel_autotags(db, channel)
+            tagged += 1
+        except Exception:
+            db.rollback()
+    removed = _cleanup_orphan_system_tags(db)
+    return {"channels_tagged": tagged, "orphan_tags_removed": removed}
+
+
+def _cleanup_orphan_system_tags(db: Session) -> int:
+    """Delete system tags that no longer link to any channel (e.g. after a mapping change)."""
+    orphans = (
+        db.execute(
+            select(Tag).where(
+                Tag.user_id.is_(None),
+                ~exists().where(ChannelTag.tag_id == Tag.id),
+            )
+        )
+        .scalars()
+        .all()
+    )
+    for tag in orphans:
+        db.delete(tag)
+    db.commit()
+    return len(orphans)

backend/requirements.txt

@@ -9,5 +9,6 @@ authlib>=1.3,<2.0
 httpx>=0.27,<1.0
 feedparser>=6.0,<7.0
 apscheduler>=3.10,<4.0
+py3langid>=0.3,<1.0
 itsdangerous>=2.1,<3.0
 cryptography>=42,<46