From 68dad91e8acece52abd7b0fc894788b718181579 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Thu, 11 Jun 2026 01:57:19 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20M3=20=E2=80=94=20automatic=20channel=20?= =?UTF-8?q?tagging=20(language=20+=20topic=20system=20tags)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Tag and ChannelTag models + migration 0003 (partial unique indexes split system vs per-user tag names) - Offline language detection (py3langid) constrained to a curated language set, with the channel's declared default language as a strong prior - Topic tags mapped from YouTube topicDetails + dominant video category; the generic "Lifestyle" catch-all is intentionally dropped - System (auto) tags are regenerated idempotently and never touch user tags; orphaned system tags are cleaned up - GET /api/tags and admin POST /api/tags/recompute; scheduled autotag job --- README.md | 3 + backend/alembic/versions/0003_tags.py | 89 ++++++++ backend/app/config.py | 8 + backend/app/main.py | 3 +- backend/app/models.py | 46 +++++ backend/app/routes/tags.py | 43 ++++ backend/app/scheduler.py | 11 + backend/app/sync/autotag.py | 279 ++++++++++++++++++++++++++ backend/requirements.txt | 1 + 9 files changed, 482 insertions(+), 1 deletion(-) create mode 100644 backend/alembic/versions/0003_tags.py create mode 100644 backend/app/routes/tags.py create mode 100644 backend/app/sync/autotag.py diff --git a/README.md b/README.md index 7a7ef25..7361bfa 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,9 @@ once and stored locally, so filtering/searching/sorting are instant and don't bu > - **M2** (ingest core): subscription import, free RSS detection, recent-first + deep backfill > from the uploads playlist, enrichment (duration/stats/category, Shorts & livestream > classification), a shared daily quota guard, and a background scheduler. +> - **M3** (auto-tagging): system tags for channel language (offline detection) and topic +> (from YouTube topics + dominant category), regenerated automatically; user tags are +> never overwritten. ## Requirements diff --git a/backend/alembic/versions/0003_tags.py b/backend/alembic/versions/0003_tags.py new file mode 100644 index 0000000..1dafaf9 --- /dev/null +++ b/backend/alembic/versions/0003_tags.py @@ -0,0 +1,89 @@ +"""tags and channel_tags + +Revision ID: 0003_tags +Revises: 0002_ingest_core +Create Date: 2026-06-11 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "0003_tags" +down_revision: Union[str, None] = "0002_ingest_core" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "tags", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=True, + ), + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("color", sa.String(length=16), nullable=True), + sa.Column("category", sa.String(length=16), nullable=False, server_default="other"), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + ) + op.create_index("ix_tags_user_id", "tags", ["user_id"]) + # System tag names unique among system tags; user tag names unique per user. + op.create_index( + "uq_tags_system_name", + "tags", + ["name"], + unique=True, + postgresql_where=sa.text("user_id IS NULL"), + ) + op.create_index( + "uq_tags_user_name", + "tags", + ["user_id", "name"], + unique=True, + postgresql_where=sa.text("user_id IS NOT NULL"), + ) + + op.create_table( + "channel_tags", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "channel_id", + sa.String(length=32), + sa.ForeignKey("channels.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "tag_id", + sa.Integer(), + sa.ForeignKey("tags.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=True, + ), + sa.Column("source", sa.String(length=8), nullable=False, server_default="auto"), + sa.Column("confidence", sa.Float(), nullable=True), + sa.UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"), + ) + op.create_index("ix_channel_tags_channel_id", "channel_tags", ["channel_id"]) + op.create_index("ix_channel_tags_tag_id", "channel_tags", ["tag_id"]) + + +def downgrade() -> None: + op.drop_table("channel_tags") + op.drop_index("uq_tags_user_name", table_name="tags") + op.drop_index("uq_tags_system_name", table_name="tags") + op.drop_index("ix_tags_user_id", table_name="tags") + op.drop_table("tags") diff --git a/backend/app/config.py b/backend/app/config.py index 5142571..c149a52 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -48,6 +48,14 @@ class Settings(BaseSettings): # interactive syncs / enrichment of fresh videos. backfill_quota_reserve: int = 2000 + # --- Auto-tagging / feed defaults --- + # Number of recent video titles sampled per channel for language detection. + autotag_title_sample: int = 40 + autotag_interval_minutes: int = 30 + # live_status values hidden from the feed by default. Completed-stream VODs + # ("was_live") are real watchable content and stay visible. + feed_default_hidden_live: str = "live,upcoming" + @property def allowed_email_set(self) -> set[str]: return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()} diff --git a/backend/app/main.py b/backend/app/main.py index 0f40203..0b97348 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -9,7 +9,7 @@ from starlette.middleware.sessions import SessionMiddleware from app import auth from app.config import settings -from app.routes import health, sync +from app.routes import health, sync, tags from app.scheduler import shutdown_scheduler, start_scheduler @@ -43,6 +43,7 @@ if settings.frontend_origin: app.include_router(health.router) app.include_router(auth.router) app.include_router(sync.router) +app.include_router(tags.router) STATIC_DIR = Path(__file__).parent / "static" app.mount("/assets", StaticFiles(directory=STATIC_DIR / "assets"), name="assets") diff --git a/backend/app/models.py b/backend/app/models.py index 3087abd..c5f9012 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -5,6 +5,7 @@ from sqlalchemy import ( Boolean, Date, DateTime, + Float, ForeignKey, Integer, JSON, @@ -155,6 +156,51 @@ class Video(Base): channel: Mapped["Channel"] = relationship(back_populates="videos") +class Tag(Base): + """A label. System tags (user_id NULL) are computed automatically and shared by all + users; user tags (user_id set) are personal. category groups them in the UI.""" + + __tablename__ = "tags" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + name: Mapped[str] = mapped_column(String(64)) + color: Mapped[str | None] = mapped_column(String(16)) + # language | topic | other + category: Mapped[str] = mapped_column( + String(16), default="other", server_default="other" + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + +class ChannelTag(Base): + """Links a tag to a channel. source='auto' rows are managed by the auto-tagger and + regenerated freely; source='user' rows are never touched by the system.""" + + __tablename__ = "channel_tags" + __table_args__ = ( + UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + channel_id: Mapped[str] = mapped_column( + ForeignKey("channels.id", ondelete="CASCADE"), index=True + ) + tag_id: Mapped[int] = mapped_column( + ForeignKey("tags.id", ondelete="CASCADE"), index=True + ) + # NULL = system/auto link visible to everyone; set = a specific user's link. + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE") + ) + source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto") + confidence: Mapped[float | None] = mapped_column(Float) + + class ApiQuotaUsage(Base): """Tracks YouTube Data API units spent per Pacific-time day (the quota resets at midnight Pacific). The whole app shares one daily budget.""" diff --git a/backend/app/routes/tags.py b/backend/app/routes/tags.py new file mode 100644 index 0000000..0269f20 --- /dev/null +++ b/backend/app/routes/tags.py @@ -0,0 +1,43 @@ +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import func, or_, select +from sqlalchemy.orm import Session + +from app.auth import current_user +from app.db import get_db +from app.models import ChannelTag, Tag, User +from app.sync.autotag import run_autotag_all + +router = APIRouter(prefix="/api/tags", tags=["tags"]) + + +@router.get("") +def list_tags(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]: + rows = db.execute( + select(Tag, func.count(ChannelTag.id)) + .outerjoin(ChannelTag, ChannelTag.tag_id == Tag.id) + .where(or_(Tag.user_id.is_(None), Tag.user_id == user.id)) + .group_by(Tag.id) + .order_by(Tag.category, Tag.name) + ).all() + return [ + { + "id": tag.id, + "name": tag.name, + "color": tag.color, + "category": tag.category, + "system": tag.user_id is None, + "channel_count": count, + } + for tag, count in rows + ] + + +@router.post("/recompute") +def recompute( + user: User = Depends(current_user), + db: Session = Depends(get_db), + only_missing: bool = False, +) -> dict: + if user.role != "admin": + raise HTTPException(status_code=403, detail="Admin only") + return run_autotag_all(db, only_missing=only_missing) diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 3b3b3ad..d95e589 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -6,6 +6,7 @@ from apscheduler.schedulers.background import BackgroundScheduler from app.config import settings from app.db import SessionLocal +from app.sync.autotag import run_autotag_all from app.sync.runner import ( run_deep_backfill, run_enrich, @@ -48,6 +49,10 @@ def _backfill_job() -> None: _job("backfill", work) +def _autotag_job() -> None: + _job("autotag", lambda db: run_autotag_all(db, only_missing=True)) + + def start_scheduler() -> None: global _scheduler if not settings.scheduler_enabled or _scheduler is not None: @@ -65,6 +70,12 @@ def start_scheduler() -> None: minutes=settings.backfill_interval_minutes, id="backfill", ) + scheduler.add_job( + _autotag_job, + "interval", + minutes=settings.autotag_interval_minutes, + id="autotag", + ) scheduler.start() _scheduler = scheduler logger.info("scheduler started") diff --git a/backend/app/sync/autotag.py b/backend/app/sync/autotag.py new file mode 100644 index 0000000..121efe3 --- /dev/null +++ b/backend/app/sync/autotag.py @@ -0,0 +1,279 @@ +"""Compute system (auto) tags for channels: a language tag and one or more topic tags. + +Language comes from the channel's declared default language or, failing that, offline +detection over a sample of recent video titles. Topics are mapped from YouTube's +topicDetails categories and the channel's dominant video category. System tags are +regenerated freely; user tags are never touched here. +""" +from sqlalchemy import and_, exists, func, select +from sqlalchemy.orm import Session + +from app.config import settings +from app.models import Channel, ChannelTag, Tag, Video + +# --- language id (offline, deterministic, small) --- +try: + from py3langid.langid import MODEL_FILE, LanguageIdentifier + + _identifier = LanguageIdentifier.from_pickled_model(MODEL_FILE, norm_probs=True) + + def _classify(text: str): + return _identifier.classify(text) + + def _restrict(codes: list[str]) -> None: + _identifier.set_languages(codes) + +except Exception: # pragma: no cover - fallback to module-level API + import py3langid as _langid + + def _classify(text: str): + return _langid.classify(text) + + def _restrict(codes: list[str]) -> None: + _langid.set_languages(codes) + + +LANG_NAMES = { + "en": "English", + "hu": "Hungarian", + "de": "German", + "fr": "French", + "es": "Spanish", + "it": "Italian", + "pt": "Portuguese", + "nl": "Dutch", + "ru": "Russian", + "pl": "Polish", + "ro": "Romanian", + "cs": "Czech", + "sk": "Slovak", + "tr": "Turkish", + "sv": "Swedish", + "da": "Danish", + "no": "Norwegian", + "fi": "Finnish", + "ja": "Japanese", + "ko": "Korean", + "zh": "Chinese", + "hi": "Hindi", + "ar": "Arabic", + "uk": "Ukrainian", + "el": "Greek", +} +# Constrain detection to plausible languages so short/branded titles can't trip the +# detector into obscure false positives (e.g. Latin, Luxembourgish). +_restrict(list(LANG_NAMES)) + +LANG_COLOR = "#3b82f6" +DEFAULT_TOPIC_COLOR = "#6b7280" +TOPIC_COLORS = { + "Gaming": "#8b5cf6", + "Music": "#ec4899", + "Cooking": "#f59e0b", + "Tech": "#06b6d4", + "Education": "#10b981", + "Sports": "#ef4444", + "Film": "#a855f7", + "TV": "#a855f7", + "Politics": "#64748b", + "Health & Fitness": "#22c55e", + "Vehicles": "#eab308", + "Animals": "#84cc16", + "Fashion & Beauty": "#f472b6", + "Travel": "#14b8a6", + "Comedy": "#fb923c", + "Entertainment": "#c084fc", + "Business": "#0ea5e9", + "Science": "#06b6d4", + "News": "#94a3b8", +} + +# Exact topic slug -> friendly tag (for slugs the heuristics below don't catch). +_TOPIC_SLUGS = { + "Knowledge": "Education", + "Food": "Cooking", + "Technology": "Tech", + # "Lifestyle_(sociology)" is YouTube's catch-all (on ~65% of channels) — too generic + # to be a useful filter, so it is intentionally not mapped. + "Society": "Society", + "Politics": "Politics", + "Health": "Health & Fitness", + "Physical_fitness": "Health & Fitness", + "Vehicle": "Vehicles", + "Pet": "Animals", + "Tourism": "Travel", + "Fashion": "Fashion & Beauty", + "Physical_attractiveness": "Fashion & Beauty", + "Humor": "Comedy", + "Entertainment": "Entertainment", + "Film": "Film", + "Television_program": "TV", + "Performing_arts": "Performing Arts", + "Business": "Business", + "Military": "Military", + "Religion": "Religion", + "Hobby": "Hobbies", +} + +# Dominant video category id -> friendly tag (overly generic ones are skipped). +_CATEGORY_NAMES = { + 1: "Film", + 2: "Vehicles", + 10: "Music", + 15: "Animals", + 17: "Sports", + 19: "Travel", + 20: "Gaming", + 23: "Comedy", + 25: "News", + 26: "Fashion & Beauty", + 27: "Education", + 28: "Tech", +} + + +def map_topic_slug(slug: str) -> str | None: + low = slug.lower() + if "game" in low: + return "Gaming" + if "music" in low: + return "Music" + if any(s in low for s in ("sport", "football", "basketball", "tennis", "boxing")): + return "Sports" + return _TOPIC_SLUGS.get(slug) + + +def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None, float]: + if channel.default_language: + return channel.default_language.split("-")[0].lower(), 0.99 + titles = ( + db.execute( + select(Video.title) + .where(Video.channel_id == channel.id, Video.title.is_not(None)) + .order_by(Video.published_at.desc()) + .limit(settings.autotag_title_sample) + ) + .scalars() + .all() + ) + text = " . ".join(t for t in titles if t).strip() + if len(text) < 15: + return None, 0.0 + lang, confidence = _classify(text) + return lang, float(confidence) + + +def compute_channel_topics(db: Session, channel: Channel) -> set[str]: + topics: set[str] = set() + for url in channel.topic_categories or []: + slug = url.rstrip("/").split("/")[-1] + name = map_topic_slug(slug) + if name: + topics.add(name) + dominant = db.execute( + select(Video.category_id) + .where(Video.channel_id == channel.id, Video.category_id.is_not(None)) + .group_by(Video.category_id) + .order_by(func.count().desc()) + .limit(1) + ).scalar_one_or_none() + if dominant is not None and dominant in _CATEGORY_NAMES: + topics.add(_CATEGORY_NAMES[dominant]) + return topics + + +def ensure_system_tag(db: Session, name: str, category: str, color: str | None) -> Tag: + tag = db.execute( + select(Tag).where(Tag.user_id.is_(None), Tag.name == name) + ).scalar_one_or_none() + if tag is None: + tag = Tag(user_id=None, name=name, category=category, color=color) + db.add(tag) + db.flush() + return tag + + +def apply_channel_autotags(db: Session, channel: Channel) -> None: + desired: dict[int, float | None] = {} + + lang, confidence = detect_channel_language(db, channel) + if lang: + tag = ensure_system_tag(db, LANG_NAMES.get(lang, lang.upper()), "language", LANG_COLOR) + desired[tag.id] = confidence + + for topic in compute_channel_topics(db, channel): + tag = ensure_system_tag( + db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR) + ) + desired.setdefault(tag.id, None) + + existing = { + ct.tag_id: ct + for ct in db.execute( + select(ChannelTag).where( + ChannelTag.channel_id == channel.id, + ChannelTag.user_id.is_(None), + ChannelTag.source == "auto", + ) + ).scalars() + } + for tag_id, ct in existing.items(): + if tag_id not in desired: + db.delete(ct) + for tag_id, conf in desired.items(): + ct = existing.get(tag_id) + if ct is None: + db.add( + ChannelTag( + channel_id=channel.id, + tag_id=tag_id, + user_id=None, + source="auto", + confidence=conf, + ) + ) + else: + ct.confidence = conf + db.commit() + + +def run_autotag_all(db: Session, only_missing: bool = False) -> dict: + query = select(Channel) + if only_missing: + query = query.where( + ~exists().where( + and_( + ChannelTag.channel_id == Channel.id, + ChannelTag.source == "auto", + ChannelTag.user_id.is_(None), + ) + ) + ) + channels = db.execute(query).scalars().all() + tagged = 0 + for channel in channels: + try: + apply_channel_autotags(db, channel) + tagged += 1 + except Exception: + db.rollback() + removed = _cleanup_orphan_system_tags(db) + return {"channels_tagged": tagged, "orphan_tags_removed": removed} + + +def _cleanup_orphan_system_tags(db: Session) -> int: + """Delete system tags that no longer link to any channel (e.g. after a mapping change).""" + orphans = ( + db.execute( + select(Tag).where( + Tag.user_id.is_(None), + ~exists().where(ChannelTag.tag_id == Tag.id), + ) + ) + .scalars() + .all() + ) + for tag in orphans: + db.delete(tag) + db.commit() + return len(orphans) diff --git a/backend/requirements.txt b/backend/requirements.txt index 44e98f0..351833b 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -9,5 +9,6 @@ authlib>=1.3,<2.0 httpx>=0.27,<1.0 feedparser>=6.0,<7.0 apscheduler>=3.10,<4.0 +py3langid>=0.3,<1.0 itsdangerous>=2.1,<3.0 cryptography>=42,<46