peter/siftlode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merge: M3 auto-tagging

Browse source
This commit is contained in:
npeter83 2026-06-11 01:57:19 +02:00
commit 96f9c5a797
9 changed files with 482 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
README.md
View file

@ -15,6 +15,9 @@ once and stored locally, so filtering/searching/sorting are instant and don't bu
> - **M2** (ingest core): subscription import, free RSS detection, recent-first + deep backfill
> from the uploads playlist, enrichment (duration/stats/category, Shorts & livestream
> classification), a shared daily quota guard, and a background scheduler.
> - **M3** (auto-tagging): system tags for channel language (offline detection) and topic
> (from YouTube topics + dominant category), regenerated automatically; user tags are
> never overwritten.
## Requirements

89
backend/alembic/versions/0003_tags.py Normal file
View file

@ -0,0 +1,89 @@
"""tags and channel_tags
Revision ID: 0003_tags
Revises: 0002_ingest_core
Create Date: 2026-06-11
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "0003_tags"
down_revision: Union[str, None] = "0002_ingest_core"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"tags",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column(
"user_id",
sa.Integer(),
sa.ForeignKey("users.id", ondelete="CASCADE"),
nullable=True,
),
sa.Column("name", sa.String(length=64), nullable=False),
sa.Column("color", sa.String(length=16), nullable=True),
sa.Column("category", sa.String(length=16), nullable=False, server_default="other"),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.func.now(),
nullable=False,
),
)
op.create_index("ix_tags_user_id", "tags", ["user_id"])
# System tag names unique among system tags; user tag names unique per user.
op.create_index(
"uq_tags_system_name",
"tags",
["name"],
unique=True,
postgresql_where=sa.text("user_id IS NULL"),
)
op.create_index(
"uq_tags_user_name",
"tags",
["user_id", "name"],
unique=True,
postgresql_where=sa.text("user_id IS NOT NULL"),
)
op.create_table(
"channel_tags",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column(
"channel_id",
sa.String(length=32),
sa.ForeignKey("channels.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"tag_id",
sa.Integer(),
sa.ForeignKey("tags.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"user_id",
sa.Integer(),
sa.ForeignKey("users.id", ondelete="CASCADE"),
nullable=True,
),
sa.Column("source", sa.String(length=8), nullable=False, server_default="auto"),
sa.Column("confidence", sa.Float(), nullable=True),
sa.UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),
)
op.create_index("ix_channel_tags_channel_id", "channel_tags", ["channel_id"])
op.create_index("ix_channel_tags_tag_id", "channel_tags", ["tag_id"])
def downgrade() -> None:
op.drop_table("channel_tags")
op.drop_index("uq_tags_user_name", table_name="tags")
op.drop_index("uq_tags_system_name", table_name="tags")
op.drop_index("ix_tags_user_id", table_name="tags")
op.drop_table("tags")

8
backend/app/config.py
View file

@ -48,6 +48,14 @@ class Settings(BaseSettings):
# interactive syncs / enrichment of fresh videos.
backfill_quota_reserve: int = 2000
# --- Auto-tagging / feed defaults ---
# Number of recent video titles sampled per channel for language detection.
autotag_title_sample: int = 40
autotag_interval_minutes: int = 30
# live_status values hidden from the feed by default. Completed-stream VODs
# ("was_live") are real watchable content and stay visible.
feed_default_hidden_live: str = "live,upcoming"
@property
def allowed_email_set(self) -> set[str]:
return {e.strip().lower() for e in self.allowed_emails.split(",") if e.strip()}

3
backend/app/main.py
View file

@ -9,7 +9,7 @@ from starlette.middleware.sessions import SessionMiddleware
from app import auth
from app.config import settings
from app.routes import health, sync
from app.routes import health, sync, tags
from app.scheduler import shutdown_scheduler, start_scheduler
@ -43,6 +43,7 @@ if settings.frontend_origin:
app.include_router(health.router)
app.include_router(auth.router)
app.include_router(sync.router)
app.include_router(tags.router)
STATIC_DIR = Path(__file__).parent / "static"
app.mount("/assets", StaticFiles(directory=STATIC_DIR / "assets"), name="assets")

46
backend/app/models.py
View file

@ -5,6 +5,7 @@ from sqlalchemy import (
Boolean,
Date,
DateTime,
Float,
ForeignKey,
Integer,
JSON,
@ -155,6 +156,51 @@ class Video(Base):
channel: Mapped["Channel"] = relationship(back_populates="videos")
class Tag(Base):
"""A label. System tags (user_id NULL) are computed automatically and shared by all
users; user tags (user_id set) are personal. category groups them in the UI."""
__tablename__ = "tags"
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
name: Mapped[str] = mapped_column(String(64))
color: Mapped[str | None] = mapped_column(String(16))
# language | topic | other
category: Mapped[str] = mapped_column(
String(16), default="other", server_default="other"
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
class ChannelTag(Base):
"""Links a tag to a channel. source='auto' rows are managed by the auto-tagger and
regenerated freely; source='user' rows are never touched by the system."""
__tablename__ = "channel_tags"
__table_args__ = (
UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),
)
id: Mapped[int] = mapped_column(primary_key=True)
channel_id: Mapped[str] = mapped_column(
ForeignKey("channels.id", ondelete="CASCADE"), index=True
)
tag_id: Mapped[int] = mapped_column(
ForeignKey("tags.id", ondelete="CASCADE"), index=True
)
# NULL = system/auto link visible to everyone; set = a specific user's link.
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE")
)
source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto")
confidence: Mapped[float | None] = mapped_column(Float)
class ApiQuotaUsage(Base):
"""Tracks YouTube Data API units spent per Pacific-time day (the quota resets at
midnight Pacific). The whole app shares one daily budget."""

43
backend/app/routes/tags.py Normal file
View file

@ -0,0 +1,43 @@
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import func, or_, select
from sqlalchemy.orm import Session
from app.auth import current_user
from app.db import get_db
from app.models import ChannelTag, Tag, User
from app.sync.autotag import run_autotag_all
router = APIRouter(prefix="/api/tags", tags=["tags"])
@router.get("")
def list_tags(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
rows = db.execute(
select(Tag, func.count(ChannelTag.id))
.outerjoin(ChannelTag, ChannelTag.tag_id == Tag.id)
.where(or_(Tag.user_id.is_(None), Tag.user_id == user.id))
.group_by(Tag.id)
.order_by(Tag.category, Tag.name)
).all()
return [
{
"id": tag.id,
"name": tag.name,
"color": tag.color,
"category": tag.category,
"system": tag.user_id is None,
"channel_count": count,
}
for tag, count in rows
]
@router.post("/recompute")
def recompute(
user: User = Depends(current_user),
db: Session = Depends(get_db),
only_missing: bool = False,
) -> dict:
if user.role != "admin":
raise HTTPException(status_code=403, detail="Admin only")
return run_autotag_all(db, only_missing=only_missing)

11
backend/app/scheduler.py
View file

@ -6,6 +6,7 @@ from apscheduler.schedulers.background import BackgroundScheduler
from app.config import settings
from app.db import SessionLocal
from app.sync.autotag import run_autotag_all
from app.sync.runner import (
run_deep_backfill,
run_enrich,
@ -48,6 +49,10 @@ def _backfill_job() -> None:
_job("backfill", work)
def _autotag_job() -> None:
_job("autotag", lambda db: run_autotag_all(db, only_missing=True))
def start_scheduler() -> None:
global _scheduler
if not settings.scheduler_enabled or _scheduler is not None:
@ -65,6 +70,12 @@ def start_scheduler() -> None:
minutes=settings.backfill_interval_minutes,
id="backfill",
)
scheduler.add_job(
_autotag_job,
"interval",
minutes=settings.autotag_interval_minutes,
id="autotag",
)
scheduler.start()
_scheduler = scheduler
logger.info("scheduler started")

279
backend/app/sync/autotag.py Normal file
View file

@ -0,0 +1,279 @@
"""Compute system (auto) tags for channels: a language tag and one or more topic tags.
Language comes from the channel's declared default language or, failing that, offline
detection over a sample of recent video titles. Topics are mapped from YouTube's
topicDetails categories and the channel's dominant video category. System tags are
regenerated freely; user tags are never touched here.
"""
from sqlalchemy import and_, exists, func, select
from sqlalchemy.orm import Session
from app.config import settings
from app.models import Channel, ChannelTag, Tag, Video
# --- language id (offline, deterministic, small) ---
try:
from py3langid.langid import MODEL_FILE, LanguageIdentifier
_identifier = LanguageIdentifier.from_pickled_model(MODEL_FILE, norm_probs=True)
def _classify(text: str):
return _identifier.classify(text)
def _restrict(codes: list[str]) -> None:
_identifier.set_languages(codes)
except Exception: # pragma: no cover - fallback to module-level API
import py3langid as _langid
def _classify(text: str):
return _langid.classify(text)
def _restrict(codes: list[str]) -> None:
_langid.set_languages(codes)
LANG_NAMES = {
"en": "English",
"hu": "Hungarian",
"de": "German",
"fr": "French",
"es": "Spanish",
"it": "Italian",
"pt": "Portuguese",
"nl": "Dutch",
"ru": "Russian",
"pl": "Polish",
"ro": "Romanian",
"cs": "Czech",
"sk": "Slovak",
"tr": "Turkish",
"sv": "Swedish",
"da": "Danish",
"no": "Norwegian",
"fi": "Finnish",
"ja": "Japanese",
"ko": "Korean",
"zh": "Chinese",
"hi": "Hindi",
"ar": "Arabic",
"uk": "Ukrainian",
"el": "Greek",
}
# Constrain detection to plausible languages so short/branded titles can't trip the
# detector into obscure false positives (e.g. Latin, Luxembourgish).
_restrict(list(LANG_NAMES))
LANG_COLOR = "#3b82f6"
DEFAULT_TOPIC_COLOR = "#6b7280"
TOPIC_COLORS = {
"Gaming": "#8b5cf6",
"Music": "#ec4899",
"Cooking": "#f59e0b",
"Tech": "#06b6d4",
"Education": "#10b981",
"Sports": "#ef4444",
"Film": "#a855f7",
"TV": "#a855f7",
"Politics": "#64748b",
"Health & Fitness": "#22c55e",
"Vehicles": "#eab308",
"Animals": "#84cc16",
"Fashion & Beauty": "#f472b6",
"Travel": "#14b8a6",
"Comedy": "#fb923c",
"Entertainment": "#c084fc",
"Business": "#0ea5e9",
"Science": "#06b6d4",
"News": "#94a3b8",
}
# Exact topic slug -> friendly tag (for slugs the heuristics below don't catch).
_TOPIC_SLUGS = {
"Knowledge": "Education",
"Food": "Cooking",
"Technology": "Tech",
# "Lifestyle_(sociology)" is YouTube's catch-all (on ~65% of channels) — too generic
# to be a useful filter, so it is intentionally not mapped.
"Society": "Society",
"Politics": "Politics",
"Health": "Health & Fitness",
"Physical_fitness": "Health & Fitness",
"Vehicle": "Vehicles",
"Pet": "Animals",
"Tourism": "Travel",
"Fashion": "Fashion & Beauty",
"Physical_attractiveness": "Fashion & Beauty",
"Humor": "Comedy",
"Entertainment": "Entertainment",
"Film": "Film",
"Television_program": "TV",
"Performing_arts": "Performing Arts",
"Business": "Business",
"Military": "Military",
"Religion": "Religion",
"Hobby": "Hobbies",
}
# Dominant video category id -> friendly tag (overly generic ones are skipped).
_CATEGORY_NAMES = {
1: "Film",
2: "Vehicles",
10: "Music",
15: "Animals",
17: "Sports",
19: "Travel",
20: "Gaming",
23: "Comedy",
25: "News",
26: "Fashion & Beauty",
27: "Education",
28: "Tech",
}
def map_topic_slug(slug: str) -> str | None:
low = slug.lower()
if "game" in low:
return "Gaming"
if "music" in low:
return "Music"
if any(s in low for s in ("sport", "football", "basketball", "tennis", "boxing")):
return "Sports"
return _TOPIC_SLUGS.get(slug)
def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None, float]:
if channel.default_language:
return channel.default_language.split("-")[0].lower(), 0.99
titles = (
db.execute(
select(Video.title)
.where(Video.channel_id == channel.id, Video.title.is_not(None))
.order_by(Video.published_at.desc())
.limit(settings.autotag_title_sample)
)
.scalars()
.all()
)
text = " . ".join(t for t in titles if t).strip()
if len(text) < 15:
return None, 0.0
lang, confidence = _classify(text)
return lang, float(confidence)
def compute_channel_topics(db: Session, channel: Channel) -> set[str]:
topics: set[str] = set()
for url in channel.topic_categories or []:
slug = url.rstrip("/").split("/")[-1]
name = map_topic_slug(slug)
if name:
topics.add(name)
dominant = db.execute(
select(Video.category_id)
.where(Video.channel_id == channel.id, Video.category_id.is_not(None))
.group_by(Video.category_id)
.order_by(func.count().desc())
.limit(1)
).scalar_one_or_none()
if dominant is not None and dominant in _CATEGORY_NAMES:
topics.add(_CATEGORY_NAMES[dominant])
return topics
def ensure_system_tag(db: Session, name: str, category: str, color: str | None) -> Tag:
tag = db.execute(
select(Tag).where(Tag.user_id.is_(None), Tag.name == name)
).scalar_one_or_none()
if tag is None:
tag = Tag(user_id=None, name=name, category=category, color=color)
db.add(tag)
db.flush()
return tag
def apply_channel_autotags(db: Session, channel: Channel) -> None:
desired: dict[int, float | None] = {}
lang, confidence = detect_channel_language(db, channel)
if lang:
tag = ensure_system_tag(db, LANG_NAMES.get(lang, lang.upper()), "language", LANG_COLOR)
desired[tag.id] = confidence
for topic in compute_channel_topics(db, channel):
tag = ensure_system_tag(
db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR)
)
desired.setdefault(tag.id, None)
existing = {
ct.tag_id: ct
for ct in db.execute(
select(ChannelTag).where(
ChannelTag.channel_id == channel.id,
ChannelTag.user_id.is_(None),
ChannelTag.source == "auto",
)
).scalars()
}
for tag_id, ct in existing.items():
if tag_id not in desired:
db.delete(ct)
for tag_id, conf in desired.items():
ct = existing.get(tag_id)
if ct is None:
db.add(
ChannelTag(
channel_id=channel.id,
tag_id=tag_id,
user_id=None,
source="auto",
confidence=conf,
)
)
else:
ct.confidence = conf
db.commit()
def run_autotag_all(db: Session, only_missing: bool = False) -> dict:
query = select(Channel)
if only_missing:
query = query.where(
~exists().where(
and_(
ChannelTag.channel_id == Channel.id,
ChannelTag.source == "auto",
ChannelTag.user_id.is_(None),
)
)
)
channels = db.execute(query).scalars().all()
tagged = 0
for channel in channels:
try:
apply_channel_autotags(db, channel)
tagged += 1
except Exception:
db.rollback()
removed = _cleanup_orphan_system_tags(db)
return {"channels_tagged": tagged, "orphan_tags_removed": removed}
def _cleanup_orphan_system_tags(db: Session) -> int:
"""Delete system tags that no longer link to any channel (e.g. after a mapping change)."""
orphans = (
db.execute(
select(Tag).where(
Tag.user_id.is_(None),
~exists().where(ChannelTag.tag_id == Tag.id),
)
)
.scalars()
.all()
)
for tag in orphans:
db.delete(tag)
db.commit()
return len(orphans)

1
backend/requirements.txt
View file

@ -9,5 +9,6 @@ authlib>=1.3,<2.0
httpx>=0.27,<1.0
feedparser>=6.0,<7.0
apscheduler>=3.10,<4.0
py3langid>=0.3,<1.0
itsdangerous>=2.1,<3.0
cryptography>=42,<46