fix: address reader-UI feedback (shorts, search, tags, login, UX)

- Shorts: confirm via youtube.com/shorts/<id> probe (SOCS cookie bypasses the
  consent redirect) instead of a 60s heuristic; concurrent probing, shorts_probed
  flag, scheduled refinement (migration 0005)
- Search: match title + channel name only (descriptions caused noisy results)
- Faceted tag filtering: AND across categories (language AND topic narrows),
  OR within a category; any/all toggle applies to topics
- Language detection: majority vote over individual titles (fixes misdetections
  like multipoleguy -> English; drops bogus Polish/Romanian)
- Login: drop forced consent so returning sign-in is quick (select_account)
- Feed cards: clickable channel name (opens channel), persistent saved badge,
  undo toast on hide, Hidden view to restore; tag chips show counts in tooltip
This commit is contained in:
npeter83 2026-06-11 03:07:49 +02:00
parent e56502789f
commit 8c245e986f
17 changed files with 306 additions and 35 deletions

View file

@ -0,0 +1,28 @@
"""add videos.shorts_probed
Revision ID: 0005_shorts_probed
Revises: 0004_feed_state_prefs
Create Date: 2026-06-11
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "0005_shorts_probed"
down_revision: Union[str, None] = "0004_feed_state_prefs"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"videos",
sa.Column("shorts_probed", sa.Boolean(), nullable=False, server_default="false"),
)
op.create_index("ix_videos_shorts_probed", "videos", ["shorts_probed"])
def downgrade() -> None:
op.drop_index("ix_videos_shorts_probed", table_name="videos")
op.drop_column("videos", "shorts_probed")

View file

@ -28,13 +28,14 @@ oauth.register(
@router.get("/login")
async def login(request: Request):
# access_type=offline + prompt=consent must be on the authorization URL so Google
# returns a refresh_token (required for background sync).
# access_type=offline ensures a refresh_token on first authorization. We avoid
# prompt=consent so returning users get a quick sign-in; the stored refresh token
# is kept when Google doesn't re-issue one (see the callback).
return await oauth.google.authorize_redirect(
request,
settings.oauth_redirect_url,
access_type="offline",
prompt="consent",
prompt="select_account",
include_granted_scopes="true",
)

View file

@ -34,8 +34,11 @@ class Settings(BaseSettings):
backfill_recent_max_videos: int = 100
backfill_recent_max_days: int = 365
# Videos at or below this duration (seconds) are treated as Shorts.
shorts_max_seconds: int = 60
# Shorts are confirmed by probing youtube.com/shorts/<id>. Only videos at or below
# this duration (and not livestreams) are probed; longer videos are never Shorts.
shorts_probe_max_seconds: int = 180
shorts_probe_batch: int = 150
shorts_probe_interval_minutes: int = 2
# videos.list accepts up to 50 ids per call.
enrich_batch_size: int = 50

View file

@ -146,6 +146,10 @@ class Video(Base):
is_short: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false", index=True
)
# Whether the youtube.com/shorts/<id> probe has run for this video.
shorts_probed: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false", index=True
)
# none | live | upcoming | premiere | was_live
live_status: Mapped[str] = mapped_column(
String(16), default="none", server_default="none", index=True

View file

@ -6,7 +6,7 @@ from sqlalchemy.orm import Session, aliased
from app.auth import current_user
from app.db import get_db
from app.models import Channel, ChannelTag, User, Video, VideoState
from app.models import Channel, ChannelTag, Tag, User, Video, VideoState
router = APIRouter(prefix="/api", tags=["feed"])
@ -14,6 +14,12 @@ VALID_STATES = {"new", "watched", "saved", "hidden"}
HIDDEN_LIVE = ("live", "upcoming")
def _channel_url(channel_id: str, handle: str | None) -> str:
if handle and handle.startswith("@"):
return f"https://www.youtube.com/{handle}"
return f"https://www.youtube.com/channel/{channel_id}"
def _serialize(row) -> dict:
return {
"id": row.id,
@ -21,6 +27,7 @@ def _serialize(row) -> dict:
"channel_id": row.channel_id,
"channel_title": row.channel_title,
"channel_thumbnail": row.channel_thumbnail,
"channel_url": _channel_url(row.channel_id, row.channel_handle),
"published_at": row.published_at.isoformat() if row.published_at else None,
"thumbnail_url": row.thumbnail_url,
"duration_seconds": row.duration_seconds,
@ -61,6 +68,7 @@ def get_feed(
Video.channel_id,
Channel.title.label("channel_title"),
Channel.thumbnail_url.label("channel_thumbnail"),
Channel.handle.label("channel_handle"),
Video.published_at,
Video.thumbnail_url,
Video.duration_seconds,
@ -91,23 +99,35 @@ def get_feed(
Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc)
)
if q:
# Title (and channel name) only — searching descriptions produced noisy matches.
like = f"%{q}%"
query = query.where(or_(Video.title.ilike(like), Video.description.ilike(like)))
query = query.where(or_(Video.title.ilike(like), Channel.title.ilike(like)))
if tags:
# Group selected tags by category: AND across categories (e.g. language AND
# topic narrows results), OR within a category. The any/all toggle controls
# whether multiple topic tags are OR'd (any) or AND'd (all).
cat_rows = db.execute(
select(Tag.id, Tag.category).where(Tag.id.in_(tags))
).all()
by_category: dict[str, list[int]] = {}
for tag_id, category in cat_rows:
by_category.setdefault(category, []).append(tag_id)
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
if tag_mode == "and":
sub = (
select(ChannelTag.channel_id)
.where(ChannelTag.tag_id.in_(tags), visible)
.group_by(ChannelTag.channel_id)
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(tags)))
)
else:
sub = select(ChannelTag.channel_id).where(
ChannelTag.tag_id.in_(tags), visible
)
query = query.where(Video.channel_id.in_(sub))
for category, ids in by_category.items():
if category == "topic" and tag_mode == "and" and len(ids) > 1:
sub = (
select(ChannelTag.channel_id)
.where(ChannelTag.tag_id.in_(ids), visible)
.group_by(ChannelTag.channel_id)
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids)))
)
else:
sub = select(ChannelTag.channel_id).where(
ChannelTag.tag_id.in_(ids), visible
)
query = query.where(Video.channel_id.in_(sub))
# Watch-state visibility.
if show == "unwatched":

View file

@ -12,6 +12,7 @@ from app.sync.runner import (
run_enrich,
run_recent_backfill,
run_rss_poll,
run_shorts,
)
logger = logging.getLogger("subfeed.scheduler")
@ -53,6 +54,10 @@ def _autotag_job() -> None:
_job("autotag", lambda db: run_autotag_all(db, only_missing=True))
def _shorts_job() -> None:
_job("shorts", run_shorts)
def start_scheduler() -> None:
global _scheduler
if not settings.scheduler_enabled or _scheduler is not None:
@ -76,6 +81,12 @@ def start_scheduler() -> None:
minutes=settings.autotag_interval_minutes,
id="autotag",
)
scheduler.add_job(
_shorts_job,
"interval",
minutes=settings.shorts_probe_interval_minutes,
id="shorts",
)
scheduler.start()
_scheduler = scheduler
logger.info("scheduler started")

View file

@ -5,6 +5,8 @@ detection over a sample of recent video titles. Topics are mapped from YouTube's
topicDetails categories and the channel's dominant video category. System tags are
regenerated freely; user tags are never touched here.
"""
from collections import Counter
from sqlalchemy import and_, exists, func, select
from sqlalchemy.orm import Session
@ -156,11 +158,20 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None,
.scalars()
.all()
)
text = " . ".join(t for t in titles if t).strip()
if len(text) < 15:
# Majority vote over individual titles is more robust than one concatenated blob
# (short/technical titles otherwise skew the detector).
votes: Counter[str] = Counter()
for title in titles:
cleaned = (title or "").strip()
if len(cleaned) < 8:
continue
lang, _conf = _classify(cleaned)
votes[lang] += 1
if not votes:
return None, 0.0
lang, confidence = _classify(text)
return lang, float(confidence)
lang, count = votes.most_common(1)[0]
total = sum(votes.values())
return lang, count / total
def compute_channel_topics(db: Session, channel: Channel) -> set[str]:

View file

@ -15,6 +15,7 @@ from app.sync.videos import (
backfill_channel_recent,
enrich_pending,
poll_rss_channel,
run_shorts_classification,
)
from app.youtube.client import YouTubeClient
@ -60,6 +61,10 @@ def run_enrich(db: Session, max_batches: int = 200, floor: int = 200) -> int:
return total
def run_shorts(db: Session) -> dict:
return run_shorts_classification(db)
def run_recent_backfill(
db: Session, channels: list[Channel] | None = None, max_channels: int | None = None
) -> dict:

View file

@ -2,9 +2,10 @@
uploads playlist, and enrichment via videos.list (duration, stats, category, Shorts
and livestream classification)."""
import re
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy import or_, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session
@ -12,6 +13,7 @@ from app.config import settings
from app.models import Channel, Video
from app.youtube.client import YouTubeClient, best_thumbnail
from app.youtube.rss import fetch_channel_feed
from app.youtube.shorts import make_client, probe_is_short
_DURATION_RE = re.compile(
r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE
@ -195,11 +197,7 @@ def apply_video_details(video: Video, item: dict) -> None:
"defaultAudioLanguage"
)
video.live_status = _live_status(snippet, live)
video.is_short = bool(
video.live_status == "none"
and video.duration_seconds
and 0 < video.duration_seconds <= settings.shorts_max_seconds
)
# is_short is decided separately by the youtube.com/shorts probe (run_shorts_classification).
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
@ -225,3 +223,60 @@ def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) ->
enriched += 1
db.commit()
return enriched
def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
"""Finalize Shorts classification: cheaply mark non-candidates, then probe the
youtube.com/shorts URL for short-enough, non-live, enriched videos."""
limit = limit or settings.shorts_probe_batch
probe_max = settings.shorts_probe_max_seconds
# 1) Anything enriched that can't be a Short -> finalize without a probe.
db.execute(
update(Video)
.where(
Video.shorts_probed.is_(False),
Video.enriched_at.is_not(None),
or_(
Video.duration_seconds.is_(None),
Video.duration_seconds > probe_max,
Video.live_status != "none",
),
)
.values(is_short=False, shorts_probed=True)
)
db.commit()
# 2) Probe the remaining candidates.
candidates = (
db.execute(
select(Video).where(
Video.shorts_probed.is_(False),
Video.enriched_at.is_not(None),
Video.duration_seconds <= probe_max,
Video.live_status == "none",
).limit(limit)
)
.scalars()
.all()
)
if not candidates:
return {"probed": 0, "shorts": 0}
probed = 0
shorts = 0
with make_client() as client:
def work(video: Video):
return video, probe_is_short(client, video.id)
with ThreadPoolExecutor(max_workers=16) as pool:
for video, result in pool.map(work, candidates):
if result is None:
continue # leave unprobed; retry on a later run
video.is_short = result
video.shorts_probed = True
probed += 1
if result:
shorts += 1
db.commit()
return {"probed": probed, "shorts": shorts}

View file

@ -0,0 +1,32 @@
"""Confirm whether a video is a Short by probing youtube.com/shorts/<id>.
A real Short returns HTTP 200 at that URL; a regular video redirects to /watch.
This uses no API quota."""
import httpx
SHORTS_URL = "https://www.youtube.com/shorts/{video_id}"
# The SOCS cookie skips YouTube's cookie-consent interstitial, which otherwise
# redirects every server-side request to consent.youtube.com.
_HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
"Cookie": "SOCS=CAI",
}
def probe_is_short(client: httpx.Client, video_id: str) -> bool | None:
"""True if a Short, False if a regular video, None if undetermined (retry later)."""
try:
resp = client.get(
SHORTS_URL.format(video_id=video_id), follow_redirects=False
)
except httpx.HTTPError:
return None
if resp.status_code == 200:
return True
if resp.status_code in (301, 302, 303, 307, 308):
return False
return None
def make_client() -> httpx.Client:
return httpx.Client(timeout=10.0, headers=_HEADERS)