fix: address reader-UI feedback (shorts, search, tags, login, UX)
- Shorts: confirm via youtube.com/shorts/<id> probe (SOCS cookie bypasses the consent redirect) instead of a 60s heuristic; concurrent probing, shorts_probed flag, scheduled refinement (migration 0005) - Search: match title + channel name only (descriptions caused noisy results) - Faceted tag filtering: AND across categories (language AND topic narrows), OR within a category; any/all toggle applies to topics - Language detection: majority vote over individual titles (fixes misdetections like multipoleguy -> English; drops bogus Polish/Romanian) - Login: drop forced consent so returning sign-in is quick (select_account) - Feed cards: clickable channel name (opens channel), persistent saved badge, undo toast on hide, Hidden view to restore; tag chips show counts in tooltip
This commit is contained in:
parent
e56502789f
commit
8c245e986f
17 changed files with 306 additions and 35 deletions
|
|
@ -28,13 +28,14 @@ oauth.register(
|
|||
|
||||
@router.get("/login")
|
||||
async def login(request: Request):
|
||||
# access_type=offline + prompt=consent must be on the authorization URL so Google
|
||||
# returns a refresh_token (required for background sync).
|
||||
# access_type=offline ensures a refresh_token on first authorization. We avoid
|
||||
# prompt=consent so returning users get a quick sign-in; the stored refresh token
|
||||
# is kept when Google doesn't re-issue one (see the callback).
|
||||
return await oauth.google.authorize_redirect(
|
||||
request,
|
||||
settings.oauth_redirect_url,
|
||||
access_type="offline",
|
||||
prompt="consent",
|
||||
prompt="select_account",
|
||||
include_granted_scopes="true",
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -34,8 +34,11 @@ class Settings(BaseSettings):
|
|||
backfill_recent_max_videos: int = 100
|
||||
backfill_recent_max_days: int = 365
|
||||
|
||||
# Videos at or below this duration (seconds) are treated as Shorts.
|
||||
shorts_max_seconds: int = 60
|
||||
# Shorts are confirmed by probing youtube.com/shorts/<id>. Only videos at or below
|
||||
# this duration (and not livestreams) are probed; longer videos are never Shorts.
|
||||
shorts_probe_max_seconds: int = 180
|
||||
shorts_probe_batch: int = 150
|
||||
shorts_probe_interval_minutes: int = 2
|
||||
# videos.list accepts up to 50 ids per call.
|
||||
enrich_batch_size: int = 50
|
||||
|
||||
|
|
|
|||
|
|
@ -146,6 +146,10 @@ class Video(Base):
|
|||
is_short: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
# Whether the youtube.com/shorts/<id> probe has run for this video.
|
||||
shorts_probed: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
# none | live | upcoming | premiere | was_live
|
||||
live_status: Mapped[str] = mapped_column(
|
||||
String(16), default="none", server_default="none", index=True
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from sqlalchemy.orm import Session, aliased
|
|||
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import Channel, ChannelTag, User, Video, VideoState
|
||||
from app.models import Channel, ChannelTag, Tag, User, Video, VideoState
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["feed"])
|
||||
|
||||
|
|
@ -14,6 +14,12 @@ VALID_STATES = {"new", "watched", "saved", "hidden"}
|
|||
HIDDEN_LIVE = ("live", "upcoming")
|
||||
|
||||
|
||||
def _channel_url(channel_id: str, handle: str | None) -> str:
|
||||
if handle and handle.startswith("@"):
|
||||
return f"https://www.youtube.com/{handle}"
|
||||
return f"https://www.youtube.com/channel/{channel_id}"
|
||||
|
||||
|
||||
def _serialize(row) -> dict:
|
||||
return {
|
||||
"id": row.id,
|
||||
|
|
@ -21,6 +27,7 @@ def _serialize(row) -> dict:
|
|||
"channel_id": row.channel_id,
|
||||
"channel_title": row.channel_title,
|
||||
"channel_thumbnail": row.channel_thumbnail,
|
||||
"channel_url": _channel_url(row.channel_id, row.channel_handle),
|
||||
"published_at": row.published_at.isoformat() if row.published_at else None,
|
||||
"thumbnail_url": row.thumbnail_url,
|
||||
"duration_seconds": row.duration_seconds,
|
||||
|
|
@ -61,6 +68,7 @@ def get_feed(
|
|||
Video.channel_id,
|
||||
Channel.title.label("channel_title"),
|
||||
Channel.thumbnail_url.label("channel_thumbnail"),
|
||||
Channel.handle.label("channel_handle"),
|
||||
Video.published_at,
|
||||
Video.thumbnail_url,
|
||||
Video.duration_seconds,
|
||||
|
|
@ -91,23 +99,35 @@ def get_feed(
|
|||
Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc)
|
||||
)
|
||||
if q:
|
||||
# Title (and channel name) only — searching descriptions produced noisy matches.
|
||||
like = f"%{q}%"
|
||||
query = query.where(or_(Video.title.ilike(like), Video.description.ilike(like)))
|
||||
query = query.where(or_(Video.title.ilike(like), Channel.title.ilike(like)))
|
||||
|
||||
if tags:
|
||||
# Group selected tags by category: AND across categories (e.g. language AND
|
||||
# topic narrows results), OR within a category. The any/all toggle controls
|
||||
# whether multiple topic tags are OR'd (any) or AND'd (all).
|
||||
cat_rows = db.execute(
|
||||
select(Tag.id, Tag.category).where(Tag.id.in_(tags))
|
||||
).all()
|
||||
by_category: dict[str, list[int]] = {}
|
||||
for tag_id, category in cat_rows:
|
||||
by_category.setdefault(category, []).append(tag_id)
|
||||
|
||||
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||
if tag_mode == "and":
|
||||
sub = (
|
||||
select(ChannelTag.channel_id)
|
||||
.where(ChannelTag.tag_id.in_(tags), visible)
|
||||
.group_by(ChannelTag.channel_id)
|
||||
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(tags)))
|
||||
)
|
||||
else:
|
||||
sub = select(ChannelTag.channel_id).where(
|
||||
ChannelTag.tag_id.in_(tags), visible
|
||||
)
|
||||
query = query.where(Video.channel_id.in_(sub))
|
||||
for category, ids in by_category.items():
|
||||
if category == "topic" and tag_mode == "and" and len(ids) > 1:
|
||||
sub = (
|
||||
select(ChannelTag.channel_id)
|
||||
.where(ChannelTag.tag_id.in_(ids), visible)
|
||||
.group_by(ChannelTag.channel_id)
|
||||
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids)))
|
||||
)
|
||||
else:
|
||||
sub = select(ChannelTag.channel_id).where(
|
||||
ChannelTag.tag_id.in_(ids), visible
|
||||
)
|
||||
query = query.where(Video.channel_id.in_(sub))
|
||||
|
||||
# Watch-state visibility.
|
||||
if show == "unwatched":
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from app.sync.runner import (
|
|||
run_enrich,
|
||||
run_recent_backfill,
|
||||
run_rss_poll,
|
||||
run_shorts,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("subfeed.scheduler")
|
||||
|
|
@ -53,6 +54,10 @@ def _autotag_job() -> None:
|
|||
_job("autotag", lambda db: run_autotag_all(db, only_missing=True))
|
||||
|
||||
|
||||
def _shorts_job() -> None:
|
||||
_job("shorts", run_shorts)
|
||||
|
||||
|
||||
def start_scheduler() -> None:
|
||||
global _scheduler
|
||||
if not settings.scheduler_enabled or _scheduler is not None:
|
||||
|
|
@ -76,6 +81,12 @@ def start_scheduler() -> None:
|
|||
minutes=settings.autotag_interval_minutes,
|
||||
id="autotag",
|
||||
)
|
||||
scheduler.add_job(
|
||||
_shorts_job,
|
||||
"interval",
|
||||
minutes=settings.shorts_probe_interval_minutes,
|
||||
id="shorts",
|
||||
)
|
||||
scheduler.start()
|
||||
_scheduler = scheduler
|
||||
logger.info("scheduler started")
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ detection over a sample of recent video titles. Topics are mapped from YouTube's
|
|||
topicDetails categories and the channel's dominant video category. System tags are
|
||||
regenerated freely; user tags are never touched here.
|
||||
"""
|
||||
from collections import Counter
|
||||
|
||||
from sqlalchemy import and_, exists, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
|
@ -156,11 +158,20 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None,
|
|||
.scalars()
|
||||
.all()
|
||||
)
|
||||
text = " . ".join(t for t in titles if t).strip()
|
||||
if len(text) < 15:
|
||||
# Majority vote over individual titles is more robust than one concatenated blob
|
||||
# (short/technical titles otherwise skew the detector).
|
||||
votes: Counter[str] = Counter()
|
||||
for title in titles:
|
||||
cleaned = (title or "").strip()
|
||||
if len(cleaned) < 8:
|
||||
continue
|
||||
lang, _conf = _classify(cleaned)
|
||||
votes[lang] += 1
|
||||
if not votes:
|
||||
return None, 0.0
|
||||
lang, confidence = _classify(text)
|
||||
return lang, float(confidence)
|
||||
lang, count = votes.most_common(1)[0]
|
||||
total = sum(votes.values())
|
||||
return lang, count / total
|
||||
|
||||
|
||||
def compute_channel_topics(db: Session, channel: Channel) -> set[str]:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from app.sync.videos import (
|
|||
backfill_channel_recent,
|
||||
enrich_pending,
|
||||
poll_rss_channel,
|
||||
run_shorts_classification,
|
||||
)
|
||||
from app.youtube.client import YouTubeClient
|
||||
|
||||
|
|
@ -60,6 +61,10 @@ def run_enrich(db: Session, max_batches: int = 200, floor: int = 200) -> int:
|
|||
return total
|
||||
|
||||
|
||||
def run_shorts(db: Session) -> dict:
|
||||
return run_shorts_classification(db)
|
||||
|
||||
|
||||
def run_recent_backfill(
|
||||
db: Session, channels: list[Channel] | None = None, max_channels: int | None = None
|
||||
) -> dict:
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@
|
|||
uploads playlist, and enrichment via videos.list (duration, stats, category, Shorts
|
||||
and livestream classification)."""
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
|
@ -12,6 +13,7 @@ from app.config import settings
|
|||
from app.models import Channel, Video
|
||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||
from app.youtube.rss import fetch_channel_feed
|
||||
from app.youtube.shorts import make_client, probe_is_short
|
||||
|
||||
_DURATION_RE = re.compile(
|
||||
r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE
|
||||
|
|
@ -195,11 +197,7 @@ def apply_video_details(video: Video, item: dict) -> None:
|
|||
"defaultAudioLanguage"
|
||||
)
|
||||
video.live_status = _live_status(snippet, live)
|
||||
video.is_short = bool(
|
||||
video.live_status == "none"
|
||||
and video.duration_seconds
|
||||
and 0 < video.duration_seconds <= settings.shorts_max_seconds
|
||||
)
|
||||
# is_short is decided separately by the youtube.com/shorts probe (run_shorts_classification).
|
||||
|
||||
|
||||
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
|
||||
|
|
@ -225,3 +223,60 @@ def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) ->
|
|||
enriched += 1
|
||||
db.commit()
|
||||
return enriched
|
||||
|
||||
|
||||
def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
|
||||
"""Finalize Shorts classification: cheaply mark non-candidates, then probe the
|
||||
youtube.com/shorts URL for short-enough, non-live, enriched videos."""
|
||||
limit = limit or settings.shorts_probe_batch
|
||||
probe_max = settings.shorts_probe_max_seconds
|
||||
|
||||
# 1) Anything enriched that can't be a Short -> finalize without a probe.
|
||||
db.execute(
|
||||
update(Video)
|
||||
.where(
|
||||
Video.shorts_probed.is_(False),
|
||||
Video.enriched_at.is_not(None),
|
||||
or_(
|
||||
Video.duration_seconds.is_(None),
|
||||
Video.duration_seconds > probe_max,
|
||||
Video.live_status != "none",
|
||||
),
|
||||
)
|
||||
.values(is_short=False, shorts_probed=True)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
# 2) Probe the remaining candidates.
|
||||
candidates = (
|
||||
db.execute(
|
||||
select(Video).where(
|
||||
Video.shorts_probed.is_(False),
|
||||
Video.enriched_at.is_not(None),
|
||||
Video.duration_seconds <= probe_max,
|
||||
Video.live_status == "none",
|
||||
).limit(limit)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not candidates:
|
||||
return {"probed": 0, "shorts": 0}
|
||||
|
||||
probed = 0
|
||||
shorts = 0
|
||||
with make_client() as client:
|
||||
def work(video: Video):
|
||||
return video, probe_is_short(client, video.id)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
for video, result in pool.map(work, candidates):
|
||||
if result is None:
|
||||
continue # leave unprobed; retry on a later run
|
||||
video.is_short = result
|
||||
video.shorts_probed = True
|
||||
probed += 1
|
||||
if result:
|
||||
shorts += 1
|
||||
db.commit()
|
||||
return {"probed": probed, "shorts": shorts}
|
||||
|
|
|
|||
32
backend/app/youtube/shorts.py
Normal file
32
backend/app/youtube/shorts.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
"""Confirm whether a video is a Short by probing youtube.com/shorts/<id>.
|
||||
|
||||
A real Short returns HTTP 200 at that URL; a regular video redirects to /watch.
|
||||
This uses no API quota."""
|
||||
import httpx
|
||||
|
||||
SHORTS_URL = "https://www.youtube.com/shorts/{video_id}"
|
||||
# The SOCS cookie skips YouTube's cookie-consent interstitial, which otherwise
|
||||
# redirects every server-side request to consent.youtube.com.
|
||||
_HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
|
||||
"Cookie": "SOCS=CAI",
|
||||
}
|
||||
|
||||
|
||||
def probe_is_short(client: httpx.Client, video_id: str) -> bool | None:
|
||||
"""True if a Short, False if a regular video, None if undetermined (retry later)."""
|
||||
try:
|
||||
resp = client.get(
|
||||
SHORTS_URL.format(video_id=video_id), follow_redirects=False
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
return None
|
||||
if resp.status_code == 200:
|
||||
return True
|
||||
if resp.status_code in (301, 302, 303, 307, 308):
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
def make_client() -> httpx.Client:
|
||||
return httpx.Client(timeout=10.0, headers=_HEADERS)
|
||||
Loading…
Add table
Add a link
Reference in a new issue