fix: address reader-UI feedback (shorts, search, tags, login, UX)
- Shorts: confirm via youtube.com/shorts/<id> probe (SOCS cookie bypasses the consent redirect) instead of a 60s heuristic; concurrent probing, shorts_probed flag, scheduled refinement (migration 0005) - Search: match title + channel name only (descriptions caused noisy results) - Faceted tag filtering: AND across categories (language AND topic narrows), OR within a category; any/all toggle applies to topics - Language detection: majority vote over individual titles (fixes misdetections like multipoleguy -> English; drops bogus Polish/Romanian) - Login: drop forced consent so returning sign-in is quick (select_account) - Feed cards: clickable channel name (opens channel), persistent saved badge, undo toast on hide, Hidden view to restore; tag chips show counts in tooltip
This commit is contained in:
parent
e56502789f
commit
8c245e986f
17 changed files with 306 additions and 35 deletions
|
|
@ -5,6 +5,8 @@ detection over a sample of recent video titles. Topics are mapped from YouTube's
|
|||
topicDetails categories and the channel's dominant video category. System tags are
|
||||
regenerated freely; user tags are never touched here.
|
||||
"""
|
||||
from collections import Counter
|
||||
|
||||
from sqlalchemy import and_, exists, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
|
@ -156,11 +158,20 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None,
|
|||
.scalars()
|
||||
.all()
|
||||
)
|
||||
text = " . ".join(t for t in titles if t).strip()
|
||||
if len(text) < 15:
|
||||
# Majority vote over individual titles is more robust than one concatenated blob
|
||||
# (short/technical titles otherwise skew the detector).
|
||||
votes: Counter[str] = Counter()
|
||||
for title in titles:
|
||||
cleaned = (title or "").strip()
|
||||
if len(cleaned) < 8:
|
||||
continue
|
||||
lang, _conf = _classify(cleaned)
|
||||
votes[lang] += 1
|
||||
if not votes:
|
||||
return None, 0.0
|
||||
lang, confidence = _classify(text)
|
||||
return lang, float(confidence)
|
||||
lang, count = votes.most_common(1)[0]
|
||||
total = sum(votes.values())
|
||||
return lang, count / total
|
||||
|
||||
|
||||
def compute_channel_topics(db: Session, channel: Channel) -> set[str]:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from app.sync.videos import (
|
|||
backfill_channel_recent,
|
||||
enrich_pending,
|
||||
poll_rss_channel,
|
||||
run_shorts_classification,
|
||||
)
|
||||
from app.youtube.client import YouTubeClient
|
||||
|
||||
|
|
@ -60,6 +61,10 @@ def run_enrich(db: Session, max_batches: int = 200, floor: int = 200) -> int:
|
|||
return total
|
||||
|
||||
|
||||
def run_shorts(db: Session) -> dict:
|
||||
return run_shorts_classification(db)
|
||||
|
||||
|
||||
def run_recent_backfill(
|
||||
db: Session, channels: list[Channel] | None = None, max_channels: int | None = None
|
||||
) -> dict:
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@
|
|||
uploads playlist, and enrichment via videos.list (duration, stats, category, Shorts
|
||||
and livestream classification)."""
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
|
@ -12,6 +13,7 @@ from app.config import settings
|
|||
from app.models import Channel, Video
|
||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||
from app.youtube.rss import fetch_channel_feed
|
||||
from app.youtube.shorts import make_client, probe_is_short
|
||||
|
||||
_DURATION_RE = re.compile(
|
||||
r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE
|
||||
|
|
@ -195,11 +197,7 @@ def apply_video_details(video: Video, item: dict) -> None:
|
|||
"defaultAudioLanguage"
|
||||
)
|
||||
video.live_status = _live_status(snippet, live)
|
||||
video.is_short = bool(
|
||||
video.live_status == "none"
|
||||
and video.duration_seconds
|
||||
and 0 < video.duration_seconds <= settings.shorts_max_seconds
|
||||
)
|
||||
# is_short is decided separately by the youtube.com/shorts probe (run_shorts_classification).
|
||||
|
||||
|
||||
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
|
||||
|
|
@ -225,3 +223,60 @@ def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) ->
|
|||
enriched += 1
|
||||
db.commit()
|
||||
return enriched
|
||||
|
||||
|
||||
def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
|
||||
"""Finalize Shorts classification: cheaply mark non-candidates, then probe the
|
||||
youtube.com/shorts URL for short-enough, non-live, enriched videos."""
|
||||
limit = limit or settings.shorts_probe_batch
|
||||
probe_max = settings.shorts_probe_max_seconds
|
||||
|
||||
# 1) Anything enriched that can't be a Short -> finalize without a probe.
|
||||
db.execute(
|
||||
update(Video)
|
||||
.where(
|
||||
Video.shorts_probed.is_(False),
|
||||
Video.enriched_at.is_not(None),
|
||||
or_(
|
||||
Video.duration_seconds.is_(None),
|
||||
Video.duration_seconds > probe_max,
|
||||
Video.live_status != "none",
|
||||
),
|
||||
)
|
||||
.values(is_short=False, shorts_probed=True)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
# 2) Probe the remaining candidates.
|
||||
candidates = (
|
||||
db.execute(
|
||||
select(Video).where(
|
||||
Video.shorts_probed.is_(False),
|
||||
Video.enriched_at.is_not(None),
|
||||
Video.duration_seconds <= probe_max,
|
||||
Video.live_status == "none",
|
||||
).limit(limit)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not candidates:
|
||||
return {"probed": 0, "shorts": 0}
|
||||
|
||||
probed = 0
|
||||
shorts = 0
|
||||
with make_client() as client:
|
||||
def work(video: Video):
|
||||
return video, probe_is_short(client, video.id)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
for video, result in pool.map(work, candidates):
|
||||
if result is None:
|
||||
continue # leave unprobed; retry on a later run
|
||||
video.is_short = result
|
||||
video.shorts_probed = True
|
||||
probed += 1
|
||||
if result:
|
||||
shorts += 1
|
||||
db.commit()
|
||||
return {"probed": probed, "shorts": shorts}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue