diff --git a/backend/alembic/versions/0005_shorts_probed.py b/backend/alembic/versions/0005_shorts_probed.py new file mode 100644 index 0000000..a336cd7 --- /dev/null +++ b/backend/alembic/versions/0005_shorts_probed.py @@ -0,0 +1,28 @@ +"""add videos.shorts_probed + +Revision ID: 0005_shorts_probed +Revises: 0004_feed_state_prefs +Create Date: 2026-06-11 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "0005_shorts_probed" +down_revision: Union[str, None] = "0004_feed_state_prefs" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "videos", + sa.Column("shorts_probed", sa.Boolean(), nullable=False, server_default="false"), + ) + op.create_index("ix_videos_shorts_probed", "videos", ["shorts_probed"]) + + +def downgrade() -> None: + op.drop_index("ix_videos_shorts_probed", table_name="videos") + op.drop_column("videos", "shorts_probed") diff --git a/backend/app/auth.py b/backend/app/auth.py index bc56522..9eab05c 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -28,13 +28,14 @@ oauth.register( @router.get("/login") async def login(request: Request): - # access_type=offline + prompt=consent must be on the authorization URL so Google - # returns a refresh_token (required for background sync). + # access_type=offline ensures a refresh_token on first authorization. We avoid + # prompt=consent so returning users get a quick sign-in; the stored refresh token + # is kept when Google doesn't re-issue one (see the callback). return await oauth.google.authorize_redirect( request, settings.oauth_redirect_url, access_type="offline", - prompt="consent", + prompt="select_account", include_granted_scopes="true", ) diff --git a/backend/app/config.py b/backend/app/config.py index c149a52..a0bc1cf 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -34,8 +34,11 @@ class Settings(BaseSettings): backfill_recent_max_videos: int = 100 backfill_recent_max_days: int = 365 - # Videos at or below this duration (seconds) are treated as Shorts. - shorts_max_seconds: int = 60 + # Shorts are confirmed by probing youtube.com/shorts/. Only videos at or below + # this duration (and not livestreams) are probed; longer videos are never Shorts. + shorts_probe_max_seconds: int = 180 + shorts_probe_batch: int = 150 + shorts_probe_interval_minutes: int = 2 # videos.list accepts up to 50 ids per call. enrich_batch_size: int = 50 diff --git a/backend/app/models.py b/backend/app/models.py index 70f93af..84f3557 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -146,6 +146,10 @@ class Video(Base): is_short: Mapped[bool] = mapped_column( Boolean, default=False, server_default="false", index=True ) + # Whether the youtube.com/shorts/ probe has run for this video. + shorts_probed: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false", index=True + ) # none | live | upcoming | premiere | was_live live_status: Mapped[str] = mapped_column( String(16), default="none", server_default="none", index=True diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 6ff013f..761e2b8 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -6,7 +6,7 @@ from sqlalchemy.orm import Session, aliased from app.auth import current_user from app.db import get_db -from app.models import Channel, ChannelTag, User, Video, VideoState +from app.models import Channel, ChannelTag, Tag, User, Video, VideoState router = APIRouter(prefix="/api", tags=["feed"]) @@ -14,6 +14,12 @@ VALID_STATES = {"new", "watched", "saved", "hidden"} HIDDEN_LIVE = ("live", "upcoming") +def _channel_url(channel_id: str, handle: str | None) -> str: + if handle and handle.startswith("@"): + return f"https://www.youtube.com/{handle}" + return f"https://www.youtube.com/channel/{channel_id}" + + def _serialize(row) -> dict: return { "id": row.id, @@ -21,6 +27,7 @@ def _serialize(row) -> dict: "channel_id": row.channel_id, "channel_title": row.channel_title, "channel_thumbnail": row.channel_thumbnail, + "channel_url": _channel_url(row.channel_id, row.channel_handle), "published_at": row.published_at.isoformat() if row.published_at else None, "thumbnail_url": row.thumbnail_url, "duration_seconds": row.duration_seconds, @@ -61,6 +68,7 @@ def get_feed( Video.channel_id, Channel.title.label("channel_title"), Channel.thumbnail_url.label("channel_thumbnail"), + Channel.handle.label("channel_handle"), Video.published_at, Video.thumbnail_url, Video.duration_seconds, @@ -91,23 +99,35 @@ def get_feed( Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc) ) if q: + # Title (and channel name) only — searching descriptions produced noisy matches. like = f"%{q}%" - query = query.where(or_(Video.title.ilike(like), Video.description.ilike(like))) + query = query.where(or_(Video.title.ilike(like), Channel.title.ilike(like))) if tags: + # Group selected tags by category: AND across categories (e.g. language AND + # topic narrows results), OR within a category. The any/all toggle controls + # whether multiple topic tags are OR'd (any) or AND'd (all). + cat_rows = db.execute( + select(Tag.id, Tag.category).where(Tag.id.in_(tags)) + ).all() + by_category: dict[str, list[int]] = {} + for tag_id, category in cat_rows: + by_category.setdefault(category, []).append(tag_id) + visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id) - if tag_mode == "and": - sub = ( - select(ChannelTag.channel_id) - .where(ChannelTag.tag_id.in_(tags), visible) - .group_by(ChannelTag.channel_id) - .having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(tags))) - ) - else: - sub = select(ChannelTag.channel_id).where( - ChannelTag.tag_id.in_(tags), visible - ) - query = query.where(Video.channel_id.in_(sub)) + for category, ids in by_category.items(): + if category == "topic" and tag_mode == "and" and len(ids) > 1: + sub = ( + select(ChannelTag.channel_id) + .where(ChannelTag.tag_id.in_(ids), visible) + .group_by(ChannelTag.channel_id) + .having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids))) + ) + else: + sub = select(ChannelTag.channel_id).where( + ChannelTag.tag_id.in_(ids), visible + ) + query = query.where(Video.channel_id.in_(sub)) # Watch-state visibility. if show == "unwatched": diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index d95e589..cc3ef1a 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -12,6 +12,7 @@ from app.sync.runner import ( run_enrich, run_recent_backfill, run_rss_poll, + run_shorts, ) logger = logging.getLogger("subfeed.scheduler") @@ -53,6 +54,10 @@ def _autotag_job() -> None: _job("autotag", lambda db: run_autotag_all(db, only_missing=True)) +def _shorts_job() -> None: + _job("shorts", run_shorts) + + def start_scheduler() -> None: global _scheduler if not settings.scheduler_enabled or _scheduler is not None: @@ -76,6 +81,12 @@ def start_scheduler() -> None: minutes=settings.autotag_interval_minutes, id="autotag", ) + scheduler.add_job( + _shorts_job, + "interval", + minutes=settings.shorts_probe_interval_minutes, + id="shorts", + ) scheduler.start() _scheduler = scheduler logger.info("scheduler started") diff --git a/backend/app/sync/autotag.py b/backend/app/sync/autotag.py index 121efe3..c1a44f7 100644 --- a/backend/app/sync/autotag.py +++ b/backend/app/sync/autotag.py @@ -5,6 +5,8 @@ detection over a sample of recent video titles. Topics are mapped from YouTube's topicDetails categories and the channel's dominant video category. System tags are regenerated freely; user tags are never touched here. """ +from collections import Counter + from sqlalchemy import and_, exists, func, select from sqlalchemy.orm import Session @@ -156,11 +158,20 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None, .scalars() .all() ) - text = " . ".join(t for t in titles if t).strip() - if len(text) < 15: + # Majority vote over individual titles is more robust than one concatenated blob + # (short/technical titles otherwise skew the detector). + votes: Counter[str] = Counter() + for title in titles: + cleaned = (title or "").strip() + if len(cleaned) < 8: + continue + lang, _conf = _classify(cleaned) + votes[lang] += 1 + if not votes: return None, 0.0 - lang, confidence = _classify(text) - return lang, float(confidence) + lang, count = votes.most_common(1)[0] + total = sum(votes.values()) + return lang, count / total def compute_channel_topics(db: Session, channel: Channel) -> set[str]: diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index 9f14932..03bf8d3 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -15,6 +15,7 @@ from app.sync.videos import ( backfill_channel_recent, enrich_pending, poll_rss_channel, + run_shorts_classification, ) from app.youtube.client import YouTubeClient @@ -60,6 +61,10 @@ def run_enrich(db: Session, max_batches: int = 200, floor: int = 200) -> int: return total +def run_shorts(db: Session) -> dict: + return run_shorts_classification(db) + + def run_recent_backfill( db: Session, channels: list[Channel] | None = None, max_channels: int | None = None ) -> dict: diff --git a/backend/app/sync/videos.py b/backend/app/sync/videos.py index 5818e5c..2ac9c56 100644 --- a/backend/app/sync/videos.py +++ b/backend/app/sync/videos.py @@ -2,9 +2,10 @@ uploads playlist, and enrichment via videos.list (duration, stats, category, Shorts and livestream classification).""" import re +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta, timezone -from sqlalchemy import select +from sqlalchemy import or_, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session @@ -12,6 +13,7 @@ from app.config import settings from app.models import Channel, Video from app.youtube.client import YouTubeClient, best_thumbnail from app.youtube.rss import fetch_channel_feed +from app.youtube.shorts import make_client, probe_is_short _DURATION_RE = re.compile( r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE @@ -195,11 +197,7 @@ def apply_video_details(video: Video, item: dict) -> None: "defaultAudioLanguage" ) video.live_status = _live_status(snippet, live) - video.is_short = bool( - video.live_status == "none" - and video.duration_seconds - and 0 < video.duration_seconds <= settings.shorts_max_seconds - ) + # is_short is decided separately by the youtube.com/shorts probe (run_shorts_classification). def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int: @@ -225,3 +223,60 @@ def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> enriched += 1 db.commit() return enriched + + +def run_shorts_classification(db: Session, limit: int | None = None) -> dict: + """Finalize Shorts classification: cheaply mark non-candidates, then probe the + youtube.com/shorts URL for short-enough, non-live, enriched videos.""" + limit = limit or settings.shorts_probe_batch + probe_max = settings.shorts_probe_max_seconds + + # 1) Anything enriched that can't be a Short -> finalize without a probe. + db.execute( + update(Video) + .where( + Video.shorts_probed.is_(False), + Video.enriched_at.is_not(None), + or_( + Video.duration_seconds.is_(None), + Video.duration_seconds > probe_max, + Video.live_status != "none", + ), + ) + .values(is_short=False, shorts_probed=True) + ) + db.commit() + + # 2) Probe the remaining candidates. + candidates = ( + db.execute( + select(Video).where( + Video.shorts_probed.is_(False), + Video.enriched_at.is_not(None), + Video.duration_seconds <= probe_max, + Video.live_status == "none", + ).limit(limit) + ) + .scalars() + .all() + ) + if not candidates: + return {"probed": 0, "shorts": 0} + + probed = 0 + shorts = 0 + with make_client() as client: + def work(video: Video): + return video, probe_is_short(client, video.id) + + with ThreadPoolExecutor(max_workers=16) as pool: + for video, result in pool.map(work, candidates): + if result is None: + continue # leave unprobed; retry on a later run + video.is_short = result + video.shorts_probed = True + probed += 1 + if result: + shorts += 1 + db.commit() + return {"probed": probed, "shorts": shorts} diff --git a/backend/app/youtube/shorts.py b/backend/app/youtube/shorts.py new file mode 100644 index 0000000..046380e --- /dev/null +++ b/backend/app/youtube/shorts.py @@ -0,0 +1,32 @@ +"""Confirm whether a video is a Short by probing youtube.com/shorts/. + +A real Short returns HTTP 200 at that URL; a regular video redirects to /watch. +This uses no API quota.""" +import httpx + +SHORTS_URL = "https://www.youtube.com/shorts/{video_id}" +# The SOCS cookie skips YouTube's cookie-consent interstitial, which otherwise +# redirects every server-side request to consent.youtube.com. +_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + "Cookie": "SOCS=CAI", +} + + +def probe_is_short(client: httpx.Client, video_id: str) -> bool | None: + """True if a Short, False if a regular video, None if undetermined (retry later).""" + try: + resp = client.get( + SHORTS_URL.format(video_id=video_id), follow_redirects=False + ) + except httpx.HTTPError: + return None + if resp.status_code == 200: + return True + if resp.status_code in (301, 302, 303, 307, 308): + return False + return None + + +def make_client() -> httpx.Client: + return httpx.Client(timeout=10.0, headers=_HEADERS) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 8153139..0d8fef7 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -12,6 +12,7 @@ import Login from "./components/Login"; import Header from "./components/Header"; import Sidebar from "./components/Sidebar"; import Feed from "./components/Feed"; +import Toaster from "./components/Toaster"; const DEFAULT_FILTERS: FeedFilters = { tags: [], @@ -83,6 +84,7 @@ export default function App() { + ); } diff --git a/frontend/src/components/Feed.tsx b/frontend/src/components/Feed.tsx index 8f42b0c..ef6ae68 100644 --- a/frontend/src/components/Feed.tsx +++ b/frontend/src/components/Feed.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState } from "react"; import { useInfiniteQuery } from "@tanstack/react-query"; import { api, type FeedFilters, type Video } from "../lib/api"; +import { toast } from "../lib/toast"; import VideoCard from "./VideoCard"; const PAGE = 60; @@ -43,12 +44,19 @@ export default function Feed({ function onState(id: string, status: string) { setOverrides((o) => ({ ...o, [id]: status })); api.setState(id, status).catch(() => {}); + if (status === "hidden") { + toast("Video hidden", { label: "Undo", onClick: () => onState(id, "new") }); + } } const items: Video[] = (query.data?.pages ?? []) .flatMap((p) => p.items) .map((v) => (overrides[v.id] ? { ...v, status: overrides[v.id] } : v)) - .filter((v) => overrides[v.id] !== "hidden"); + .filter((v) => { + const ov = overrides[v.id]; + // In the Hidden view, drop just-unhidden items; elsewhere drop just-hidden ones. + return filters.show === "hidden" ? ov !== "new" : ov !== "hidden"; + }); if (query.isLoading) return
Loading feed…
; if (query.isError) return
Couldn't load the feed.
; diff --git a/frontend/src/components/Sidebar.tsx b/frontend/src/components/Sidebar.tsx index 9e73399..4355995 100644 --- a/frontend/src/components/Sidebar.tsx +++ b/frontend/src/components/Sidebar.tsx @@ -15,6 +15,7 @@ const SHOWS = [ { id: "all", label: "All" }, { id: "watched", label: "Watched" }, { id: "saved", label: "Saved" }, + { id: "hidden", label: "Hidden" }, ]; function TagChip({ @@ -29,6 +30,7 @@ function TagChip({ return ( ); } diff --git a/frontend/src/components/Toaster.tsx b/frontend/src/components/Toaster.tsx new file mode 100644 index 0000000..58a653a --- /dev/null +++ b/frontend/src/components/Toaster.tsx @@ -0,0 +1,29 @@ +import { useSyncExternalStore } from "react"; +import { dismiss, getToasts, subscribe } from "../lib/toast"; + +export default function Toaster() { + const toasts = useSyncExternalStore(subscribe, getToasts, getToasts); + return ( +
+ {toasts.map((t) => ( +
+ {t.message} + {t.action && ( + + )} +
+ ))} +
+ ); +} diff --git a/frontend/src/components/VideoCard.tsx b/frontend/src/components/VideoCard.tsx index 5c8da76..71169e3 100644 --- a/frontend/src/components/VideoCard.tsx +++ b/frontend/src/components/VideoCard.tsx @@ -80,6 +80,11 @@ function Thumb({ video, className }: { video: Video; className?: string }) { stream )} + {video.status === "saved" && ( + + + + )} ); } @@ -122,7 +127,15 @@ export default function VideoCard({ @@ -144,7 +157,15 @@ export default function VideoCard({ )} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 7f391d5..f6628c2 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -22,6 +22,7 @@ export interface Video { channel_id: string; channel_title: string | null; channel_thumbnail: string | null; + channel_url: string; published_at: string | null; thumbnail_url: string | null; duration_seconds: number | null; diff --git a/frontend/src/lib/toast.ts b/frontend/src/lib/toast.ts new file mode 100644 index 0000000..ce5fe97 --- /dev/null +++ b/frontend/src/lib/toast.ts @@ -0,0 +1,41 @@ +export interface ToastAction { + label: string; + onClick: () => void; +} +export interface ToastItem { + id: number; + message: string; + action?: ToastAction; +} + +let items: ToastItem[] = []; +let listeners: Array<() => void> = []; +let counter = 1; + +function emit() { + listeners.forEach((l) => l()); +} + +export function toast(message: string, action?: ToastAction): number { + const id = counter++; + items = [...items, { id, message, action }]; + emit(); + setTimeout(() => dismiss(id), 6000); + return id; +} + +export function dismiss(id: number) { + items = items.filter((i) => i.id !== id); + emit(); +} + +export function getToasts(): ToastItem[] { + return items; +} + +export function subscribe(listener: () => void): () => void { + listeners.push(listener); + return () => { + listeners = listeners.filter((l) => l !== listener); + }; +}