peter/siftlode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(search): live YouTube search backend

Browse source 
Add a live YouTube search that materialises results into the shared catalog so
they render with the normal feed cards + in-app player and gain per-user state.

- YouTubeClient.search_videos(): search.list (100 units), embeddable-only, returns
  flat stubs + nextPageToken; surfaces liveBroadcastContent for live filtering.
- routes/search.py GET /api/search/youtube: require_human + per-user daily cap
  (search_daily_limit_per_user, default 70) + can_spend pre-check (429 on either);
  drops live/upcoming, upserts channel stubs (channels.list) + video stubs, enriches
  (videos.list), runs the youtube.com/shorts probe, then excludes Shorts/live and
  returns feed cards in relevance order with the YouTube pageToken as the cursor.
- Provenance: videos.via_search / channels.from_search (migration 0028) flag
  search-discovered rows; the feed hides them from the Library (scope=all) by default
  via exclude_search_discovered, leaving the Mine feed untouched.
- quota.actions_today() counts a user's per-action events today for the cap; only the
  search.list call is attributed VIDEOS_SEARCH so the counter is exactly 1 per search.
This commit is contained in:
npeter83 2026-06-29 02:01:31 +02:00
parent b1ed706cab
commit 9b1bdb6b42
9 changed files with 341 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

3
backend/app/config.py
View file

@ -71,6 +71,9 @@ class Settings(BaseSettings):
youtube_api_proxy: str = ""
# Daily quota budget (units). Default leaves headroom under the 10,000/day free limit.
quota_daily_budget: int = 9000
# Per-user cap on live YouTube searches per day. search.list costs 100 units each, so the
# shared budget only affords ~80-90/day total — this stops one user draining it. Admin-tunable.
search_daily_limit_per_user: int = 70
# Recent-first backfill: how far back to fetch on the first pass per channel.
backfill_recent_max_videos: int = 100
backfill_recent_max_days: int = 365

2
backend/app/main.py
View file

@ -39,6 +39,7 @@ from app.routes import (
playlists,
quota,
scheduler as scheduler_routes,
search as search_routes,
setup as setup_routes,
sync,
tags,
@ -113,6 +114,7 @@ app.include_router(auth.router)
app.include_router(sync.router)
app.include_router(tags.router)
app.include_router(feed.router)
app.include_router(search_routes.router)
app.include_router(me.router)
app.include_router(notifications.router)
app.include_router(messages.router)

12
backend/app/models.py
View file

@ -175,6 +175,12 @@ class Channel(Base, TimestampMixin):
backfill_done: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
# True when this channel only entered the catalog because a live YouTube search
# surfaced one of its videos (nobody subscribes to it / it has no organic upload sync).
# Lets the Library hide search-only "noise" by default.
from_search: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false", index=True
)
videos: Mapped[list["Video"]] = relationship(back_populates="channel")
subscriptions: Mapped[list["Subscription"]] = relationship(back_populates="channel")
@ -238,6 +244,12 @@ class Video(Base, TimestampMixin):
shorts_probed: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false", index=True
)
# True when this video first entered the catalog via a live YouTube search (not an
# organic channel/playlist sync). Used to hide search-discovered "noise" from the
# Library by default; left False by the organic ingest paths.
via_search: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false", index=True
)
# none | live | upcoming | premiere | was_live
live_status: Mapped[str] = mapped_column(
String(16), default="none", server_default="none", index=True

21
backend/app/quota.py
View file

@ -36,6 +36,7 @@ class QuotaAction:
VIDEOS_BACKFILL_FULL = "videos_backfill_full"
VIDEOS_ENRICH = "videos_enrich"
VIDEOS_LOOKUP = "videos_lookup"
VIDEOS_SEARCH = "videos_search"
CHANNELS_DISCOVER = "channels_discover"
CHANNELS_SUBSCRIBE = "channels_subscribe"
CHANNELS_UNSUBSCRIBE = "channels_unsubscribe"
@ -109,6 +110,26 @@ def can_spend(db: Session, units: int) -> bool:
return remaining_today(db) >= units
def actions_today(db: Session, user_id: int, action: str) -> int:
"""How many quota events of `action` this user has logged so far in the current Pacific
day — for per-user, per-action daily caps (e.g. the live-search limit). Counts events,
not units, so it only works for actions charged exactly once per user action."""
from sqlalchemy import func, select # local import: keep the module's import head lean
return (
db.scalar(
select(func.count())
.select_from(QuotaEvent)
.where(
QuotaEvent.user_id == user_id,
QuotaEvent.action == action,
QuotaEvent.created_at >= pacific_day_start_utc(),
)
)
or 0
)
def record_usage(db: Session, units: int) -> None:
"""Atomically add `units` to today's counter (upsert) and log an attribution event."""
if units <= 0:

9
backend/app/routes/feed.py
View file

@ -124,6 +124,7 @@ def _filtered_query(
include_live: bool,
show: str,
scope: str = "my",
exclude_search_discovered: bool = True,
exclude_tag_category: str | None = None,
) -> tuple[Select, object]:
"""Build the feed query (joins + all WHERE filters), shared by /feed and /feed/count.
@ -171,6 +172,12 @@ def _filtered_query(
# either way they shouldn't show in any feed view meanwhile.
query = query.where(Video.unavailable_since.is_(None))
# Hide videos that only entered the catalog via a live YouTube search (search-discovered
# "noise") from the Library by default. Only meaningful in "all" scope: "my" already
# restricts to subscribed channels, where a once-searched video is legitimately the user's.
if scope == "all" and exclude_search_discovered:
query = query.where(Video.via_search.is_(False))
if channel_id:
query = query.where(Video.channel_id == channel_id)
if min_duration is not None:
@ -271,6 +278,7 @@ def _feed_params(
include_live: bool = False,
show: str = "unwatched",
scope: str = "my",
exclude_search_discovered: bool = True,
) -> dict:
return {
"tags": tags,
@ -287,6 +295,7 @@ def _feed_params(
"include_live": include_live,
"show": show,
"scope": scope,
"exclude_search_discovered": exclude_search_discovered,
}

204
backend/app/routes/search.py Normal file
View file

@ -0,0 +1,204 @@
"""Live YouTube search.
Searches YouTube directly (search.list) and materialises the results into the shared catalog
so they render with the normal feed cards + in-app player and gain per-user state (watch /
save / resume). search.list costs **100 quota units per call**, so it's the most expensive
read by far: it's gated behind a per-user daily cap + a budget pre-check, and only real users
(never the shared demo account) may call it.
Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
dropped from the search response up front)."""
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import and_, select
from sqlalchemy.orm import Session, aliased
from app import quota, sysconfig
from app.auth import require_human
from app.db import get_db
from app.models import Channel, User, Video, VideoState
from app.routes.feed import _serialize, feed_columns
from app.sync.subscriptions import apply_channel_details
from app.sync.videos import _insert_stubs, apply_video_details, parse_dt
from app.youtube.client import YouTubeClient, YouTubeError
from app.youtube.shorts import make_client, probe_is_short
router = APIRouter(prefix="/api/search", tags=["search"])
# search.list costs this many quota units per page.
SEARCH_COST = 100
# live_status values we never surface from a live search (was_live VODs are real content).
_LIVE_HIDDEN = ("live", "upcoming")
def _classify_shorts(db: Session, videos: list[Video]) -> None:
"""Confirm/deny Shorts for the freshly-ingested videos so none slip into the feed.
Mirrors the scheduler's run_shorts_classification, but scoped to just these rows and run
synchronously (the search is an explicit, awaited action). Non-candidates (too long, no
duration, or live) are finalised without a probe; the rest hit youtube.com/shorts (no
quota). Undetermined probes are left for the scheduler to retry."""
probe_max = sysconfig.get_int(db, "shorts_probe_max_seconds")
to_probe: list[Video] = []
for v in videos:
if v.shorts_probed:
continue
if (
v.live_status != "none"
or v.duration_seconds is None
or v.duration_seconds > probe_max
):
v.is_short = False
v.shorts_probed = True
else:
to_probe.append(v)
if to_probe:
with make_client() as client:
def work(v: Video):
return v, probe_is_short(client, v.id)
with ThreadPoolExecutor(max_workers=16) as pool:
for v, result in pool.map(work, to_probe):
if result is None:
continue
v.is_short = result
v.shorts_probed = True
db.commit()
def _serialize_ids(db: Session, user: User, ids: list[str]) -> list[dict]:
"""Serialize the given video ids with the shared feed projection (joined channel + this
user's watch state + saved), preserving the input (search relevance) order."""
if not ids:
return []
state = aliased(VideoState)
rows = db.execute(
select(*feed_columns(user, state))
.join(Channel, Channel.id == Video.channel_id)
.outerjoin(state, and_(state.video_id == Video.id, state.user_id == user.id))
.where(Video.id.in_(ids))
).all()
by_id = {r.id: _serialize(r) for r in rows}
return [by_id[i] for i in ids if i in by_id]
@router.get("/youtube")
def search_youtube(
q: str | None = None,
cursor: str | None = None,
user: User = Depends(require_human),
db: Session = Depends(get_db),
) -> dict:
"""One page of live YouTube search results, materialised into the catalog. `cursor` is the
YouTube nextPageToken (each page is another 100-unit search)."""
term = (q or "").strip()
if not term:
raise HTTPException(status_code=400, detail="A search term is required.")
# Guard the shared quota BEFORE spending: per-user daily cap, then today's budget.
daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
if used >= daily_limit:
raise HTTPException(
status_code=429,
detail=f"You've reached today's YouTube search limit ({daily_limit}). Try again tomorrow.",
)
if not quota.can_spend(db, SEARCH_COST):
raise HTTPException(
status_code=429,
detail="The shared YouTube quota for today is used up. Try again tomorrow.",
)
with YouTubeClient(db, user) as yt:
try:
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
page = yt.search_videos(term, page_token=cursor)
except YouTubeError as exc:
raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}")
# Drop currently-live/upcoming results (search can't filter them and we never ingest
# them this way) and anything missing a channel id.
candidates = [
it
for it in page["items"]
if it["live_broadcast"] == "none" and it["channel_id"]
]
ids = [it["id"] for it in candidates]
if not ids:
return {
"items": [],
"next_cursor": page["next_page_token"],
"has_more": bool(page["next_page_token"]),
}
# 1) Channels are the FK target — create stubs for unknown ones (flagged from_search),
# then enrich the new ones (channels.list, 1 unit). Stubs survive an enrich failure.
channel_ids = list({it["channel_id"] for it in candidates})
existing = set(
db.scalars(select(Channel.id).where(Channel.id.in_(channel_ids))).all()
)
new_channel_ids = [c for c in channel_ids if c not in existing]
if new_channel_ids:
titles = {it["channel_id"]: it["channel_title"] for it in candidates}
for cid in new_channel_ids:
db.add(Channel(id=cid, title=titles.get(cid), from_search=True))
db.commit()
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER):
details = yt.get_channels(new_channel_ids)
apply_channel_details(db, details)
db.commit()
except YouTubeError:
db.rollback() # details can be backfilled later; the stubs remain
# 2) Video stubs, flagged via_search. Existing rows keep their flags (ON CONFLICT DO
# NOTHING), so an organically-synced video stays organic.
_insert_stubs(
db,
[
{
"id": it["id"],
"channel_id": it["channel_id"],
"title": it["title"],
"published_at": parse_dt(it["published_at"]),
"thumbnail_url": it["thumbnail_url"],
"via_search": True,
}
for it in candidates
],
)
# 3) Enrich (videos.list, 1 unit/50) so cards have duration/stats and we can judge live.
videos = db.scalars(select(Video).where(Video.id.in_(ids))).all()
try:
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_ENRICH):
detail_by_id = {it["id"]: it for it in yt.get_videos(ids)}
except YouTubeError:
detail_by_id = {}
now = datetime.now(timezone.utc)
for v in videos:
item = detail_by_id.get(v.id)
if item is None:
continue
apply_video_details(v, item)
v.enriched_at = now
db.commit()
# 4) Confirm/deny Shorts (no quota) so none enter via search.
_classify_shorts(db, videos)
# 5) Exclude Shorts and still-live results; keep the search's relevance order.
keep = {
v.id for v in videos if not v.is_short and v.live_status not in _LIVE_HIDDEN
}
ordered = [vid for vid in ids if vid in keep]
return {
"items": _serialize_ids(db, user, ordered),
"next_cursor": page["next_page_token"],
"has_more": bool(page["next_page_token"]),
}

1
backend/app/sysconfig.py
View file

@ -39,6 +39,7 @@ SPECS: tuple[ConfigSpec, ...] = (
# --- Quota (shared YouTube Data API daily budget) ---
ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000),
ConfigSpec("search_daily_limit_per_user", "int", "quota", "search_daily_limit_per_user", min=0, max=100_000),
# --- Backfill (recent-first first pass per channel) ---
ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000),
ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500),

43
backend/app/youtube/client.py
View file

@ -351,6 +351,49 @@ class YouTubeClient:
"""Delete a whole playlist on YouTube. 50 units."""
self._write("DELETE", "playlists", params={"id": playlist_id})
def search_videos(
self, q: str, page_token: str | None = None, order: str = "relevance"
) -> dict:
"""Live YouTube search (search.list) for embeddable videos. **100 quota units per
call** — by far the most expensive read, so callers must gate it (per-user cap +
budget pre-check). Public read: prefers the API key over OAuth.
Returns one page (up to 50) as ``{"items": [...], "next_page_token": str|None}``
where each item is a flat dict ready for stub insertion. `liveBroadcastContent` is
surfaced so the caller can drop currently-live/upcoming results (search can't filter
those out for us). Snippet carries no duration/stats — the caller enriches via
videos.list afterwards."""
params = {
"part": "snippet",
"q": q,
"type": "video",
"maxResults": 50,
"order": order,
"videoEmbeddable": "true",
"safeSearch": "none",
}
if page_token:
params["pageToken"] = page_token
data = self._get("search", params, cost=100)
items: list[dict] = []
for it in data.get("items", []):
vid = it.get("id", {}).get("videoId")
if not vid:
continue
sn = it.get("snippet", {})
items.append(
{
"id": vid,
"channel_id": sn.get("channelId"),
"channel_title": sn.get("channelTitle"),
"title": sn.get("title"),
"published_at": sn.get("publishedAt"),
"thumbnail_url": best_thumbnail(sn.get("thumbnails")),
"live_broadcast": sn.get("liveBroadcastContent") or "none",
}
)
return {"items": items, "next_page_token": data.get("nextPageToken")}
def get_videos(self, video_ids: list[str]) -> list[dict]:
items: list[dict] = []
for batch in _chunks(video_ids, 50):