feat(search): zero-quota InnerTube scrape source + search_source toggle

Live YouTube search can now use YouTube's internal InnerTube endpoint instead
of search.list, materialising results through the same enrich/provenance path
at zero API quota (search.list costs 100 units/page; scrape costs nothing, only
the cheap shared videos.list enrich is charged).

- youtube/search_scrape.py: InnerTube search returning the same page shape as
  YouTubeClient.search_videos (items + continuation cursor); SOCS consent cookie,
  videoRenderer walk (channels/playlists/Shorts shelves naturally excluded),
  type:video filter, cached InnerTube key/version with constant fallbacks.
- routes/search.py: admin-selectable source (search_source, default scrape).
  Scrape path skips the 100-unit budget pre-check and logs a zero-cost search
  event so the per-user daily cap still counts it; api path unchanged. Response
  carries the active source.
- sysconfig/config: new search_source key (scrape|api).
- quota.log_action: record a zero-cost action event for per-user rate limits.
This commit is contained in:
npeter83 2026-06-29 22:29:54 +02:00
parent 17c17c1d3c
commit 396e09189b
5 changed files with 228 additions and 12 deletions

View file

@ -1,10 +1,12 @@
"""Live YouTube search.
Searches YouTube directly (search.list) and materialises the results into the shared catalog
so they render with the normal feed cards + in-app player and gain per-user state (watch /
save / resume). search.list costs **100 quota units per call**, so it's the most expensive
read by far: it's gated behind a per-user daily cap + a budget pre-check, and only real users
(never the shared demo account) may call it.
Searches YouTube directly and materialises the results into the shared catalog so they render
with the normal feed cards + in-app player and gain per-user state (watch / save / resume). The
source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own
endpoint, zero API quota the default) or **api** (search.list, **100 quota units per page**).
Either way it's gated behind a per-user daily cap, the API source additionally behind a budget
pre-check, and only real users (never the shared demo account) may call it. The result ids are
enriched through the cheap shared videos.list path (1 unit / 50) regardless of source.
Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
@ -24,6 +26,7 @@ from app.models import Channel, User, Video, VideoState
from app.routes.feed import _serialize, feed_columns
from app.sync.subscriptions import apply_channel_details
from app.sync.videos import _insert_stubs, apply_video_details, parse_dt
from app.youtube import search_scrape
from app.youtube.client import YouTubeClient, YouTubeError
from app.youtube.shorts import make_client, probe_is_short
@ -100,7 +103,13 @@ def search_youtube(
if not term:
raise HTTPException(status_code=400, detail="A search term is required.")
# Guard the shared quota BEFORE spending: per-user daily cap, then today's budget.
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
source = (sysconfig.get_str(db, "search_source") or "scrape").lower()
if source != "scrape":
source = "api"
# Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
# since the search itself spends no quota.
daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
if used >= daily_limit:
@ -108,7 +117,8 @@ def search_youtube(
status_code=429,
detail=f"You've reached today's YouTube search limit ({daily_limit}). Try again tomorrow.",
)
if not quota.can_spend(db, SEARCH_COST):
# The 100-unit budget pre-check only applies to the API source.
if source == "api" and not quota.can_spend(db, SEARCH_COST):
raise HTTPException(
status_code=429,
detail="The shared YouTube quota for today is used up. Try again tomorrow.",
@ -116,9 +126,14 @@ def search_youtube(
with YouTubeClient(db, user) as yt:
try:
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
page = yt.search_videos(term, page_token=cursor)
except YouTubeError as exc:
if source == "api":
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
page = yt.search_videos(term, page_token=cursor)
else:
page = search_scrape.search(term, continuation=cursor)
# Zero-cost, but logged so the per-user daily cap above keeps counting it.
quota.log_action(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
except (YouTubeError, search_scrape.ScrapeError) as exc:
raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}")
# Drop currently-live/upcoming results (search can't filter them and we never ingest
@ -134,6 +149,7 @@ def search_youtube(
"items": [],
"next_cursor": page["next_page_token"],
"has_more": bool(page["next_page_token"]),
"source": source,
}
# 1) Channels are the FK target — create stubs for unknown ones (flagged from_search),
@ -201,4 +217,5 @@ def search_youtube(
"items": _serialize_ids(db, user, ordered),
"next_cursor": page["next_page_token"],
"has_more": bool(page["next_page_token"]),
"source": source,
}