feat(search): zero-quota InnerTube scrape source + search_source toggle
Live YouTube search can now use YouTube's internal InnerTube endpoint instead of search.list, materialising results through the same enrich/provenance path at zero API quota (search.list costs 100 units/page; scrape costs nothing, only the cheap shared videos.list enrich is charged). - youtube/search_scrape.py: InnerTube search returning the same page shape as YouTubeClient.search_videos (items + continuation cursor); SOCS consent cookie, videoRenderer walk (channels/playlists/Shorts shelves naturally excluded), type:video filter, cached InnerTube key/version with constant fallbacks. - routes/search.py: admin-selectable source (search_source, default scrape). Scrape path skips the 100-unit budget pre-check and logs a zero-cost search event so the per-user daily cap still counts it; api path unchanged. Response carries the active source. - sysconfig/config: new search_source key (scrape|api). - quota.log_action: record a zero-cost action event for per-user rate limits.
This commit is contained in:
parent
17c17c1d3c
commit
396e09189b
5 changed files with 228 additions and 12 deletions
|
|
@ -1,10 +1,12 @@
|
|||
"""Live YouTube search.
|
||||
|
||||
Searches YouTube directly (search.list) and materialises the results into the shared catalog
|
||||
so they render with the normal feed cards + in-app player and gain per-user state (watch /
|
||||
save / resume). search.list costs **100 quota units per call**, so it's the most expensive
|
||||
read by far: it's gated behind a per-user daily cap + a budget pre-check, and only real users
|
||||
(never the shared demo account) may call it.
|
||||
Searches YouTube directly and materialises the results into the shared catalog so they render
|
||||
with the normal feed cards + in-app player and gain per-user state (watch / save / resume). The
|
||||
source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own
|
||||
endpoint, zero API quota — the default) or **api** (search.list, **100 quota units per page**).
|
||||
Either way it's gated behind a per-user daily cap, the API source additionally behind a budget
|
||||
pre-check, and only real users (never the shared demo account) may call it. The result ids are
|
||||
enriched through the cheap shared videos.list path (1 unit / 50) regardless of source.
|
||||
|
||||
Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
|
||||
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
|
||||
|
|
@ -24,6 +26,7 @@ from app.models import Channel, User, Video, VideoState
|
|||
from app.routes.feed import _serialize, feed_columns
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.sync.videos import _insert_stubs, apply_video_details, parse_dt
|
||||
from app.youtube import search_scrape
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.youtube.shorts import make_client, probe_is_short
|
||||
|
||||
|
|
@ -100,7 +103,13 @@ def search_youtube(
|
|||
if not term:
|
||||
raise HTTPException(status_code=400, detail="A search term is required.")
|
||||
|
||||
# Guard the shared quota BEFORE spending: per-user daily cap, then today's budget.
|
||||
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
|
||||
source = (sysconfig.get_str(db, "search_source") or "scrape").lower()
|
||||
if source != "scrape":
|
||||
source = "api"
|
||||
|
||||
# Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
|
||||
# since the search itself spends no quota.
|
||||
daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
|
||||
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
||||
if used >= daily_limit:
|
||||
|
|
@ -108,7 +117,8 @@ def search_youtube(
|
|||
status_code=429,
|
||||
detail=f"You've reached today's YouTube search limit ({daily_limit}). Try again tomorrow.",
|
||||
)
|
||||
if not quota.can_spend(db, SEARCH_COST):
|
||||
# The 100-unit budget pre-check only applies to the API source.
|
||||
if source == "api" and not quota.can_spend(db, SEARCH_COST):
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="The shared YouTube quota for today is used up. Try again tomorrow.",
|
||||
|
|
@ -116,9 +126,14 @@ def search_youtube(
|
|||
|
||||
with YouTubeClient(db, user) as yt:
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
|
||||
page = yt.search_videos(term, page_token=cursor)
|
||||
except YouTubeError as exc:
|
||||
if source == "api":
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
|
||||
page = yt.search_videos(term, page_token=cursor)
|
||||
else:
|
||||
page = search_scrape.search(term, continuation=cursor)
|
||||
# Zero-cost, but logged so the per-user daily cap above keeps counting it.
|
||||
quota.log_action(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
||||
except (YouTubeError, search_scrape.ScrapeError) as exc:
|
||||
raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}")
|
||||
|
||||
# Drop currently-live/upcoming results (search can't filter them and we never ingest
|
||||
|
|
@ -134,6 +149,7 @@ def search_youtube(
|
|||
"items": [],
|
||||
"next_cursor": page["next_page_token"],
|
||||
"has_more": bool(page["next_page_token"]),
|
||||
"source": source,
|
||||
}
|
||||
|
||||
# 1) Channels are the FK target — create stubs for unknown ones (flagged from_search),
|
||||
|
|
@ -201,4 +217,5 @@ def search_youtube(
|
|||
"items": _serialize_ids(db, user, ordered),
|
||||
"next_cursor": page["next_page_token"],
|
||||
"has_more": bool(page["next_page_token"]),
|
||||
"source": source,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue