From 396e09189b63533d7466f5400ac5f661ffd524e8 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Mon, 29 Jun 2026 22:29:54 +0200 Subject: [PATCH] feat(search): zero-quota InnerTube scrape source + search_source toggle Live YouTube search can now use YouTube's internal InnerTube endpoint instead of search.list, materialising results through the same enrich/provenance path at zero API quota (search.list costs 100 units/page; scrape costs nothing, only the cheap shared videos.list enrich is charged). - youtube/search_scrape.py: InnerTube search returning the same page shape as YouTubeClient.search_videos (items + continuation cursor); SOCS consent cookie, videoRenderer walk (channels/playlists/Shorts shelves naturally excluded), type:video filter, cached InnerTube key/version with constant fallbacks. - routes/search.py: admin-selectable source (search_source, default scrape). Scrape path skips the 100-unit budget pre-check and logs a zero-cost search event so the per-user daily cap still counts it; api path unchanged. Response carries the active source. - sysconfig/config: new search_source key (scrape|api). - quota.log_action: record a zero-cost action event for per-user rate limits. --- backend/app/config.py | 8 +- backend/app/quota.py | 9 ++ backend/app/routes/search.py | 37 ++++-- backend/app/sysconfig.py | 2 + backend/app/youtube/search_scrape.py | 184 +++++++++++++++++++++++++++ 5 files changed, 228 insertions(+), 12 deletions(-) create mode 100644 backend/app/youtube/search_scrape.py diff --git a/backend/app/config.py b/backend/app/config.py index 933d036..78c1bd8 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -71,9 +71,13 @@ class Settings(BaseSettings): youtube_api_proxy: str = "" # Daily quota budget (units). Default leaves headroom under the 10,000/day free limit. quota_daily_budget: int = 9000 - # Per-user cap on live YouTube searches per day. search.list costs 100 units each, so the - # shared budget only affords ~80-90/day total — this stops one user draining it. Admin-tunable. + # Per-user cap on live YouTube searches per day. With the official API (search_source="api") + # each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with + # the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable. search_daily_limit_per_user: int = 70 + # Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list, + # 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks. + search_source: str = "scrape" # Recent-first backfill: how far back to fetch on the first pass per channel. backfill_recent_max_videos: int = 100 backfill_recent_max_days: int = 365 diff --git a/backend/app/quota.py b/backend/app/quota.py index 0737675..1e72e56 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -110,6 +110,15 @@ def can_spend(db: Session, units: int) -> bool: return remaining_today(db) >= units +def log_action(db: Session, user_id: int, action: str) -> None: + """Record a zero-cost action event so per-user daily caps still count it. + + `record_usage` only logs when units are actually charged; a scrape-based search spends no + quota yet must still be rate-limited per user, so it logs its event here directly.""" + db.add(QuotaEvent(user_id=user_id, action=action, units=0)) + db.commit() + + def actions_today(db: Session, user_id: int, action: str) -> int: """How many quota events of `action` this user has logged so far in the current Pacific day — for per-user, per-action daily caps (e.g. the live-search limit). Counts events, diff --git a/backend/app/routes/search.py b/backend/app/routes/search.py index bb29119..efba64f 100644 --- a/backend/app/routes/search.py +++ b/backend/app/routes/search.py @@ -1,10 +1,12 @@ """Live YouTube search. -Searches YouTube directly (search.list) and materialises the results into the shared catalog -so they render with the normal feed cards + in-app player and gain per-user state (watch / -save / resume). search.list costs **100 quota units per call**, so it's the most expensive -read by far: it's gated behind a per-user daily cap + a budget pre-check, and only real users -(never the shared demo account) may call it. +Searches YouTube directly and materialises the results into the shared catalog so they render +with the normal feed cards + in-app player and gain per-user state (watch / save / resume). The +source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own +endpoint, zero API quota — the default) or **api** (search.list, **100 quota units per page**). +Either way it's gated behind a per-user daily cap, the API source additionally behind a budget +pre-check, and only real users (never the shared demo account) may call it. The result ids are +enriched through the cheap shared videos.list path (1 unit / 50) regardless of source. Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never @@ -24,6 +26,7 @@ from app.models import Channel, User, Video, VideoState from app.routes.feed import _serialize, feed_columns from app.sync.subscriptions import apply_channel_details from app.sync.videos import _insert_stubs, apply_video_details, parse_dt +from app.youtube import search_scrape from app.youtube.client import YouTubeClient, YouTubeError from app.youtube.shorts import make_client, probe_is_short @@ -100,7 +103,13 @@ def search_youtube( if not term: raise HTTPException(status_code=400, detail="A search term is required.") - # Guard the shared quota BEFORE spending: per-user daily cap, then today's budget. + # Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page). + source = (sysconfig.get_str(db, "search_source") or "scrape").lower() + if source != "scrape": + source = "api" + + # Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard + # since the search itself spends no quota. daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user") used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH) if used >= daily_limit: @@ -108,7 +117,8 @@ def search_youtube( status_code=429, detail=f"You've reached today's YouTube search limit ({daily_limit}). Try again tomorrow.", ) - if not quota.can_spend(db, SEARCH_COST): + # The 100-unit budget pre-check only applies to the API source. + if source == "api" and not quota.can_spend(db, SEARCH_COST): raise HTTPException( status_code=429, detail="The shared YouTube quota for today is used up. Try again tomorrow.", @@ -116,9 +126,14 @@ def search_youtube( with YouTubeClient(db, user) as yt: try: - with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH): - page = yt.search_videos(term, page_token=cursor) - except YouTubeError as exc: + if source == "api": + with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH): + page = yt.search_videos(term, page_token=cursor) + else: + page = search_scrape.search(term, continuation=cursor) + # Zero-cost, but logged so the per-user daily cap above keeps counting it. + quota.log_action(db, user.id, quota.QuotaAction.VIDEOS_SEARCH) + except (YouTubeError, search_scrape.ScrapeError) as exc: raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}") # Drop currently-live/upcoming results (search can't filter them and we never ingest @@ -134,6 +149,7 @@ def search_youtube( "items": [], "next_cursor": page["next_page_token"], "has_more": bool(page["next_page_token"]), + "source": source, } # 1) Channels are the FK target — create stubs for unknown ones (flagged from_search), @@ -201,4 +217,5 @@ def search_youtube( "items": _serialize_ids(db, user, ordered), "next_cursor": page["next_page_token"], "has_more": bool(page["next_page_token"]), + "source": source, } diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index 1b45155..c6d8004 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -40,6 +40,8 @@ SPECS: tuple[ConfigSpec, ...] = ( ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000), ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000), ConfigSpec("search_daily_limit_per_user", "int", "quota", "search_daily_limit_per_user", min=0, max=100_000), + # Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page). + ConfigSpec("search_source", "str", "quota", "search_source"), # --- Backfill (recent-first first pass per channel) --- ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000), ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500), diff --git a/backend/app/youtube/search_scrape.py b/backend/app/youtube/search_scrape.py new file mode 100644 index 0000000..05a4f78 --- /dev/null +++ b/backend/app/youtube/search_scrape.py @@ -0,0 +1,184 @@ +"""Zero-quota YouTube search via InnerTube (the website's own youtubei/v1/search endpoint). + +Live YouTube search **without spending any YouTube Data API quota**: instead of search.list +(100 units per page) we POST to YouTube's internal InnerTube endpoint — the same one the website +calls — and read the result list straight out of the JSON. The returned video ids then go +through the normal enrich path (videos.list, 1 unit / 50), exactly as the official search does, +so the search itself is effectively free and only the (cheap, shared) enrichment is charged. + +The SOCS=CAI consent-skip cookie is used like the Shorts probe so a server-side request isn't +bounced to consent.youtube.com. This deliberately scrapes an undocumented endpoint: if YouTube +changes the JSON shape or blocks the host, an admin can flip ``search_source`` back to ``api``. + +The returned page mirrors :meth:`YouTubeClient.search_videos` exactly +(``{"items": [...], "next_page_token": str | None}``) so the route is source-agnostic — the +only difference is the cursor is an InnerTube *continuation token*, not a Data API pageToken. +``published_at`` is left ``None`` (the scrape only exposes a relative time like "3 years ago"); +the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"`` +and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path).""" +import re + +import httpx + +_SEARCH_URL = "https://www.youtube.com/youtubei/v1/search" +_RESULTS_URL = "https://www.youtube.com/results" +# Public WEB InnerTube key + a recent client version. These are also embedded in every results +# page; we fetch the live pair once and cache it, falling back to these constants. +_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" +_FALLBACK_CLIENT_VERSION = "2.20240101.00.00" +# sp / params value selecting the "type: video" search filter (base64 protobuf EgIQAQ==). +_TYPE_VIDEO = "EgIQAQ==" + +_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + # SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is + # redirected to consent.youtube.com). + "Cookie": "SOCS=CAI", + "Accept-Language": "en-US,en;q=0.9", +} + +_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"') +_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"') + +# Cached (api_key, client_version) extracted from a results page; refreshed on demand. +_config: tuple[str, str] | None = None + + +class ScrapeError(Exception): + """A scrape-based search could not be completed (network, blocked, or unexpected shape).""" + + +def _runs_text(node: dict | None) -> str | None: + if not node: + return None + if node.get("simpleText"): + return node["simpleText"] + runs = node.get("runs") or [] + if runs: + return "".join(r.get("text", "") for r in runs) + return None + + +def _get_config(client: httpx.Client) -> tuple[str, str]: + """The current InnerTube (api_key, client_version), cached after the first fetch.""" + global _config + if _config is not None: + return _config + try: + resp = client.get(_RESULTS_URL, params={"search_query": "music"}) + html = resp.text + key = _KEY_RE.search(html) + ver = _VER_RE.search(html) + _config = ( + key.group(1) if key else _FALLBACK_KEY, + ver.group(1) if ver else _FALLBACK_CLIENT_VERSION, + ) + except httpx.HTTPError: + _config = (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION) + return _config + + +def _video_item(vr: dict) -> dict | None: + """Flatten a YouTube `videoRenderer` into the same shape as YouTubeClient.search_videos.""" + vid = vr.get("videoId") + if not vid: + return None + owner = vr.get("ownerText") or vr.get("longBylineText") or {} + runs = owner.get("runs") or [] + channel_id = channel_title = None + if runs: + channel_title = runs[0].get("text") + channel_id = ( + runs[0] + .get("navigationEndpoint", {}) + .get("browseEndpoint", {}) + .get("browseId") + ) + thumbs = (vr.get("thumbnail") or {}).get("thumbnails") or [] + return { + "id": vid, + "channel_id": channel_id, + "channel_title": channel_title, + "title": _runs_text(vr.get("title")), + "published_at": None, # scrape exposes only a relative time; enrich fills the real date + "thumbnail_url": thumbs[-1]["url"] if thumbs else None, + "live_broadcast": "none", # live/upcoming filtered post-enrich via live_status + } + + +def _collect_videos(obj, out: list[dict], seen: set[str]) -> None: + """Walk the response collecting videoRenderer items in document (relevance) order. Only + `videoRenderer` is taken, so channels/playlists/Shorts shelves (reelItemRenderer) are + naturally excluded.""" + if isinstance(obj, dict): + vr = obj.get("videoRenderer") + if isinstance(vr, dict): + item = _video_item(vr) + if item and item["id"] not in seen: + seen.add(item["id"]) + out.append(item) + for v in obj.values(): + _collect_videos(v, out, seen) + elif isinstance(obj, list): + for v in obj: + _collect_videos(v, out, seen) + + +def _find_continuation(obj) -> str | None: + """First continuationItemRenderer token (the 'load more' cursor), or None at the end.""" + if isinstance(obj, dict): + cir = obj.get("continuationItemRenderer") + if isinstance(cir, dict): + tok = ( + cir.get("continuationEndpoint", {}) + .get("continuationCommand", {}) + .get("token") + ) + if tok: + return tok + for v in obj.values(): + tok = _find_continuation(v) + if tok: + return tok + elif isinstance(obj, list): + for v in obj: + tok = _find_continuation(v) + if tok: + return tok + return None + + +def search(q: str, continuation: str | None = None) -> dict: + """One page of zero-quota YouTube search results. + + `continuation` is the InnerTube continuation token from a previous page (None for the first + page). Returns ``{"items": [...], "next_page_token": str | None}`` — the same shape as + YouTubeClient.search_videos so the route handles both sources identically.""" + with httpx.Client(timeout=20.0, headers=_HEADERS) as client: + api_key, client_version = _get_config(client) + context = { + "client": { + "clientName": "WEB", + "clientVersion": client_version, + "hl": "en", + "gl": "US", + } + } + body: dict = {"context": context} + if continuation: + body["continuation"] = continuation + else: + body["query"] = q + body["params"] = _TYPE_VIDEO + try: + resp = client.post(_SEARCH_URL, params={"key": api_key}, json=body) + resp.raise_for_status() + data = resp.json() + except httpx.HTTPError as exc: + raise ScrapeError(f"InnerTube request failed: {exc}") from exc + except ValueError as exc: # non-JSON body (blocked / consent / captcha) + raise ScrapeError("InnerTube returned a non-JSON response") from exc + + items: list[dict] = [] + _collect_videos(data, items, set()) + return {"items": items, "next_page_token": _find_continuation(data)}