feat(search): zero-quota InnerTube scrape source + search_source toggle

Live YouTube search can now use YouTube's internal InnerTube endpoint instead
of search.list, materialising results through the same enrich/provenance path
at zero API quota (search.list costs 100 units/page; scrape costs nothing, only
the cheap shared videos.list enrich is charged).

- youtube/search_scrape.py: InnerTube search returning the same page shape as
  YouTubeClient.search_videos (items + continuation cursor); SOCS consent cookie,
  videoRenderer walk (channels/playlists/Shorts shelves naturally excluded),
  type:video filter, cached InnerTube key/version with constant fallbacks.
- routes/search.py: admin-selectable source (search_source, default scrape).
  Scrape path skips the 100-unit budget pre-check and logs a zero-cost search
  event so the per-user daily cap still counts it; api path unchanged. Response
  carries the active source.
- sysconfig/config: new search_source key (scrape|api).
- quota.log_action: record a zero-cost action event for per-user rate limits.

backend/app/config.py

@@ -71,9 +71,13 @@ class Settings(BaseSettings):
     youtube_api_proxy: str = ""
     # Daily quota budget (units). Default leaves headroom under the 10,000/day free limit.
     quota_daily_budget: int = 9000
-    # Per-user cap on live YouTube searches per day. search.list costs 100 units each, so the
+    # Per-user cap on live YouTube searches per day. With the official API (search_source="api")
-    # shared budget only affords ~80-90/day total — this stops one user draining it. Admin-tunable.
+    # each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with
+    # the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable.
     search_daily_limit_per_user: int = 70
+    # Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
+    # 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
+    search_source: str = "scrape"
     # Recent-first backfill: how far back to fetch on the first pass per channel.
     backfill_recent_max_videos: int = 100
     backfill_recent_max_days: int = 365

backend/app/quota.py

@@ -110,6 +110,15 @@ def can_spend(db: Session, units: int) -> bool:
     return remaining_today(db) >= units
 
 
+def log_action(db: Session, user_id: int, action: str) -> None:
+    """Record a zero-cost action event so per-user daily caps still count it.
+
+    `record_usage` only logs when units are actually charged; a scrape-based search spends no
+    quota yet must still be rate-limited per user, so it logs its event here directly."""
+    db.add(QuotaEvent(user_id=user_id, action=action, units=0))
+    db.commit()
+
+
 def actions_today(db: Session, user_id: int, action: str) -> int:
     """How many quota events of `action` this user has logged so far in the current Pacific
     day — for per-user, per-action daily caps (e.g. the live-search limit). Counts events,

backend/app/routes/search.py

@@ -1,10 +1,12 @@
 """Live YouTube search.
 
-Searches YouTube directly (search.list) and materialises the results into the shared catalog
+Searches YouTube directly and materialises the results into the shared catalog so they render
-so they render with the normal feed cards + in-app player and gain per-user state (watch /
+with the normal feed cards + in-app player and gain per-user state (watch / save / resume). The
-save / resume). search.list costs **100 quota units per call**, so it's the most expensive
+source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own
-read by far: it's gated behind a per-user daily cap + a budget pre-check, and only real users
+endpoint, zero API quota — the default) or **api** (search.list, **100 quota units per page**).
-(never the shared demo account) may call it.
+Either way it's gated behind a per-user daily cap, the API source additionally behind a budget
+pre-check, and only real users (never the shared demo account) may call it. The result ids are
+enriched through the cheap shared videos.list path (1 unit / 50) regardless of source.
 
 Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
 this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
@@ -24,6 +26,7 @@ from app.models import Channel, User, Video, VideoState
 from app.routes.feed import _serialize, feed_columns
 from app.sync.subscriptions import apply_channel_details
 from app.sync.videos import _insert_stubs, apply_video_details, parse_dt
+from app.youtube import search_scrape
 from app.youtube.client import YouTubeClient, YouTubeError
 from app.youtube.shorts import make_client, probe_is_short
 
@@ -100,7 +103,13 @@ def search_youtube(
     if not term:
         raise HTTPException(status_code=400, detail="A search term is required.")
 
-    # Guard the shared quota BEFORE spending: per-user daily cap, then today's budget.
+    # Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
+    source = (sysconfig.get_str(db, "search_source") or "scrape").lower()
+    if source != "scrape":
+        source = "api"
+
+    # Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
+    # since the search itself spends no quota.
     daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
     used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
     if used >= daily_limit:
@@ -108,7 +117,8 @@ def search_youtube(
             status_code=429,
             detail=f"You've reached today's YouTube search limit ({daily_limit}). Try again tomorrow.",
         )
-    if not quota.can_spend(db, SEARCH_COST):
+    # The 100-unit budget pre-check only applies to the API source.
+    if source == "api" and not quota.can_spend(db, SEARCH_COST):
         raise HTTPException(
             status_code=429,
             detail="The shared YouTube quota for today is used up. Try again tomorrow.",
@@ -116,9 +126,14 @@ def search_youtube(
 
     with YouTubeClient(db, user) as yt:
         try:
+            if source == "api":
                 with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
                     page = yt.search_videos(term, page_token=cursor)
-        except YouTubeError as exc:
+            else:
+                page = search_scrape.search(term, continuation=cursor)
+                # Zero-cost, but logged so the per-user daily cap above keeps counting it.
+                quota.log_action(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
+        except (YouTubeError, search_scrape.ScrapeError) as exc:
             raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}")
 
         # Drop currently-live/upcoming results (search can't filter them and we never ingest
@@ -134,6 +149,7 @@ def search_youtube(
                 "items": [],
                 "next_cursor": page["next_page_token"],
                 "has_more": bool(page["next_page_token"]),
+                "source": source,
             }
 
         # 1) Channels are the FK target — create stubs for unknown ones (flagged from_search),
@@ -201,4 +217,5 @@ def search_youtube(
         "items": _serialize_ids(db, user, ordered),
         "next_cursor": page["next_page_token"],
         "has_more": bool(page["next_page_token"]),
+        "source": source,
     }

backend/app/sysconfig.py

@@ -40,6 +40,8 @@ SPECS: tuple[ConfigSpec, ...] = (
     ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
     ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000),
     ConfigSpec("search_daily_limit_per_user", "int", "quota", "search_daily_limit_per_user", min=0, max=100_000),
+    # Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page).
+    ConfigSpec("search_source", "str", "quota", "search_source"),
     # --- Backfill (recent-first first pass per channel) ---
     ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000),
     ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500),

backend/app/youtube/search_scrape.py

@@ -0,0 +1,184 @@
+"""Zero-quota YouTube search via InnerTube (the website's own youtubei/v1/search endpoint).
+
+Live YouTube search **without spending any YouTube Data API quota**: instead of search.list
+(100 units per page) we POST to YouTube's internal InnerTube endpoint — the same one the website
+calls — and read the result list straight out of the JSON. The returned video ids then go
+through the normal enrich path (videos.list, 1 unit / 50), exactly as the official search does,
+so the search itself is effectively free and only the (cheap, shared) enrichment is charged.
+
+The SOCS=CAI consent-skip cookie is used like the Shorts probe so a server-side request isn't
+bounced to consent.youtube.com. This deliberately scrapes an undocumented endpoint: if YouTube
+changes the JSON shape or blocks the host, an admin can flip ``search_source`` back to ``api``.
+
+The returned page mirrors :meth:`YouTubeClient.search_videos` exactly
+(``{"items": [...], "next_page_token": str | None}``) so the route is source-agnostic — the
+only difference is the cursor is an InnerTube *continuation token*, not a Data API pageToken.
+``published_at`` is left ``None`` (the scrape only exposes a relative time like "3 years ago");
+the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"``
+and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path)."""
+import re
+
+import httpx
+
+_SEARCH_URL = "https://www.youtube.com/youtubei/v1/search"
+_RESULTS_URL = "https://www.youtube.com/results"
+# Public WEB InnerTube key + a recent client version. These are also embedded in every results
+# page; we fetch the live pair once and cache it, falling back to these constants.
+_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
+_FALLBACK_CLIENT_VERSION = "2.20240101.00.00"
+# sp / params value selecting the "type: video" search filter (base64 protobuf EgIQAQ==).
+_TYPE_VIDEO = "EgIQAQ=="
+
+_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
+    # SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is
+    # redirected to consent.youtube.com).
+    "Cookie": "SOCS=CAI",
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
+_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
+
+# Cached (api_key, client_version) extracted from a results page; refreshed on demand.
+_config: tuple[str, str] | None = None
+
+
+class ScrapeError(Exception):
+    """A scrape-based search could not be completed (network, blocked, or unexpected shape)."""
+
+
+def _runs_text(node: dict | None) -> str | None:
+    if not node:
+        return None
+    if node.get("simpleText"):
+        return node["simpleText"]
+    runs = node.get("runs") or []
+    if runs:
+        return "".join(r.get("text", "") for r in runs)
+    return None
+
+
+def _get_config(client: httpx.Client) -> tuple[str, str]:
+    """The current InnerTube (api_key, client_version), cached after the first fetch."""
+    global _config
+    if _config is not None:
+        return _config
+    try:
+        resp = client.get(_RESULTS_URL, params={"search_query": "music"})
+        html = resp.text
+        key = _KEY_RE.search(html)
+        ver = _VER_RE.search(html)
+        _config = (
+            key.group(1) if key else _FALLBACK_KEY,
+            ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
+        )
+    except httpx.HTTPError:
+        _config = (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
+    return _config
+
+
+def _video_item(vr: dict) -> dict | None:
+    """Flatten a YouTube `videoRenderer` into the same shape as YouTubeClient.search_videos."""
+    vid = vr.get("videoId")
+    if not vid:
+        return None
+    owner = vr.get("ownerText") or vr.get("longBylineText") or {}
+    runs = owner.get("runs") or []
+    channel_id = channel_title = None
+    if runs:
+        channel_title = runs[0].get("text")
+        channel_id = (
+            runs[0]
+            .get("navigationEndpoint", {})
+            .get("browseEndpoint", {})
+            .get("browseId")
+        )
+    thumbs = (vr.get("thumbnail") or {}).get("thumbnails") or []
+    return {
+        "id": vid,
+        "channel_id": channel_id,
+        "channel_title": channel_title,
+        "title": _runs_text(vr.get("title")),
+        "published_at": None,  # scrape exposes only a relative time; enrich fills the real date
+        "thumbnail_url": thumbs[-1]["url"] if thumbs else None,
+        "live_broadcast": "none",  # live/upcoming filtered post-enrich via live_status
+    }
+
+
+def _collect_videos(obj, out: list[dict], seen: set[str]) -> None:
+    """Walk the response collecting videoRenderer items in document (relevance) order. Only
+    `videoRenderer` is taken, so channels/playlists/Shorts shelves (reelItemRenderer) are
+    naturally excluded."""
+    if isinstance(obj, dict):
+        vr = obj.get("videoRenderer")
+        if isinstance(vr, dict):
+            item = _video_item(vr)
+            if item and item["id"] not in seen:
+                seen.add(item["id"])
+                out.append(item)
+        for v in obj.values():
+            _collect_videos(v, out, seen)
+    elif isinstance(obj, list):
+        for v in obj:
+            _collect_videos(v, out, seen)
+
+
+def _find_continuation(obj) -> str | None:
+    """First continuationItemRenderer token (the 'load more' cursor), or None at the end."""
+    if isinstance(obj, dict):
+        cir = obj.get("continuationItemRenderer")
+        if isinstance(cir, dict):
+            tok = (
+                cir.get("continuationEndpoint", {})
+                .get("continuationCommand", {})
+                .get("token")
+            )
+            if tok:
+                return tok
+        for v in obj.values():
+            tok = _find_continuation(v)
+            if tok:
+                return tok
+    elif isinstance(obj, list):
+        for v in obj:
+            tok = _find_continuation(v)
+            if tok:
+                return tok
+    return None
+
+
+def search(q: str, continuation: str | None = None) -> dict:
+    """One page of zero-quota YouTube search results.
+
+    `continuation` is the InnerTube continuation token from a previous page (None for the first
+    page). Returns ``{"items": [...], "next_page_token": str | None}`` — the same shape as
+    YouTubeClient.search_videos so the route handles both sources identically."""
+    with httpx.Client(timeout=20.0, headers=_HEADERS) as client:
+        api_key, client_version = _get_config(client)
+        context = {
+            "client": {
+                "clientName": "WEB",
+                "clientVersion": client_version,
+                "hl": "en",
+                "gl": "US",
+            }
+        }
+        body: dict = {"context": context}
+        if continuation:
+            body["continuation"] = continuation
+        else:
+            body["query"] = q
+            body["params"] = _TYPE_VIDEO
+        try:
+            resp = client.post(_SEARCH_URL, params={"key": api_key}, json=body)
+            resp.raise_for_status()
+            data = resp.json()
+        except httpx.HTTPError as exc:
+            raise ScrapeError(f"InnerTube request failed: {exc}") from exc
+        except ValueError as exc:  # non-JSON body (blocked / consent / captcha)
+            raise ScrapeError("InnerTube returned a non-JSON response") from exc
+
+    items: list[dict] = []
+    _collect_videos(data, items, set())
+    return {"items": items, "next_page_token": _find_continuation(data)}