peter/siftlode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(search): zero-quota InnerTube scrape source + search_source toggle

Browse source 
Live YouTube search can now use YouTube's internal InnerTube endpoint instead
of search.list, materialising results through the same enrich/provenance path
at zero API quota (search.list costs 100 units/page; scrape costs nothing, only
the cheap shared videos.list enrich is charged).

- youtube/search_scrape.py: InnerTube search returning the same page shape as
  YouTubeClient.search_videos (items + continuation cursor); SOCS consent cookie,
  videoRenderer walk (channels/playlists/Shorts shelves naturally excluded),
  type:video filter, cached InnerTube key/version with constant fallbacks.
- routes/search.py: admin-selectable source (search_source, default scrape).
  Scrape path skips the 100-unit budget pre-check and logs a zero-cost search
  event so the per-user daily cap still counts it; api path unchanged. Response
  carries the active source.
- sysconfig/config: new search_source key (scrape|api).
- quota.log_action: record a zero-cost action event for per-user rate limits.
This commit is contained in:
npeter83 2026-06-29 22:29:54 +02:00
parent 17c17c1d3c
commit 396e09189b
5 changed files with 228 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

8
backend/app/config.py
View file

@ -71,9 +71,13 @@ class Settings(BaseSettings):
youtube_api_proxy: str = ""
# Daily quota budget (units). Default leaves headroom under the 10,000/day free limit.
quota_daily_budget: int = 9000
# Per-user cap on live YouTube searches per day. search.list costs 100 units each, so the
# shared budget only affords ~80-90/day total — this stops one user draining it. Admin-tunable.
# Per-user cap on live YouTube searches per day. With the official API (search_source="api")
# each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with
# the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable.
search_daily_limit_per_user: int = 70
# Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
# 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
search_source: str = "scrape"
# Recent-first backfill: how far back to fetch on the first pass per channel.
backfill_recent_max_videos: int = 100
backfill_recent_max_days: int = 365

9
backend/app/quota.py
View file

@ -110,6 +110,15 @@ def can_spend(db: Session, units: int) -> bool:
return remaining_today(db) >= units
def log_action(db: Session, user_id: int, action: str) -> None:
"""Record a zero-cost action event so per-user daily caps still count it.
`record_usage` only logs when units are actually charged; a scrape-based search spends no
quota yet must still be rate-limited per user, so it logs its event here directly."""
db.add(QuotaEvent(user_id=user_id, action=action, units=0))
db.commit()
def actions_today(db: Session, user_id: int, action: str) -> int:
"""How many quota events of `action` this user has logged so far in the current Pacific
day — for per-user, per-action daily caps (e.g. the live-search limit). Counts events,

37
backend/app/routes/search.py
View file

@ -1,10 +1,12 @@
"""Live YouTube search.
Searches YouTube directly (search.list) and materialises the results into the shared catalog
so they render with the normal feed cards + in-app player and gain per-user state (watch /
save / resume). search.list costs **100 quota units per call**, so it's the most expensive
read by far: it's gated behind a per-user daily cap + a budget pre-check, and only real users
(never the shared demo account) may call it.
Searches YouTube directly and materialises the results into the shared catalog so they render
with the normal feed cards + in-app player and gain per-user state (watch / save / resume). The
source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own
endpoint, zero API quota — the default) or **api** (search.list, **100 quota units per page**).
Either way it's gated behind a per-user daily cap, the API source additionally behind a budget
pre-check, and only real users (never the shared demo account) may call it. The result ids are
enriched through the cheap shared videos.list path (1 unit / 50) regardless of source.
Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
@ -24,6 +26,7 @@ from app.models import Channel, User, Video, VideoState
from app.routes.feed import _serialize, feed_columns
from app.sync.subscriptions import apply_channel_details
from app.sync.videos import _insert_stubs, apply_video_details, parse_dt
from app.youtube import search_scrape
from app.youtube.client import YouTubeClient, YouTubeError
from app.youtube.shorts import make_client, probe_is_short
@ -100,7 +103,13 @@ def search_youtube(
if not term:
raise HTTPException(status_code=400, detail="A search term is required.")
# Guard the shared quota BEFORE spending: per-user daily cap, then today's budget.
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
source = (sysconfig.get_str(db, "search_source") or "scrape").lower()
if source != "scrape":
source = "api"
# Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
# since the search itself spends no quota.
daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
if used >= daily_limit:
@ -108,7 +117,8 @@ def search_youtube(
status_code=429,
detail=f"You've reached today's YouTube search limit ({daily_limit}). Try again tomorrow.",
)
if not quota.can_spend(db, SEARCH_COST):
# The 100-unit budget pre-check only applies to the API source.
if source == "api" and not quota.can_spend(db, SEARCH_COST):
raise HTTPException(
status_code=429,
detail="The shared YouTube quota for today is used up. Try again tomorrow.",
@ -116,9 +126,14 @@ def search_youtube(
with YouTubeClient(db, user) as yt:
try:
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
page = yt.search_videos(term, page_token=cursor)
except YouTubeError as exc:
if source == "api":
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
page = yt.search_videos(term, page_token=cursor)
else:
page = search_scrape.search(term, continuation=cursor)
# Zero-cost, but logged so the per-user daily cap above keeps counting it.
quota.log_action(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
except (YouTubeError, search_scrape.ScrapeError) as exc:
raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}")
# Drop currently-live/upcoming results (search can't filter them and we never ingest
@ -134,6 +149,7 @@ def search_youtube(
"items": [],
"next_cursor": page["next_page_token"],
"has_more": bool(page["next_page_token"]),
"source": source,
}
# 1) Channels are the FK target — create stubs for unknown ones (flagged from_search),
@ -201,4 +217,5 @@ def search_youtube(
"items": _serialize_ids(db, user, ordered),
"next_cursor": page["next_page_token"],
"has_more": bool(page["next_page_token"]),
"source": source,
}

2
backend/app/sysconfig.py
View file

@ -40,6 +40,8 @@ SPECS: tuple[ConfigSpec, ...] = (
ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000),
ConfigSpec("search_daily_limit_per_user", "int", "quota", "search_daily_limit_per_user", min=0, max=100_000),
# Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page).
ConfigSpec("search_source", "str", "quota", "search_source"),
# --- Backfill (recent-first first pass per channel) ---
ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000),
ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500),

184
backend/app/youtube/search_scrape.py Normal file
View file

@ -0,0 +1,184 @@
"""Zero-quota YouTube search via InnerTube (the website's own youtubei/v1/search endpoint).
Live YouTube search **without spending any YouTube Data API quota**: instead of search.list
(100 units per page) we POST to YouTube's internal InnerTube endpoint — the same one the website
calls — and read the result list straight out of the JSON. The returned video ids then go
through the normal enrich path (videos.list, 1 unit / 50), exactly as the official search does,
so the search itself is effectively free and only the (cheap, shared) enrichment is charged.
The SOCS=CAI consent-skip cookie is used like the Shorts probe so a server-side request isn't
bounced to consent.youtube.com. This deliberately scrapes an undocumented endpoint: if YouTube
changes the JSON shape or blocks the host, an admin can flip ``search_source`` back to ``api``.
The returned page mirrors :meth:`YouTubeClient.search_videos` exactly
(``{"items": [...], "next_page_token": str | None}``) so the route is source-agnostic — the
only difference is the cursor is an InnerTube *continuation token*, not a Data API pageToken.
``published_at`` is left ``None`` (the scrape only exposes a relative time like "3 years ago");
the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"``
and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path)."""
import re
import httpx
_SEARCH_URL = "https://www.youtube.com/youtubei/v1/search"
_RESULTS_URL = "https://www.youtube.com/results"
# Public WEB InnerTube key + a recent client version. These are also embedded in every results
# page; we fetch the live pair once and cache it, falling back to these constants.
_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
_FALLBACK_CLIENT_VERSION = "2.20240101.00.00"
# sp / params value selecting the "type: video" search filter (base64 protobuf EgIQAQ==).
_TYPE_VIDEO = "EgIQAQ=="
_HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
# SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is
# redirected to consent.youtube.com).
"Cookie": "SOCS=CAI",
"Accept-Language": "en-US,en;q=0.9",
}
_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
# Cached (api_key, client_version) extracted from a results page; refreshed on demand.
_config: tuple[str, str] | None = None
class ScrapeError(Exception):
"""A scrape-based search could not be completed (network, blocked, or unexpected shape)."""
def _runs_text(node: dict | None) -> str | None:
if not node:
return None
if node.get("simpleText"):
return node["simpleText"]
runs = node.get("runs") or []
if runs:
return "".join(r.get("text", "") for r in runs)
return None
def _get_config(client: httpx.Client) -> tuple[str, str]:
"""The current InnerTube (api_key, client_version), cached after the first fetch."""
global _config
if _config is not None:
return _config
try:
resp = client.get(_RESULTS_URL, params={"search_query": "music"})
html = resp.text
key = _KEY_RE.search(html)
ver = _VER_RE.search(html)
_config = (
key.group(1) if key else _FALLBACK_KEY,
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
)
except httpx.HTTPError:
_config = (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
return _config
def _video_item(vr: dict) -> dict | None:
"""Flatten a YouTube `videoRenderer` into the same shape as YouTubeClient.search_videos."""
vid = vr.get("videoId")
if not vid:
return None
owner = vr.get("ownerText") or vr.get("longBylineText") or {}
runs = owner.get("runs") or []
channel_id = channel_title = None
if runs:
channel_title = runs[0].get("text")
channel_id = (
runs[0]
.get("navigationEndpoint", {})
.get("browseEndpoint", {})
.get("browseId")
)
thumbs = (vr.get("thumbnail") or {}).get("thumbnails") or []
return {
"id": vid,
"channel_id": channel_id,
"channel_title": channel_title,
"title": _runs_text(vr.get("title")),
"published_at": None, # scrape exposes only a relative time; enrich fills the real date
"thumbnail_url": thumbs[-1]["url"] if thumbs else None,
"live_broadcast": "none", # live/upcoming filtered post-enrich via live_status
}
def _collect_videos(obj, out: list[dict], seen: set[str]) -> None:
"""Walk the response collecting videoRenderer items in document (relevance) order. Only
`videoRenderer` is taken, so channels/playlists/Shorts shelves (reelItemRenderer) are
naturally excluded."""
if isinstance(obj, dict):
vr = obj.get("videoRenderer")
if isinstance(vr, dict):
item = _video_item(vr)
if item and item["id"] not in seen:
seen.add(item["id"])
out.append(item)
for v in obj.values():
_collect_videos(v, out, seen)
elif isinstance(obj, list):
for v in obj:
_collect_videos(v, out, seen)
def _find_continuation(obj) -> str | None:
"""First continuationItemRenderer token (the 'load more' cursor), or None at the end."""
if isinstance(obj, dict):
cir = obj.get("continuationItemRenderer")
if isinstance(cir, dict):
tok = (
cir.get("continuationEndpoint", {})
.get("continuationCommand", {})
.get("token")
)
if tok:
return tok
for v in obj.values():
tok = _find_continuation(v)
if tok:
return tok
elif isinstance(obj, list):
for v in obj:
tok = _find_continuation(v)
if tok:
return tok
return None
def search(q: str, continuation: str | None = None) -> dict:
"""One page of zero-quota YouTube search results.
`continuation` is the InnerTube continuation token from a previous page (None for the first
page). Returns ``{"items": [...], "next_page_token": str | None}`` — the same shape as
YouTubeClient.search_videos so the route handles both sources identically."""
with httpx.Client(timeout=20.0, headers=_HEADERS) as client:
api_key, client_version = _get_config(client)
context = {
"client": {
"clientName": "WEB",
"clientVersion": client_version,
"hl": "en",
"gl": "US",
}
}
body: dict = {"context": context}
if continuation:
body["continuation"] = continuation
else:
body["query"] = q
body["params"] = _TYPE_VIDEO
try:
resp = client.post(_SEARCH_URL, params={"key": api_key}, json=body)
resp.raise_for_status()
data = resp.json()
except httpx.HTTPError as exc:
raise ScrapeError(f"InnerTube request failed: {exc}") from exc
except ValueError as exc: # non-JSON body (blocked / consent / captcha)
raise ScrapeError("InnerTube returned a non-JSON response") from exc
items: list[dict] = []
_collect_videos(data, items, set())
return {"items": items, "next_page_token": _find_continuation(data)}