diff --git a/backend/app/config.py b/backend/app/config.py index 933d036..78c1bd8 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -71,9 +71,13 @@ class Settings(BaseSettings): youtube_api_proxy: str = "" # Daily quota budget (units). Default leaves headroom under the 10,000/day free limit. quota_daily_budget: int = 9000 - # Per-user cap on live YouTube searches per day. search.list costs 100 units each, so the - # shared budget only affords ~80-90/day total — this stops one user draining it. Admin-tunable. + # Per-user cap on live YouTube searches per day. With the official API (search_source="api") + # each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with + # the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable. search_daily_limit_per_user: int = 70 + # Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list, + # 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks. + search_source: str = "scrape" # Recent-first backfill: how far back to fetch on the first pass per channel. backfill_recent_max_videos: int = 100 backfill_recent_max_days: int = 365 diff --git a/backend/app/quota.py b/backend/app/quota.py index 0737675..1e72e56 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -110,6 +110,15 @@ def can_spend(db: Session, units: int) -> bool: return remaining_today(db) >= units +def log_action(db: Session, user_id: int, action: str) -> None: + """Record a zero-cost action event so per-user daily caps still count it. + + `record_usage` only logs when units are actually charged; a scrape-based search spends no + quota yet must still be rate-limited per user, so it logs its event here directly.""" + db.add(QuotaEvent(user_id=user_id, action=action, units=0)) + db.commit() + + def actions_today(db: Session, user_id: int, action: str) -> int: """How many quota events of `action` this user has logged so far in the current Pacific day — for per-user, per-action daily caps (e.g. the live-search limit). Counts events, diff --git a/backend/app/routes/search.py b/backend/app/routes/search.py index bb29119..efba64f 100644 --- a/backend/app/routes/search.py +++ b/backend/app/routes/search.py @@ -1,10 +1,12 @@ """Live YouTube search. -Searches YouTube directly (search.list) and materialises the results into the shared catalog -so they render with the normal feed cards + in-app player and gain per-user state (watch / -save / resume). search.list costs **100 quota units per call**, so it's the most expensive -read by far: it's gated behind a per-user daily cap + a budget pre-check, and only real users -(never the shared demo account) may call it. +Searches YouTube directly and materialises the results into the shared catalog so they render +with the normal feed cards + in-app player and gain per-user state (watch / save / resume). The +source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own +endpoint, zero API quota — the default) or **api** (search.list, **100 quota units per page**). +Either way it's gated behind a per-user daily cap, the API source additionally behind a budget +pre-check, and only real users (never the shared demo account) may call it. The result ids are +enriched through the cheap shared videos.list path (1 unit / 50) regardless of source. Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never @@ -24,6 +26,7 @@ from app.models import Channel, User, Video, VideoState from app.routes.feed import _serialize, feed_columns from app.sync.subscriptions import apply_channel_details from app.sync.videos import _insert_stubs, apply_video_details, parse_dt +from app.youtube import search_scrape from app.youtube.client import YouTubeClient, YouTubeError from app.youtube.shorts import make_client, probe_is_short @@ -100,7 +103,13 @@ def search_youtube( if not term: raise HTTPException(status_code=400, detail="A search term is required.") - # Guard the shared quota BEFORE spending: per-user daily cap, then today's budget. + # Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page). + source = (sysconfig.get_str(db, "search_source") or "scrape").lower() + if source != "scrape": + source = "api" + + # Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard + # since the search itself spends no quota. daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user") used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH) if used >= daily_limit: @@ -108,7 +117,8 @@ def search_youtube( status_code=429, detail=f"You've reached today's YouTube search limit ({daily_limit}). Try again tomorrow.", ) - if not quota.can_spend(db, SEARCH_COST): + # The 100-unit budget pre-check only applies to the API source. + if source == "api" and not quota.can_spend(db, SEARCH_COST): raise HTTPException( status_code=429, detail="The shared YouTube quota for today is used up. Try again tomorrow.", @@ -116,9 +126,14 @@ def search_youtube( with YouTubeClient(db, user) as yt: try: - with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH): - page = yt.search_videos(term, page_token=cursor) - except YouTubeError as exc: + if source == "api": + with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH): + page = yt.search_videos(term, page_token=cursor) + else: + page = search_scrape.search(term, continuation=cursor) + # Zero-cost, but logged so the per-user daily cap above keeps counting it. + quota.log_action(db, user.id, quota.QuotaAction.VIDEOS_SEARCH) + except (YouTubeError, search_scrape.ScrapeError) as exc: raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}") # Drop currently-live/upcoming results (search can't filter them and we never ingest @@ -134,6 +149,7 @@ def search_youtube( "items": [], "next_cursor": page["next_page_token"], "has_more": bool(page["next_page_token"]), + "source": source, } # 1) Channels are the FK target — create stubs for unknown ones (flagged from_search), @@ -201,4 +217,5 @@ def search_youtube( "items": _serialize_ids(db, user, ordered), "next_cursor": page["next_page_token"], "has_more": bool(page["next_page_token"]), + "source": source, } diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index 1b45155..c6d8004 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -40,6 +40,8 @@ SPECS: tuple[ConfigSpec, ...] = ( ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000), ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000), ConfigSpec("search_daily_limit_per_user", "int", "quota", "search_daily_limit_per_user", min=0, max=100_000), + # Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page). + ConfigSpec("search_source", "str", "quota", "search_source"), # --- Backfill (recent-first first pass per channel) --- ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000), ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500), diff --git a/backend/app/youtube/search_scrape.py b/backend/app/youtube/search_scrape.py new file mode 100644 index 0000000..05a4f78 --- /dev/null +++ b/backend/app/youtube/search_scrape.py @@ -0,0 +1,184 @@ +"""Zero-quota YouTube search via InnerTube (the website's own youtubei/v1/search endpoint). + +Live YouTube search **without spending any YouTube Data API quota**: instead of search.list +(100 units per page) we POST to YouTube's internal InnerTube endpoint — the same one the website +calls — and read the result list straight out of the JSON. The returned video ids then go +through the normal enrich path (videos.list, 1 unit / 50), exactly as the official search does, +so the search itself is effectively free and only the (cheap, shared) enrichment is charged. + +The SOCS=CAI consent-skip cookie is used like the Shorts probe so a server-side request isn't +bounced to consent.youtube.com. This deliberately scrapes an undocumented endpoint: if YouTube +changes the JSON shape or blocks the host, an admin can flip ``search_source`` back to ``api``. + +The returned page mirrors :meth:`YouTubeClient.search_videos` exactly +(``{"items": [...], "next_page_token": str | None}``) so the route is source-agnostic — the +only difference is the cursor is an InnerTube *continuation token*, not a Data API pageToken. +``published_at`` is left ``None`` (the scrape only exposes a relative time like "3 years ago"); +the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"`` +and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path).""" +import re + +import httpx + +_SEARCH_URL = "https://www.youtube.com/youtubei/v1/search" +_RESULTS_URL = "https://www.youtube.com/results" +# Public WEB InnerTube key + a recent client version. These are also embedded in every results +# page; we fetch the live pair once and cache it, falling back to these constants. +_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" +_FALLBACK_CLIENT_VERSION = "2.20240101.00.00" +# sp / params value selecting the "type: video" search filter (base64 protobuf EgIQAQ==). +_TYPE_VIDEO = "EgIQAQ==" + +_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + # SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is + # redirected to consent.youtube.com). + "Cookie": "SOCS=CAI", + "Accept-Language": "en-US,en;q=0.9", +} + +_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"') +_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"') + +# Cached (api_key, client_version) extracted from a results page; refreshed on demand. +_config: tuple[str, str] | None = None + + +class ScrapeError(Exception): + """A scrape-based search could not be completed (network, blocked, or unexpected shape).""" + + +def _runs_text(node: dict | None) -> str | None: + if not node: + return None + if node.get("simpleText"): + return node["simpleText"] + runs = node.get("runs") or [] + if runs: + return "".join(r.get("text", "") for r in runs) + return None + + +def _get_config(client: httpx.Client) -> tuple[str, str]: + """The current InnerTube (api_key, client_version), cached after the first fetch.""" + global _config + if _config is not None: + return _config + try: + resp = client.get(_RESULTS_URL, params={"search_query": "music"}) + html = resp.text + key = _KEY_RE.search(html) + ver = _VER_RE.search(html) + _config = ( + key.group(1) if key else _FALLBACK_KEY, + ver.group(1) if ver else _FALLBACK_CLIENT_VERSION, + ) + except httpx.HTTPError: + _config = (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION) + return _config + + +def _video_item(vr: dict) -> dict | None: + """Flatten a YouTube `videoRenderer` into the same shape as YouTubeClient.search_videos.""" + vid = vr.get("videoId") + if not vid: + return None + owner = vr.get("ownerText") or vr.get("longBylineText") or {} + runs = owner.get("runs") or [] + channel_id = channel_title = None + if runs: + channel_title = runs[0].get("text") + channel_id = ( + runs[0] + .get("navigationEndpoint", {}) + .get("browseEndpoint", {}) + .get("browseId") + ) + thumbs = (vr.get("thumbnail") or {}).get("thumbnails") or [] + return { + "id": vid, + "channel_id": channel_id, + "channel_title": channel_title, + "title": _runs_text(vr.get("title")), + "published_at": None, # scrape exposes only a relative time; enrich fills the real date + "thumbnail_url": thumbs[-1]["url"] if thumbs else None, + "live_broadcast": "none", # live/upcoming filtered post-enrich via live_status + } + + +def _collect_videos(obj, out: list[dict], seen: set[str]) -> None: + """Walk the response collecting videoRenderer items in document (relevance) order. Only + `videoRenderer` is taken, so channels/playlists/Shorts shelves (reelItemRenderer) are + naturally excluded.""" + if isinstance(obj, dict): + vr = obj.get("videoRenderer") + if isinstance(vr, dict): + item = _video_item(vr) + if item and item["id"] not in seen: + seen.add(item["id"]) + out.append(item) + for v in obj.values(): + _collect_videos(v, out, seen) + elif isinstance(obj, list): + for v in obj: + _collect_videos(v, out, seen) + + +def _find_continuation(obj) -> str | None: + """First continuationItemRenderer token (the 'load more' cursor), or None at the end.""" + if isinstance(obj, dict): + cir = obj.get("continuationItemRenderer") + if isinstance(cir, dict): + tok = ( + cir.get("continuationEndpoint", {}) + .get("continuationCommand", {}) + .get("token") + ) + if tok: + return tok + for v in obj.values(): + tok = _find_continuation(v) + if tok: + return tok + elif isinstance(obj, list): + for v in obj: + tok = _find_continuation(v) + if tok: + return tok + return None + + +def search(q: str, continuation: str | None = None) -> dict: + """One page of zero-quota YouTube search results. + + `continuation` is the InnerTube continuation token from a previous page (None for the first + page). Returns ``{"items": [...], "next_page_token": str | None}`` — the same shape as + YouTubeClient.search_videos so the route handles both sources identically.""" + with httpx.Client(timeout=20.0, headers=_HEADERS) as client: + api_key, client_version = _get_config(client) + context = { + "client": { + "clientName": "WEB", + "clientVersion": client_version, + "hl": "en", + "gl": "US", + } + } + body: dict = {"context": context} + if continuation: + body["continuation"] = continuation + else: + body["query"] = q + body["params"] = _TYPE_VIDEO + try: + resp = client.post(_SEARCH_URL, params={"key": api_key}, json=body) + resp.raise_for_status() + data = resp.json() + except httpx.HTTPError as exc: + raise ScrapeError(f"InnerTube request failed: {exc}") from exc + except ValueError as exc: # non-JSON body (blocked / consent / captcha) + raise ScrapeError("InnerTube returned a non-JSON response") from exc + + items: list[dict] = [] + _collect_videos(data, items, set()) + return {"items": items, "next_page_token": _find_continuation(data)} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 36f4f8a..b777c3a 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -124,7 +124,22 @@ export default function App() { const [page, setPageState] = useState(loadInitialPage); // Live YouTube search term (null = normal feed). Ephemeral: not persisted and not in the URL, // so a reload returns to the normal feed (a search spends quota, so we never auto-replay it). + // The search is a feed SUB-VIEW that owns a browser-history entry (history.state._yt): the + // popstate handler below derives ytSearch from it, so Back steps player → search → feed in + // that order (a player opened over the results pops first, then the search, then the feed) — + // instead of the search vanishing because it had no history entry of its own. const [ytSearch, setYtSearch] = useState(null); + const enterYtSearch = useCallback((q: string) => { + setYtSearch(q); + const st = window.history.state || {}; + if (st._yt) window.history.replaceState({ ...st, _yt: q }, ""); // refine current search + else window.history.pushState({ ...st, sfPage: "feed", _yt: q }, ""); // new sub-view entry + }, []); + const exitYtSearch = useCallback(() => { + // Pop our search entry (so Forward still works); fall back to a plain clear if we don't own one. + if (window.history.state?._yt) window.history.back(); + else setYtSearch(null); + }, []); const [wizardOpen, setWizardOpen] = useState(false); // Channel status chip, persisted so a reload (F5) keeps it; clamp a stale value at render. const [channelFilterRaw, setChannelFilter] = usePersistedState(LS.channelFilter, "all"); @@ -230,10 +245,10 @@ export default function App() { // from history.state on popstate. (stripUrlParams preserves history.state, so this stamp // survives a later query-string strip.) useEffect(() => { - window.history.replaceState( - { ...window.history.state, sfPage: page }, - "" - ); + // Drop any stale _yt from a prior session (a reload starts on the normal feed; ytSearch + // begins null), so the first Back doesn't resurrect a search we're no longer showing. + const { _yt: _staleYt, ...rest } = window.history.state || {}; + window.history.replaceState({ ...rest, sfPage: page }, ""); function onPop(e: PopStateEvent) { const p = (e.state?.sfPage as Page) ?? "feed"; // Guard a Back step that leaves Settings with unsaved changes: re-assert the Settings @@ -255,6 +270,9 @@ export default function App() { } setPageState(p); localStorage.setItem(PAGE_KEY, p); + // The YouTube-search sub-view rides in history.state._yt, so Back/Forward restores or + // clears it to match the entry we landed on (and a player popped first leaves it intact). + setYtSearch((e.state?._yt as string) ?? null); } window.addEventListener("popstate", onPop); return () => window.removeEventListener("popstate", onPop); @@ -505,7 +523,7 @@ export default function App() { filters={filters} setFilters={setFilters} page={page} - onYtSearch={setYtSearch} + onYtSearch={enterYtSearch} onGoToFullHistory={() => { setChannelFilter("needs_full"); setChannelsView("subscribed"); // the status filter applies to the subscriptions tab @@ -579,7 +597,8 @@ export default function App() { isDemo={meQuery.data!.is_demo} onOpenWizard={() => setWizardOpen(true)} ytSearch={ytSearch} - setYtSearch={setYtSearch} + onYtSearch={enterYtSearch} + onExitYtSearch={exitYtSearch} /> )} diff --git a/frontend/src/components/Feed.tsx b/frontend/src/components/Feed.tsx index 1fbadee..0223735 100644 --- a/frontend/src/components/Feed.tsx +++ b/frontend/src/components/Feed.tsx @@ -72,7 +72,8 @@ export default function Feed({ isDemo = false, onOpenWizard, ytSearch, - setYtSearch, + onYtSearch, + onExitYtSearch, }: { filters: FeedFilters; setFilters: (f: FeedFilters) => void; @@ -80,10 +81,13 @@ export default function Feed({ canRead: boolean; isDemo?: boolean; onOpenWizard: () => void; - // Live YouTube search: the active search term (null = normal feed), and its setter so the - // empty-state CTA / "back to feed" can toggle it. Search affordances are hidden for demo. + // Live YouTube search: the active search term (null = normal feed). Entering a search and + // leaving it both step browser history (the search is a feed sub-view with its own history + // entry), so the empty-state CTA enters via onYtSearch and "back to feed" pops via + // onExitYtSearch. Search affordances are hidden for demo. ytSearch: string | null; - setYtSearch: (q: string | null) => void; + onYtSearch: (q: string) => void; + onExitYtSearch: () => void; }) { const { t } = useTranslation(); const [overrides, setOverrides] = useState>({}); @@ -116,9 +120,10 @@ export default function Feed({ placeholderData: keepPreviousData, }); - // Live YouTube search results. Each page spends 100 quota units, so we never auto-paginate - // on scroll — the user pulls more with an explicit button. retry:false so a 429 (quota/limit) - // surfaces at once for inline display. staleTime keeps a revisited search from re-spending. + // Live YouTube search results. We never auto-paginate on scroll — the user pulls more with an + // explicit button (each api-source page spends 100 quota units; scrape-source pages are free). + // retry:false so a 429 (quota/limit) surfaces at once for inline display. staleTime keeps a + // revisited search from re-spending. const ytQuery = useInfiniteQuery({ queryKey: ["yt-search", ytSearch], queryFn: ({ pageParam }) => api.searchYoutube(ytSearch as string, pageParam as string | null), @@ -279,6 +284,9 @@ export default function Feed({ : ytQuery.isError ? t("feed.yt.error") : null; + // The backend reports which source served the search; scrape spends no search quota, so we + // drop the quota warning + "uses quota" wording in that mode. + const zeroQuota = ytQuery.data?.pages?.[0]?.source === "scrape"; return (
@@ -288,7 +296,7 @@ export default function Feed({ // The search just ingested new catalog videos, so the normal feed (which was // disabled and still holds its pre-search cache) is stale. Drop those caches so // it re-fetches fresh on return instead of showing the old (often empty) result. - setYtSearch(null); + onExitYtSearch(); qc.removeQueries({ queryKey: ["feed"] }); qc.removeQueries({ queryKey: ["feed-count"] }); qc.removeQueries({ queryKey: ["facets"] }); @@ -303,7 +311,9 @@ export default function Feed({ {t("feed.yt.resultsFor", { query: ytSearch })} - {t("feed.yt.quotaNote")} + + {zeroQuota ? t("feed.yt.freeNote") : t("feed.yt.quotaNote")} +
{ytQuery.isLoading ? ( @@ -333,7 +343,11 @@ export default function Feed({ disabled={ytQuery.isFetchingNextPage} className="inline-flex items-center gap-2 px-4 py-2 rounded-xl border border-border bg-card text-sm font-medium hover:border-accent hover:text-accent disabled:opacity-50 transition" > - {ytQuery.isFetchingNextPage ? t("feed.loadingMore") : t("feed.yt.loadMore")} + {ytQuery.isFetchingNextPage + ? t("feed.loadingMore") + : zeroQuota + ? t("feed.yt.loadMoreFree") + : t("feed.yt.loadMore")} )} @@ -519,7 +533,7 @@ export default function Feed({

{t("feed.noMatches")}

{filters.q.trim() && !isDemo && (