Merge: zero-quota live-search scrape source + search back-nav fix
feature/search-scrape-zero-quota: live YouTube search can use YouTube's internal InnerTube endpoint (admin-selectable search_source, default scrape) so it spends no API quota; and the live-search results view now owns a history entry so Back closes the player first and keeps the results.
This commit is contained in:
npeter83
commit
844eed46eb
14 changed files with 298 additions and 36 deletions
|
|
@ -71,9 +71,13 @@ class Settings(BaseSettings):
|
|||
youtube_api_proxy: str = ""
|
||||
# Daily quota budget (units). Default leaves headroom under the 10,000/day free limit.
|
||||
quota_daily_budget: int = 9000
|
||||
# Per-user cap on live YouTube searches per day. search.list costs 100 units each, so the
|
||||
# shared budget only affords ~80-90/day total — this stops one user draining it. Admin-tunable.
|
||||
# Per-user cap on live YouTube searches per day. With the official API (search_source="api")
|
||||
# each search.list costs 100 units, so the shared budget only affords ~80-90/day total; with
|
||||
# the zero-quota scrape source it's a pure anti-abuse rate limit. Admin-tunable.
|
||||
search_daily_limit_per_user: int = 70
|
||||
# Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
|
||||
# 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
|
||||
search_source: str = "scrape"
|
||||
# Recent-first backfill: how far back to fetch on the first pass per channel.
|
||||
backfill_recent_max_videos: int = 100
|
||||
backfill_recent_max_days: int = 365
|
||||
|
|
|
|||
|
|
@ -110,6 +110,15 @@ def can_spend(db: Session, units: int) -> bool:
|
|||
return remaining_today(db) >= units
|
||||
|
||||
|
||||
def log_action(db: Session, user_id: int, action: str) -> None:
|
||||
"""Record a zero-cost action event so per-user daily caps still count it.
|
||||
|
||||
`record_usage` only logs when units are actually charged; a scrape-based search spends no
|
||||
quota yet must still be rate-limited per user, so it logs its event here directly."""
|
||||
db.add(QuotaEvent(user_id=user_id, action=action, units=0))
|
||||
db.commit()
|
||||
|
||||
|
||||
def actions_today(db: Session, user_id: int, action: str) -> int:
|
||||
"""How many quota events of `action` this user has logged so far in the current Pacific
|
||||
day — for per-user, per-action daily caps (e.g. the live-search limit). Counts events,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
"""Live YouTube search.
|
||||
|
||||
Searches YouTube directly (search.list) and materialises the results into the shared catalog
|
||||
so they render with the normal feed cards + in-app player and gain per-user state (watch /
|
||||
save / resume). search.list costs **100 quota units per call**, so it's the most expensive
|
||||
read by far: it's gated behind a per-user daily cap + a budget pre-check, and only real users
|
||||
(never the shared demo account) may call it.
|
||||
Searches YouTube directly and materialises the results into the shared catalog so they render
|
||||
with the normal feed cards + in-app player and gain per-user state (watch / save / resume). The
|
||||
source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own
|
||||
endpoint, zero API quota — the default) or **api** (search.list, **100 quota units per page**).
|
||||
Either way it's gated behind a per-user daily cap, the API source additionally behind a budget
|
||||
pre-check, and only real users (never the shared demo account) may call it. The result ids are
|
||||
enriched through the cheap shared videos.list path (1 unit / 50) regardless of source.
|
||||
|
||||
Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
|
||||
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
|
||||
|
|
@ -24,6 +26,7 @@ from app.models import Channel, User, Video, VideoState
|
|||
from app.routes.feed import _serialize, feed_columns
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.sync.videos import _insert_stubs, apply_video_details, parse_dt
|
||||
from app.youtube import search_scrape
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.youtube.shorts import make_client, probe_is_short
|
||||
|
||||
|
|
@ -100,7 +103,13 @@ def search_youtube(
|
|||
if not term:
|
||||
raise HTTPException(status_code=400, detail="A search term is required.")
|
||||
|
||||
# Guard the shared quota BEFORE spending: per-user daily cap, then today's budget.
|
||||
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
|
||||
source = (sysconfig.get_str(db, "search_source") or "scrape").lower()
|
||||
if source != "scrape":
|
||||
source = "api"
|
||||
|
||||
# Per-user daily cap applies to both sources (anti-abuse); with scrape it's the only guard
|
||||
# since the search itself spends no quota.
|
||||
daily_limit = sysconfig.get_int(db, "search_daily_limit_per_user")
|
||||
used = quota.actions_today(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
||||
if used >= daily_limit:
|
||||
|
|
@ -108,7 +117,8 @@ def search_youtube(
|
|||
status_code=429,
|
||||
detail=f"You've reached today's YouTube search limit ({daily_limit}). Try again tomorrow.",
|
||||
)
|
||||
if not quota.can_spend(db, SEARCH_COST):
|
||||
# The 100-unit budget pre-check only applies to the API source.
|
||||
if source == "api" and not quota.can_spend(db, SEARCH_COST):
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="The shared YouTube quota for today is used up. Try again tomorrow.",
|
||||
|
|
@ -116,9 +126,14 @@ def search_youtube(
|
|||
|
||||
with YouTubeClient(db, user) as yt:
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
|
||||
page = yt.search_videos(term, page_token=cursor)
|
||||
except YouTubeError as exc:
|
||||
if source == "api":
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
|
||||
page = yt.search_videos(term, page_token=cursor)
|
||||
else:
|
||||
page = search_scrape.search(term, continuation=cursor)
|
||||
# Zero-cost, but logged so the per-user daily cap above keeps counting it.
|
||||
quota.log_action(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
|
||||
except (YouTubeError, search_scrape.ScrapeError) as exc:
|
||||
raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}")
|
||||
|
||||
# Drop currently-live/upcoming results (search can't filter them and we never ingest
|
||||
|
|
@ -134,6 +149,7 @@ def search_youtube(
|
|||
"items": [],
|
||||
"next_cursor": page["next_page_token"],
|
||||
"has_more": bool(page["next_page_token"]),
|
||||
"source": source,
|
||||
}
|
||||
|
||||
# 1) Channels are the FK target — create stubs for unknown ones (flagged from_search),
|
||||
|
|
@ -201,4 +217,5 @@ def search_youtube(
|
|||
"items": _serialize_ids(db, user, ordered),
|
||||
"next_cursor": page["next_page_token"],
|
||||
"has_more": bool(page["next_page_token"]),
|
||||
"source": source,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ SPECS: tuple[ConfigSpec, ...] = (
|
|||
ConfigSpec("quota_daily_budget", "int", "quota", "quota_daily_budget", min=1, max=1_000_000),
|
||||
ConfigSpec("backfill_quota_reserve", "int", "quota", "backfill_quota_reserve", min=0, max=1_000_000),
|
||||
ConfigSpec("search_daily_limit_per_user", "int", "quota", "search_daily_limit_per_user", min=0, max=100_000),
|
||||
# Live-search source: "scrape" (InnerTube, zero API quota) or "api" (search.list, 100u/page).
|
||||
ConfigSpec("search_source", "str", "quota", "search_source"),
|
||||
# --- Backfill (recent-first first pass per channel) ---
|
||||
ConfigSpec("backfill_recent_max_videos", "int", "backfill", "backfill_recent_max_videos", min=1, max=100_000),
|
||||
ConfigSpec("backfill_recent_max_days", "int", "backfill", "backfill_recent_max_days", min=1, max=36_500),
|
||||
|
|
|
|||
184
backend/app/youtube/search_scrape.py
Normal file
184
backend/app/youtube/search_scrape.py
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
"""Zero-quota YouTube search via InnerTube (the website's own youtubei/v1/search endpoint).
|
||||
|
||||
Live YouTube search **without spending any YouTube Data API quota**: instead of search.list
|
||||
(100 units per page) we POST to YouTube's internal InnerTube endpoint — the same one the website
|
||||
calls — and read the result list straight out of the JSON. The returned video ids then go
|
||||
through the normal enrich path (videos.list, 1 unit / 50), exactly as the official search does,
|
||||
so the search itself is effectively free and only the (cheap, shared) enrichment is charged.
|
||||
|
||||
The SOCS=CAI consent-skip cookie is used like the Shorts probe so a server-side request isn't
|
||||
bounced to consent.youtube.com. This deliberately scrapes an undocumented endpoint: if YouTube
|
||||
changes the JSON shape or blocks the host, an admin can flip ``search_source`` back to ``api``.
|
||||
|
||||
The returned page mirrors :meth:`YouTubeClient.search_videos` exactly
|
||||
(``{"items": [...], "next_page_token": str | None}``) so the route is source-agnostic — the
|
||||
only difference is the cursor is an InnerTube *continuation token*, not a Data API pageToken.
|
||||
``published_at`` is left ``None`` (the scrape only exposes a relative time like "3 years ago");
|
||||
the real date is filled by the videos.list enrich. ``live_broadcast`` is reported as ``"none"``
|
||||
and live/upcoming filtering happens post-enrich via ``live_status`` (same as the API path)."""
|
||||
import re
|
||||
|
||||
import httpx
|
||||
|
||||
_SEARCH_URL = "https://www.youtube.com/youtubei/v1/search"
|
||||
_RESULTS_URL = "https://www.youtube.com/results"
|
||||
# Public WEB InnerTube key + a recent client version. These are also embedded in every results
|
||||
# page; we fetch the live pair once and cache it, falling back to these constants.
|
||||
_FALLBACK_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
|
||||
_FALLBACK_CLIENT_VERSION = "2.20240101.00.00"
|
||||
# sp / params value selecting the "type: video" search filter (base64 protobuf EgIQAQ==).
|
||||
_TYPE_VIDEO = "EgIQAQ=="
|
||||
|
||||
_HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
|
||||
# SOCS skips YouTube's cookie-consent interstitial (otherwise every server-side request is
|
||||
# redirected to consent.youtube.com).
|
||||
"Cookie": "SOCS=CAI",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
|
||||
_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
|
||||
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
|
||||
|
||||
# Cached (api_key, client_version) extracted from a results page; refreshed on demand.
|
||||
_config: tuple[str, str] | None = None
|
||||
|
||||
|
||||
class ScrapeError(Exception):
|
||||
"""A scrape-based search could not be completed (network, blocked, or unexpected shape)."""
|
||||
|
||||
|
||||
def _runs_text(node: dict | None) -> str | None:
|
||||
if not node:
|
||||
return None
|
||||
if node.get("simpleText"):
|
||||
return node["simpleText"]
|
||||
runs = node.get("runs") or []
|
||||
if runs:
|
||||
return "".join(r.get("text", "") for r in runs)
|
||||
return None
|
||||
|
||||
|
||||
def _get_config(client: httpx.Client) -> tuple[str, str]:
|
||||
"""The current InnerTube (api_key, client_version), cached after the first fetch."""
|
||||
global _config
|
||||
if _config is not None:
|
||||
return _config
|
||||
try:
|
||||
resp = client.get(_RESULTS_URL, params={"search_query": "music"})
|
||||
html = resp.text
|
||||
key = _KEY_RE.search(html)
|
||||
ver = _VER_RE.search(html)
|
||||
_config = (
|
||||
key.group(1) if key else _FALLBACK_KEY,
|
||||
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
_config = (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
|
||||
return _config
|
||||
|
||||
|
||||
def _video_item(vr: dict) -> dict | None:
|
||||
"""Flatten a YouTube `videoRenderer` into the same shape as YouTubeClient.search_videos."""
|
||||
vid = vr.get("videoId")
|
||||
if not vid:
|
||||
return None
|
||||
owner = vr.get("ownerText") or vr.get("longBylineText") or {}
|
||||
runs = owner.get("runs") or []
|
||||
channel_id = channel_title = None
|
||||
if runs:
|
||||
channel_title = runs[0].get("text")
|
||||
channel_id = (
|
||||
runs[0]
|
||||
.get("navigationEndpoint", {})
|
||||
.get("browseEndpoint", {})
|
||||
.get("browseId")
|
||||
)
|
||||
thumbs = (vr.get("thumbnail") or {}).get("thumbnails") or []
|
||||
return {
|
||||
"id": vid,
|
||||
"channel_id": channel_id,
|
||||
"channel_title": channel_title,
|
||||
"title": _runs_text(vr.get("title")),
|
||||
"published_at": None, # scrape exposes only a relative time; enrich fills the real date
|
||||
"thumbnail_url": thumbs[-1]["url"] if thumbs else None,
|
||||
"live_broadcast": "none", # live/upcoming filtered post-enrich via live_status
|
||||
}
|
||||
|
||||
|
||||
def _collect_videos(obj, out: list[dict], seen: set[str]) -> None:
|
||||
"""Walk the response collecting videoRenderer items in document (relevance) order. Only
|
||||
`videoRenderer` is taken, so channels/playlists/Shorts shelves (reelItemRenderer) are
|
||||
naturally excluded."""
|
||||
if isinstance(obj, dict):
|
||||
vr = obj.get("videoRenderer")
|
||||
if isinstance(vr, dict):
|
||||
item = _video_item(vr)
|
||||
if item and item["id"] not in seen:
|
||||
seen.add(item["id"])
|
||||
out.append(item)
|
||||
for v in obj.values():
|
||||
_collect_videos(v, out, seen)
|
||||
elif isinstance(obj, list):
|
||||
for v in obj:
|
||||
_collect_videos(v, out, seen)
|
||||
|
||||
|
||||
def _find_continuation(obj) -> str | None:
|
||||
"""First continuationItemRenderer token (the 'load more' cursor), or None at the end."""
|
||||
if isinstance(obj, dict):
|
||||
cir = obj.get("continuationItemRenderer")
|
||||
if isinstance(cir, dict):
|
||||
tok = (
|
||||
cir.get("continuationEndpoint", {})
|
||||
.get("continuationCommand", {})
|
||||
.get("token")
|
||||
)
|
||||
if tok:
|
||||
return tok
|
||||
for v in obj.values():
|
||||
tok = _find_continuation(v)
|
||||
if tok:
|
||||
return tok
|
||||
elif isinstance(obj, list):
|
||||
for v in obj:
|
||||
tok = _find_continuation(v)
|
||||
if tok:
|
||||
return tok
|
||||
return None
|
||||
|
||||
|
||||
def search(q: str, continuation: str | None = None) -> dict:
|
||||
"""One page of zero-quota YouTube search results.
|
||||
|
||||
`continuation` is the InnerTube continuation token from a previous page (None for the first
|
||||
page). Returns ``{"items": [...], "next_page_token": str | None}`` — the same shape as
|
||||
YouTubeClient.search_videos so the route handles both sources identically."""
|
||||
with httpx.Client(timeout=20.0, headers=_HEADERS) as client:
|
||||
api_key, client_version = _get_config(client)
|
||||
context = {
|
||||
"client": {
|
||||
"clientName": "WEB",
|
||||
"clientVersion": client_version,
|
||||
"hl": "en",
|
||||
"gl": "US",
|
||||
}
|
||||
}
|
||||
body: dict = {"context": context}
|
||||
if continuation:
|
||||
body["continuation"] = continuation
|
||||
else:
|
||||
body["query"] = q
|
||||
body["params"] = _TYPE_VIDEO
|
||||
try:
|
||||
resp = client.post(_SEARCH_URL, params={"key": api_key}, json=body)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except httpx.HTTPError as exc:
|
||||
raise ScrapeError(f"InnerTube request failed: {exc}") from exc
|
||||
except ValueError as exc: # non-JSON body (blocked / consent / captcha)
|
||||
raise ScrapeError("InnerTube returned a non-JSON response") from exc
|
||||
|
||||
items: list[dict] = []
|
||||
_collect_videos(data, items, set())
|
||||
return {"items": items, "next_page_token": _find_continuation(data)}
|
||||
|
|
@ -124,7 +124,22 @@ export default function App() {
|
|||
const [page, setPageState] = useState<Page>(loadInitialPage);
|
||||
// Live YouTube search term (null = normal feed). Ephemeral: not persisted and not in the URL,
|
||||
// so a reload returns to the normal feed (a search spends quota, so we never auto-replay it).
|
||||
// The search is a feed SUB-VIEW that owns a browser-history entry (history.state._yt): the
|
||||
// popstate handler below derives ytSearch from it, so Back steps player → search → feed in
|
||||
// that order (a player opened over the results pops first, then the search, then the feed) —
|
||||
// instead of the search vanishing because it had no history entry of its own.
|
||||
const [ytSearch, setYtSearch] = useState<string | null>(null);
|
||||
const enterYtSearch = useCallback((q: string) => {
|
||||
setYtSearch(q);
|
||||
const st = window.history.state || {};
|
||||
if (st._yt) window.history.replaceState({ ...st, _yt: q }, ""); // refine current search
|
||||
else window.history.pushState({ ...st, sfPage: "feed", _yt: q }, ""); // new sub-view entry
|
||||
}, []);
|
||||
const exitYtSearch = useCallback(() => {
|
||||
// Pop our search entry (so Forward still works); fall back to a plain clear if we don't own one.
|
||||
if (window.history.state?._yt) window.history.back();
|
||||
else setYtSearch(null);
|
||||
}, []);
|
||||
const [wizardOpen, setWizardOpen] = useState(false);
|
||||
// Channel status chip, persisted so a reload (F5) keeps it; clamp a stale value at render.
|
||||
const [channelFilterRaw, setChannelFilter] = usePersistedState(LS.channelFilter, "all");
|
||||
|
|
@ -230,10 +245,10 @@ export default function App() {
|
|||
// from history.state on popstate. (stripUrlParams preserves history.state, so this stamp
|
||||
// survives a later query-string strip.)
|
||||
useEffect(() => {
|
||||
window.history.replaceState(
|
||||
{ ...window.history.state, sfPage: page },
|
||||
""
|
||||
);
|
||||
// Drop any stale _yt from a prior session (a reload starts on the normal feed; ytSearch
|
||||
// begins null), so the first Back doesn't resurrect a search we're no longer showing.
|
||||
const { _yt: _staleYt, ...rest } = window.history.state || {};
|
||||
window.history.replaceState({ ...rest, sfPage: page }, "");
|
||||
function onPop(e: PopStateEvent) {
|
||||
const p = (e.state?.sfPage as Page) ?? "feed";
|
||||
// Guard a Back step that leaves Settings with unsaved changes: re-assert the Settings
|
||||
|
|
@ -255,6 +270,9 @@ export default function App() {
|
|||
}
|
||||
setPageState(p);
|
||||
localStorage.setItem(PAGE_KEY, p);
|
||||
// The YouTube-search sub-view rides in history.state._yt, so Back/Forward restores or
|
||||
// clears it to match the entry we landed on (and a player popped first leaves it intact).
|
||||
setYtSearch((e.state?._yt as string) ?? null);
|
||||
}
|
||||
window.addEventListener("popstate", onPop);
|
||||
return () => window.removeEventListener("popstate", onPop);
|
||||
|
|
@ -505,7 +523,7 @@ export default function App() {
|
|||
filters={filters}
|
||||
setFilters={setFilters}
|
||||
page={page}
|
||||
onYtSearch={setYtSearch}
|
||||
onYtSearch={enterYtSearch}
|
||||
onGoToFullHistory={() => {
|
||||
setChannelFilter("needs_full");
|
||||
setChannelsView("subscribed"); // the status filter applies to the subscriptions tab
|
||||
|
|
@ -579,7 +597,8 @@ export default function App() {
|
|||
isDemo={meQuery.data!.is_demo}
|
||||
onOpenWizard={() => setWizardOpen(true)}
|
||||
ytSearch={ytSearch}
|
||||
setYtSearch={setYtSearch}
|
||||
onYtSearch={enterYtSearch}
|
||||
onExitYtSearch={exitYtSearch}
|
||||
/>
|
||||
)}
|
||||
</main>
|
||||
|
|
|
|||
|
|
@ -72,7 +72,8 @@ export default function Feed({
|
|||
isDemo = false,
|
||||
onOpenWizard,
|
||||
ytSearch,
|
||||
setYtSearch,
|
||||
onYtSearch,
|
||||
onExitYtSearch,
|
||||
}: {
|
||||
filters: FeedFilters;
|
||||
setFilters: (f: FeedFilters) => void;
|
||||
|
|
@ -80,10 +81,13 @@ export default function Feed({
|
|||
canRead: boolean;
|
||||
isDemo?: boolean;
|
||||
onOpenWizard: () => void;
|
||||
// Live YouTube search: the active search term (null = normal feed), and its setter so the
|
||||
// empty-state CTA / "back to feed" can toggle it. Search affordances are hidden for demo.
|
||||
// Live YouTube search: the active search term (null = normal feed). Entering a search and
|
||||
// leaving it both step browser history (the search is a feed sub-view with its own history
|
||||
// entry), so the empty-state CTA enters via onYtSearch and "back to feed" pops via
|
||||
// onExitYtSearch. Search affordances are hidden for demo.
|
||||
ytSearch: string | null;
|
||||
setYtSearch: (q: string | null) => void;
|
||||
onYtSearch: (q: string) => void;
|
||||
onExitYtSearch: () => void;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const [overrides, setOverrides] = useState<Record<string, string>>({});
|
||||
|
|
@ -116,9 +120,10 @@ export default function Feed({
|
|||
placeholderData: keepPreviousData,
|
||||
});
|
||||
|
||||
// Live YouTube search results. Each page spends 100 quota units, so we never auto-paginate
|
||||
// on scroll — the user pulls more with an explicit button. retry:false so a 429 (quota/limit)
|
||||
// surfaces at once for inline display. staleTime keeps a revisited search from re-spending.
|
||||
// Live YouTube search results. We never auto-paginate on scroll — the user pulls more with an
|
||||
// explicit button (each api-source page spends 100 quota units; scrape-source pages are free).
|
||||
// retry:false so a 429 (quota/limit) surfaces at once for inline display. staleTime keeps a
|
||||
// revisited search from re-spending.
|
||||
const ytQuery = useInfiniteQuery({
|
||||
queryKey: ["yt-search", ytSearch],
|
||||
queryFn: ({ pageParam }) => api.searchYoutube(ytSearch as string, pageParam as string | null),
|
||||
|
|
@ -279,6 +284,9 @@ export default function Feed({
|
|||
: ytQuery.isError
|
||||
? t("feed.yt.error")
|
||||
: null;
|
||||
// The backend reports which source served the search; scrape spends no search quota, so we
|
||||
// drop the quota warning + "uses quota" wording in that mode.
|
||||
const zeroQuota = ytQuery.data?.pages?.[0]?.source === "scrape";
|
||||
|
||||
return (
|
||||
<div className="p-4">
|
||||
|
|
@ -288,7 +296,7 @@ export default function Feed({
|
|||
// The search just ingested new catalog videos, so the normal feed (which was
|
||||
// disabled and still holds its pre-search cache) is stale. Drop those caches so
|
||||
// it re-fetches fresh on return instead of showing the old (often empty) result.
|
||||
setYtSearch(null);
|
||||
onExitYtSearch();
|
||||
qc.removeQueries({ queryKey: ["feed"] });
|
||||
qc.removeQueries({ queryKey: ["feed-count"] });
|
||||
qc.removeQueries({ queryKey: ["facets"] });
|
||||
|
|
@ -303,7 +311,9 @@ export default function Feed({
|
|||
<span className="text-sm font-semibold truncate">
|
||||
{t("feed.yt.resultsFor", { query: ytSearch })}
|
||||
</span>
|
||||
<span className="text-xs text-muted">{t("feed.yt.quotaNote")}</span>
|
||||
<span className="text-xs text-muted">
|
||||
{zeroQuota ? t("feed.yt.freeNote") : t("feed.yt.quotaNote")}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{ytQuery.isLoading ? (
|
||||
|
|
@ -333,7 +343,11 @@ export default function Feed({
|
|||
disabled={ytQuery.isFetchingNextPage}
|
||||
className="inline-flex items-center gap-2 px-4 py-2 rounded-xl border border-border bg-card text-sm font-medium hover:border-accent hover:text-accent disabled:opacity-50 transition"
|
||||
>
|
||||
{ytQuery.isFetchingNextPage ? t("feed.loadingMore") : t("feed.yt.loadMore")}
|
||||
{ytQuery.isFetchingNextPage
|
||||
? t("feed.loadingMore")
|
||||
: zeroQuota
|
||||
? t("feed.yt.loadMoreFree")
|
||||
: t("feed.yt.loadMore")}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
|
@ -519,7 +533,7 @@ export default function Feed({
|
|||
<p>{t("feed.noMatches")}</p>
|
||||
{filters.q.trim() && !isDemo && (
|
||||
<button
|
||||
onClick={() => setYtSearch(filters.q.trim())}
|
||||
onClick={() => onYtSearch(filters.q.trim())}
|
||||
className="mt-4 inline-flex items-center gap-2 px-4 py-2 rounded-xl font-semibold bg-accent text-accent-fg hover:opacity-90 transition"
|
||||
>
|
||||
<Youtube className="w-4 h-4" />
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@
|
|||
"smtp_password": { "label": "SMTP-Passwort", "hint": "App-Passwort. Verschlüsselt gespeichert; nur schreibbar — wird nie wieder angezeigt." },
|
||||
"quota_daily_budget": { "label": "Tägliches Kontingentbudget", "hint": "YouTube-Data-API-Einheiten pro Tag (kostenloses Limit 10.000). Standard 9000 lässt Puffer." },
|
||||
"backfill_quota_reserve": { "label": "Nachlade-Kontingentreserve", "hint": "Reservierte Einheiten, damit geplantes Nachladen interaktive Syncs / Anreicherung nie aushungert." },
|
||||
"search_daily_limit_per_user": { "label": "Live-Suche — Tageslimit pro Nutzer", "hint": "Maximale Live-YouTube-Suchen pro Nutzer und Tag. Jede Suche kostet 100 Einheiten, das gemeinsame Budget reicht also für insgesamt nur ~80-90/Tag. 0 = Live-Suche deaktiviert." },
|
||||
"search_daily_limit_per_user": { "label": "Live-Suche — Tageslimit pro Nutzer", "hint": "Maximale Live-YouTube-Suchen pro Nutzer und Tag. Bei der API-Quelle kostet jede Suche 100 Einheiten (das gemeinsame Budget reicht also für insgesamt nur ~80-90/Tag); bei der Scrape-Quelle ist es ein reines Missbrauchslimit. 0 = Live-Suche deaktiviert." },
|
||||
"search_source": { "label": "Quelle der Live-Suche", "hint": "Woher die Ergebnisse der Live-YouTube-Suche stammen. \"scrape\" nutzt YouTubes internen Endpunkt und verbraucht kein API-Kontingent (empfohlen). \"api\" nutzt das offizielle search.list (100 Einheiten/Seite). Auf \"api\" umstellen, falls das Scraping ausfällt oder blockiert wird." },
|
||||
"backfill_recent_max_videos": { "label": "Aktuelles Nachladen — max. Videos", "hint": "Neueste Videos pro Kanal beim ersten Durchlauf." },
|
||||
"backfill_recent_max_days": { "label": "Aktuelles Nachladen — max. Alter (Tage)", "hint": "Wie weit der erste Durchlauf pro Kanal zurückreicht." },
|
||||
"shorts_probe_max_seconds": { "label": "Shorts-Prüfung — max. Dauer (s)", "hint": "Nur Videos bis zu dieser Länge werden als mögliche Shorts geprüft." },
|
||||
|
|
|
|||
|
|
@ -41,10 +41,12 @@
|
|||
"searchFor": "Auf YouTube suchen nach „{{query}}”",
|
||||
"resultsFor": "YouTube-Ergebnisse für „{{query}}”",
|
||||
"quotaNote": "Live-Ergebnisse — verbraucht gemeinsames API-Kontingent",
|
||||
"freeNote": "Live-Ergebnisse — kein Suchkontingent verbraucht",
|
||||
"back": "Zurück zum Feed",
|
||||
"searching": "Suche auf YouTube…",
|
||||
"noResults": "Keine YouTube-Videos gefunden.",
|
||||
"error": "YouTube-Suche fehlgeschlagen.",
|
||||
"loadMore": "Mehr laden (verbraucht Kontingent)"
|
||||
"loadMore": "Mehr laden (verbraucht Kontingent)",
|
||||
"loadMoreFree": "Mehr laden"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@
|
|||
"smtp_password": { "label": "SMTP password", "hint": "App password. Stored encrypted; write-only — it's never shown back." },
|
||||
"quota_daily_budget": { "label": "Daily quota budget", "hint": "Total YouTube Data API units to spend per day (free tier is 10,000). Default 9000 leaves headroom." },
|
||||
"backfill_quota_reserve": { "label": "Backfill quota reserve", "hint": "Units kept in reserve so scheduled backfill never starves interactive syncs / enrichment." },
|
||||
"search_daily_limit_per_user": { "label": "Live search — daily limit per user", "hint": "Max live YouTube searches per user per day. Each search costs 100 units, so the shared budget only affords ~80-90/day total. Set 0 to disable live search." },
|
||||
"search_daily_limit_per_user": { "label": "Live search — daily limit per user", "hint": "Max live YouTube searches per user per day. With the API source each search costs 100 units (so the shared budget affords only ~80-90/day total); with the scrape source it's a pure anti-abuse limit. Set 0 to disable live search." },
|
||||
"search_source": { "label": "Live search source", "hint": "Where live YouTube search results come from. \"scrape\" uses YouTube's internal endpoint and spends no API quota (recommended). \"api\" uses the official search.list (100 units/page). Switch to \"api\" if scraping ever breaks or is blocked." },
|
||||
"backfill_recent_max_videos": { "label": "Recent backfill — max videos", "hint": "Newest videos fetched per channel on the first pass." },
|
||||
"backfill_recent_max_days": { "label": "Recent backfill — max age (days)", "hint": "How far back the first pass reaches per channel." },
|
||||
"shorts_probe_max_seconds": { "label": "Shorts probe — max duration (s)", "hint": "Only videos at or below this length are probed as possible Shorts." },
|
||||
|
|
|
|||
|
|
@ -41,10 +41,12 @@
|
|||
"searchFor": "Search YouTube for “{{query}}”",
|
||||
"resultsFor": "YouTube results for “{{query}}”",
|
||||
"quotaNote": "Live results — uses shared API quota",
|
||||
"freeNote": "Live results — no search quota used",
|
||||
"back": "Back to feed",
|
||||
"searching": "Searching YouTube…",
|
||||
"noResults": "No YouTube videos found.",
|
||||
"error": "YouTube search failed.",
|
||||
"loadMore": "Load more (uses quota)"
|
||||
"loadMore": "Load more (uses quota)",
|
||||
"loadMoreFree": "Load more"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@
|
|||
"smtp_password": { "label": "SMTP-jelszó", "hint": "Alkalmazásjelszó. Titkosítva tárolva; csak írható — soha nem jelenik meg újra." },
|
||||
"quota_daily_budget": { "label": "Napi kvótakeret", "hint": "Naponta elkölthető YouTube Data API-egységek (az ingyenes keret 10 000). Az alap 9000 tartalékot hagy." },
|
||||
"backfill_quota_reserve": { "label": "Letöltési kvótatartalék", "hint": "Tartalékban tartott egységek, hogy az ütemezett letöltés ne éheztesse ki az interaktív szinkront / gazdagítást." },
|
||||
"search_daily_limit_per_user": { "label": "Élő keresés — napi limit / felhasználó", "hint": "Maximális élő YouTube-keresés felhasználónként naponta. Egy keresés 100 egységbe kerül, így a közös büdzséből összesen csak ~80-90 fér bele naponta. 0 = élő keresés kikapcsolva." },
|
||||
"search_daily_limit_per_user": { "label": "Élő keresés — napi limit / felhasználó", "hint": "Maximális élő YouTube-keresés felhasználónként naponta. Az API-forrásnál egy keresés 100 egységbe kerül (így a közös büdzséből összesen csak ~80-90 fér bele naponta); a scrape-forrásnál ez tiszta visszaélés-elleni limit. 0 = élő keresés kikapcsolva." },
|
||||
"search_source": { "label": "Élő keresés forrása", "hint": "Honnan jönnek az élő YouTube-keresés találatai. A „scrape” a YouTube belső végpontját használja és nem fogyaszt API-kvótát (ajánlott). Az „api” a hivatalos search.list-et (100 egység/oldal). Válts „api”-ra, ha a scrape valaha elromlik vagy letiltják." },
|
||||
"backfill_recent_max_videos": { "label": "Friss letöltés — max videó", "hint": "Csatornánként az első körben letöltött legújabb videók száma." },
|
||||
"backfill_recent_max_days": { "label": "Friss letöltés — max kor (nap)", "hint": "Az első kör csatornánként meddig nyúl vissza." },
|
||||
"shorts_probe_max_seconds": { "label": "Shorts-vizsgálat — max hossz (mp)", "hint": "Csak az ennél nem hosszabb videókat vizsgáljuk lehetséges Shortsként." },
|
||||
|
|
|
|||
|
|
@ -41,10 +41,12 @@
|
|||
"searchFor": "Keresés a YouTube-on: „{{query}}”",
|
||||
"resultsFor": "YouTube találatok erre: „{{query}}”",
|
||||
"quotaNote": "Élő találatok — a közös API-kvótát fogyasztja",
|
||||
"freeNote": "Élő találatok — nem fogyaszt keresési kvótát",
|
||||
"back": "Vissza a hírfolyamhoz",
|
||||
"searching": "Keresés a YouTube-on…",
|
||||
"noResults": "Nincs YouTube-találat.",
|
||||
"error": "A YouTube-keresés nem sikerült.",
|
||||
"loadMore": "Továbbiak betöltése (kvótát fogyaszt)"
|
||||
"loadMore": "Továbbiak betöltése (kvótát fogyaszt)",
|
||||
"loadMoreFree": "Továbbiak betöltése"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,6 +82,9 @@ export interface FeedResponse {
|
|||
// Opaque keyset cursor for the next page, or null when this is the last page.
|
||||
next_cursor: string | null;
|
||||
limit: number;
|
||||
// Only set by the live YouTube search: "scrape" (InnerTube, no API quota) or "api"
|
||||
// (search.list, 100 units/page) — lets the UI drop the quota warning in scrape mode.
|
||||
source?: "scrape" | "api";
|
||||
}
|
||||
|
||||
export interface Playlist {
|
||||
|
|
@ -565,7 +568,8 @@ export const api = {
|
|||
req(`/api/feed/count?${filterParams(f).toString()}`),
|
||||
// Live YouTube search: results are materialised into the catalog and returned as feed cards.
|
||||
// `quiet` so the YT-search panel can render quota/limit errors (incl. 429) inline instead of
|
||||
// popping the global modal. `cursor` is the YouTube nextPageToken (each page spends 100 units).
|
||||
// popping the global modal. `cursor` is the next-page token (an InnerTube continuation token
|
||||
// in scrape mode, or a Data API pageToken in api mode); the response's `source` says which.
|
||||
searchYoutube: (q: string, cursor: string | null): Promise<FeedResponse> => {
|
||||
const p = new URLSearchParams({ q });
|
||||
if (cursor) p.set("cursor", cursor);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue