peter/siftlode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(search): ephemeral results, count selector, blocklist, new-first ordering (backend)

Browse source 
- Live-search results stay ephemeral: the discovery-cleanup job now also reclaims un-kept
  via_search videos (no watch state / not playlisted / channel not subscribed) after a grace
  period (search_grace_days), and POST /api/search/clear discards a given result set 'as if
  never added' (drops the user's search-finds + deletes the now-orphaned, un-kept videos).
  Admin POST /api/admin/purge-discovery runs it on demand (grace 0).
- Count selector: GET /api/search/youtube gains a 'limit' — the free scrape source pages
  through continuations until that many results are gathered (no manual load-more); the API
  source stays one ≤50 page (cost).
- Per-user channel blocklist (migration 0034 blocked_channels): block/unblock/list endpoints;
  blocked channels' videos are dropped from live search before ingest and hidden from the feed
  / Library / explore / channel page; explore refuses a blocked channel.
- New-first ordering: results you already have (subscribed channel, watched/in-progress/saved/
  playlisted) sink below genuinely-new discoveries, preserving relevance within each group.
This commit is contained in:
npeter83 2026-07-01 00:42:32 +02:00
parent 77707a3cbe
commit f27f31b6db
10 changed files with 396 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

293
backend/app/routes/search.py
View file

@ -16,14 +16,24 @@ from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import and_, func, select, update
from sqlalchemy import and_, delete, func, or_, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session, aliased
from app import quota, sysconfig
from app.auth import require_human
from app.db import get_db
from app.models import Channel, SearchFind, User, Video, VideoState
from app.models import (
BlockedChannel,
Channel,
Playlist,
PlaylistItem,
SearchFind,
Subscription,
User,
Video,
VideoState,
)
from app.routes.feed import _serialize, feed_columns
from app.sync.subscriptions import apply_channel_details
from app.sync.videos import _insert_stubs, apply_video_details, parse_dt
@ -91,18 +101,119 @@ def _serialize_ids(db: Session, user: User, ids: list[str]) -> list[dict]:
return [by_id[i] for i in ids if i in by_id]
def _ingest_candidates(
db: Session, yt: YouTubeClient, user: User, candidates: list[dict]
) -> list[str]:
"""Materialise one page of search candidates into the catalog (channel stubs + enrich,
video stubs flagged via_search, enrich, Shorts classification) and return the KEEPABLE
video ids — non-Short, non-live — in the input relevance order."""
ids = [it["id"] for it in candidates]
if not ids:
return []
# 1) Channels are the FK target — create stubs for unknown ones (flagged from_search),
# then enrich the new ones (channels.list, 1 unit). Stubs survive an enrich failure.
channel_ids = list({it["channel_id"] for it in candidates})
existing = set(db.scalars(select(Channel.id).where(Channel.id.in_(channel_ids))).all())
new_channel_ids = [c for c in channel_ids if c not in existing]
if new_channel_ids:
titles = {it["channel_id"]: it["channel_title"] for it in candidates}
for cid in new_channel_ids:
db.add(Channel(id=cid, title=titles.get(cid), from_search=True))
db.commit()
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER):
details = yt.get_channels(new_channel_ids)
apply_channel_details(db, details)
db.commit()
except YouTubeError:
db.rollback() # details can be backfilled later; the stubs remain
# 2) Video stubs, flagged via_search. Existing rows keep their flags (ON CONFLICT DO
# NOTHING), so an organically-synced video stays organic.
_insert_stubs(
db,
[
{
"id": it["id"],
"channel_id": it["channel_id"],
"title": it["title"],
"published_at": parse_dt(it["published_at"]),
"thumbnail_url": it["thumbnail_url"],
"via_search": True,
}
for it in candidates
],
)
# 3) Enrich (videos.list, 1 unit/50) so cards have duration/stats and we can judge live.
videos = db.scalars(select(Video).where(Video.id.in_(ids))).all()
try:
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_ENRICH):
detail_by_id = {it["id"]: it for it in yt.get_videos(ids)}
except YouTubeError:
detail_by_id = {}
now = datetime.now(timezone.utc)
for v in videos:
item = detail_by_id.get(v.id)
if item is None:
continue
apply_video_details(v, item)
v.enriched_at = now
db.commit()
# 4) Confirm/deny Shorts (no quota) so none enter via search.
_classify_shorts(db, videos)
keep = {v.id for v in videos if not v.is_short and v.live_status not in _LIVE_HIDDEN}
return [vid for vid in ids if vid in keep]
def _new_first(db: Session, user: User, ids: list[str]) -> list[str]:
"""Reorder results so genuinely-new discoveries come first and things the user already has
(a subscribed channel, or a video they've watched / in-progress / saved / playlisted) sink
to the back — keeping the relevance order within each group."""
if not ids:
return ids
sub_ch = select(Subscription.channel_id).where(Subscription.user_id == user.id)
has_state = (
select(VideoState.id)
.where(VideoState.video_id == Video.id, VideoState.user_id == user.id)
.exists()
)
in_playlist = (
select(PlaylistItem.id)
.join(Playlist, Playlist.id == PlaylistItem.playlist_id)
.where(PlaylistItem.video_id == Video.id, Playlist.user_id == user.id)
.exists()
)
have = set(
db.scalars(
select(Video.id).where(
Video.id.in_(ids),
or_(Video.channel_id.in_(sub_ch), has_state, in_playlist),
)
).all()
)
return [i for i in ids if i not in have] + [i for i in ids if i in have]
@router.get("/youtube")
def search_youtube(
q: str | None = None,
cursor: str | None = None,
limit: int = 20,
user: User = Depends(require_human),
db: Session = Depends(get_db),
) -> dict:
"""One page of live YouTube search results, materialised into the catalog. `cursor` is the
YouTube nextPageToken (each page is another 100-unit search)."""
"""Live YouTube search results materialised into the catalog. `limit` is how many results to
gather (the count selector); with the free scrape source we page through continuations until
that many are collected, so there's no manual "load more". The API source costs 100 units per
page, so it returns a single page (≤50) regardless. `cursor` lets the caller fetch further."""
term = (q or "").strip()
if not term:
raise HTTPException(status_code=400, detail="A search term is required.")
limit = max(1, min(limit, 100))
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
source = (sysconfig.get_str(db, "search_source") or "scrape").lower()
@ -125,108 +236,56 @@ def search_youtube(
detail="The shared YouTube quota for today is used up. Try again tomorrow.",
)
# Channels this user has blocked are dropped before ingestion — never shown, never stored.
blocked = set(
db.scalars(
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
).all()
)
collected: list[str] = []
next_cursor = cursor
MAX_PAGES = 12 # scrape safety bound
with YouTubeClient(db, user) as yt:
try:
if source == "api":
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
page = yt.search_videos(term, page_token=cursor)
else:
page = search_scrape.search(term, continuation=cursor)
# Zero-cost, but logged so the per-user daily cap above keeps counting it.
quota.log_action(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
except (YouTubeError, search_scrape.ScrapeError) as exc:
raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}")
# Drop currently-live/upcoming results (search can't filter them and we never ingest
# them this way) and anything missing a channel id.
candidates = [
it
for it in page["items"]
if it["live_broadcast"] == "none" and it["channel_id"]
]
ids = [it["id"] for it in candidates]
if not ids:
return {
"items": [],
"next_cursor": page["next_page_token"],
"has_more": bool(page["next_page_token"]),
"source": source,
}
# 1) Channels are the FK target — create stubs for unknown ones (flagged from_search),
# then enrich the new ones (channels.list, 1 unit). Stubs survive an enrich failure.
channel_ids = list({it["channel_id"] for it in candidates})
existing = set(
db.scalars(select(Channel.id).where(Channel.id.in_(channel_ids))).all()
)
new_channel_ids = [c for c in channel_ids if c not in existing]
if new_channel_ids:
titles = {it["channel_id"]: it["channel_title"] for it in candidates}
for cid in new_channel_ids:
db.add(Channel(id=cid, title=titles.get(cid), from_search=True))
db.commit()
for _ in range(MAX_PAGES):
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER):
details = yt.get_channels(new_channel_ids)
apply_channel_details(db, details)
db.commit()
except YouTubeError:
db.rollback() # details can be backfilled later; the stubs remain
if source == "api":
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_SEARCH):
page = yt.search_videos(term, page_token=next_cursor)
else:
page = search_scrape.search(term, continuation=next_cursor)
# Zero-cost, but logged so the per-user daily cap above keeps counting it.
quota.log_action(db, user.id, quota.QuotaAction.VIDEOS_SEARCH)
except (YouTubeError, search_scrape.ScrapeError) as exc:
if collected:
break # keep what we already gathered
raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}")
# 2) Video stubs, flagged via_search. Existing rows keep their flags (ON CONFLICT DO
# NOTHING), so an organically-synced video stays organic.
_insert_stubs(
db,
[
{
"id": it["id"],
"channel_id": it["channel_id"],
"title": it["title"],
"published_at": parse_dt(it["published_at"]),
"thumbnail_url": it["thumbnail_url"],
"via_search": True,
}
for it in candidates
],
)
# Drop currently-live/upcoming, channel-less, and blocked-channel results.
candidates = [
it
for it in page["items"]
if it["live_broadcast"] == "none"
and it["channel_id"]
and it["channel_id"] not in blocked
]
collected.extend(_ingest_candidates(db, yt, user, candidates))
next_cursor = page["next_page_token"]
# The API source pages cost 100 units each — never auto-page; one page per request.
if source == "api" or len(collected) >= limit or not next_cursor:
break
# 3) Enrich (videos.list, 1 unit/50) so cards have duration/stats and we can judge live.
videos = db.scalars(select(Video).where(Video.id.in_(ids))).all()
try:
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_ENRICH):
detail_by_id = {it["id"]: it for it in yt.get_videos(ids)}
except YouTubeError:
detail_by_id = {}
now = datetime.now(timezone.utc)
for v in videos:
item = detail_by_id.get(v.id)
if item is None:
continue
apply_video_details(v, item)
v.enriched_at = now
db.commit()
ordered = collected[:limit]
# 4) Confirm/deny Shorts (no quota) so none enter via search.
_classify_shorts(db, videos)
# 5) Exclude Shorts and still-live results; keep the search's relevance order.
keep = {
v.id for v in videos if not v.is_short and v.live_status not in _LIVE_HIDDEN
}
ordered = [vid for vid in ids if vid in keep]
# 6) Record these as THIS user's search finds so they surface in their own Mine feed
# (and the Mine Source filter), independent of the global via_search flag. Idempotent.
if ordered:
# Record these as THIS user's search finds (Mine feed / Source filter), and fold the
# query into each result's search_terms so local full-text search inherits YouTube's
# relevance. Idempotent; the generated search_vector updates automatically.
db.execute(
pg_insert(SearchFind)
.values([{"user_id": user.id, "video_id": vid} for vid in ordered])
.on_conflict_do_nothing(index_elements=["user_id", "video_id"])
)
# 7) Fold the query into each result's search_terms (shared across users) so local
# full-text search inherits YouTube's relevance — a video YT returned for this query
# becomes findable by it even when its title doesn't contain the words. Skip rows
# that already include the term; the generated search_vector updates automatically.
db.execute(
update(Video)
.where(Video.id.in_(ordered))
@ -239,9 +298,49 @@ def search_youtube(
)
db.commit()
ordered = _new_first(db, user, ordered)
return {
"items": _serialize_ids(db, user, ordered),
"next_cursor": page["next_page_token"],
"has_more": bool(page["next_page_token"]),
"next_cursor": next_cursor,
"has_more": bool(next_cursor),
"source": source,
}
@router.post("/clear")
def clear_search(
payload: dict,
user: User = Depends(require_human),
db: Session = Depends(get_db),
) -> dict:
"""Discard the given live-search results "as if never added": drop this user's search-finds
for them, then hard-delete the now-orphaned ones — a via_search video nobody KEPT (no watch
state, not playlisted, channel not subscribed) and that no other user still has as a search
find. Kept videos and organically-owned ones are left untouched."""
ids = [str(v) for v in (payload.get("video_ids") or [])]
if not ids:
return {"removed": 0}
db.execute(
delete(SearchFind).where(
SearchFind.user_id == user.id, SearchFind.video_id.in_(ids)
)
)
db.commit()
has_state = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
in_playlist = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
subscribed = (
select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
)
other_find = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists()
removed = db.execute(
delete(Video).where(
Video.id.in_(ids),
Video.via_search.is_(True),
~has_state,
~in_playlist,
~subscribed,
~other_find,
)
).rowcount
db.commit()
return {"removed": removed}