feat(feed): per-user search finds in Mine scope + full-text relevance search

Two related search improvements:

1) Your own live-search results now belong to your Mine feed. A new per-user
   search_finds table (migration 0030) records each video you surface via your
   YouTube search (the route inserts them idempotently). The Mine feed becomes
   'your non-hidden subscriptions OR your search finds', and the Source filter
   now applies in Mine too: organic = subscriptions, search = your search finds,
   all = both (default stays organic, so the main feed is unchanged). The shared
   Library keeps using the global via_search flag.

2) Feed search ranks by relevance instead of a whole-phrase substring. A custom
   unaccent_simple text-search config + GIN index (migration 0031) back a
   YouTube-like fuzzy match: word-order-independent, multi-word AND, prefix on
   the word being typed, accent-insensitive. A new 'relevance' sort orders by
   ts_rank; the channel name still matches as a substring. The rank is scaled to
   an integer so the keyset cursor pages it exactly (a raw float4 breaks paging).
   _filtered_query returns the rank expr so only the feed list uses it.
This commit is contained in:
npeter83 2026-06-30 00:39:09 +02:00
parent a4abfa402f
commit 6cef826ecc
5 changed files with 208 additions and 22 deletions

View file

@ -0,0 +1,47 @@
"""per-user search finds
Revision ID: 0030_search_finds
Revises: 0029_unaccent_search
Create Date: 2026-06-30
Adds the `search_finds` table: a per-user record of which videos a user surfaced via their own
live YouTube search. This lets "videos I searched for" appear in that user's Mine feed (and the
Mine Source filter), independent of the global `videos.via_search` flag (which marks a video as
search-discovered by anyone, for the shared Library's Source filter).
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0030_search_finds"
down_revision: Union[str, None] = "0029_unaccent_search"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"search_finds",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("user_id", sa.Integer(), nullable=False),
sa.Column("video_id", sa.String(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["video_id"], ["videos.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),
)
op.create_index("ix_search_finds_user_id", "search_finds", ["user_id"])
op.create_index("ix_search_finds_video_id", "search_finds", ["video_id"])
def downgrade() -> None:
op.drop_index("ix_search_finds_video_id", table_name="search_finds")
op.drop_index("ix_search_finds_user_id", table_name="search_finds")
op.drop_table("search_finds")

View file

@ -0,0 +1,49 @@
"""full-text relevance search on video titles
Revision ID: 0031_title_fts
Revises: 0030_search_finds
Create Date: 2026-06-30
Adds a PostgreSQL full-text search index over video titles so the feed search box can rank by
relevance (a YouTube-like "fuzzy" search: word-order-independent, multi-word AND, prefix on the
word being typed) instead of a whole-phrase substring match.
A custom text-search configuration `unaccent_simple` = the `simple` config (no stemming, no
stopwords right for a multilingual HU/EN/DE catalog) plus the `unaccent` dictionary, so the
search stays accent-insensitive ("tiesto" matches "Tiësto"). A GIN expression index on
`to_tsvector('public.unaccent_simple', coalesce(title,''))` makes the @@ match fast; the feed
query uses the exact same expression so the planner picks the index. `unaccent` is already
enabled (migration 0029). The config-based 2-arg to_tsvector is IMMUTABLE, so it's index-safe.
"""
from typing import Sequence, Union
from alembic import op
revision: str = "0031_title_fts"
down_revision: Union[str, None] = "0030_search_finds"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute(
"CREATE TEXT SEARCH CONFIGURATION public.unaccent_simple (COPY = pg_catalog.simple)"
)
op.execute(
"""
ALTER TEXT SEARCH CONFIGURATION public.unaccent_simple
ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, word, hword, hword_part
WITH unaccent, simple
"""
)
op.execute(
"""
CREATE INDEX ix_videos_title_fts ON videos
USING gin (to_tsvector('public.unaccent_simple', coalesce(title, '')))
"""
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_videos_title_fts")
op.execute("DROP TEXT SEARCH CONFIGURATION IF EXISTS public.unaccent_simple")

View file

@ -344,6 +344,25 @@ class VideoState(Base, UpdatedAtMixin):
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class SearchFind(Base, TimestampMixin):
"""Per-user record of a video the user surfaced via their own live YouTube search.
Lets "videos I searched for" appear in that user's own (Mine) feed and the Mine Source
filter distinct from the GLOBAL `Video.via_search` provenance, which marks a video as
search-discovered by *anyone* and drives the shared Library's Source filter."""
__tablename__ = "search_finds"
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
video_id: Mapped[str] = mapped_column(
ForeignKey("videos.id", ondelete="CASCADE"), index=True
)
class ApiQuotaUsage(Base):
"""Tracks YouTube Data API units spent per Pacific-time day (the quota resets at
midnight Pacific). The whole app shares one daily budget."""

View file

@ -1,10 +1,11 @@
import base64
import binascii
import json
import re
from datetime import date, datetime, timedelta, timezone
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import Select, and_, false, func, or_, select
from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session, aliased
from sqlalchemy.orm.exc import StaleDataError
@ -17,6 +18,7 @@ from app.models import (
ChannelTag,
Playlist,
PlaylistItem,
SearchFind,
Subscription,
Tag,
User,
@ -153,14 +155,20 @@ def _filtered_query(
),
)
else:
# Only channels this user is subscribed to (and hasn't hidden).
query = query.join(
# "Mine": the user's own videos = their non-hidden subscriptions OR videos they
# surfaced via their own live YouTube search. Both are LEFT-joined so the Source
# filter below can include either or both, and the subscription row stays available
# for the priority sort.
query = query.outerjoin(
Subscription,
and_(
Subscription.channel_id == Video.channel_id,
Subscription.user_id == user.id,
Subscription.hidden.is_(False),
),
).outerjoin(
SearchFind,
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
)
query = query.outerjoin(
@ -172,16 +180,25 @@ def _filtered_query(
# either way they shouldn't show in any feed view meanwhile.
query = query.where(Video.unavailable_since.is_(None))
# Provenance filter for the Library: search-discovered videos are "noise" hidden by
# default ("organic"), can be mixed in ("all"), or shown exclusively ("search"). Only
# meaningful in "all" scope: "my" already restricts to subscribed channels, where a
# once-searched video is legitimately the user's.
# Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag
# (search-discovered by anyone). In "my" scope it's the user's OWN provenance:
# "organic" = your subscriptions, "search" = videos you found via your own search,
# "all" = both. Default "organic" keeps the Mine feed = subscriptions only.
if scope == "all":
if library_source == "organic":
query = query.where(Video.via_search.is_(False))
elif library_source == "search":
query = query.where(Video.via_search.is_(True))
# "all" → both organic and search-discovered videos
else:
subscribed = Subscription.channel_id.isnot(None)
searched = SearchFind.video_id.isnot(None)
if library_source == "search":
query = query.where(searched)
elif library_source == "all":
query = query.where(or_(subscribed, searched))
else: # "organic" (default): subscriptions only
query = query.where(subscribed)
if channel_id:
query = query.where(Video.channel_id == channel_id)
@ -204,9 +221,32 @@ def _filtered_query(
published_before, datetime.min.time(), tzinfo=timezone.utc
) + timedelta(days=1)
query = query.where(Video.published_at < end)
# Full-text relevance search on the title: YouTube-like "fuzzy" matching (word-order-
# independent, multi-word AND, prefix on the word being typed) + accent-insensitive via
# the unaccent_simple config. The channel name still matches as a substring so channel
# searches work. `rank_expr` (ts_rank) drives the optional "relevance" sort. The
# to_tsvector expression must match the GIN index (migration 0031) verbatim to be used.
rank_expr = None
if q:
# Accent-insensitive: unaccent() both sides so "tiesto" matches "Tiësto" (ILIKE alone
# is case- but not diacritic-insensitive). unaccent is enabled by migration 0029.
ts_str = _to_tsquery_str(q)
if ts_str:
tsq = func.to_tsquery("public.unaccent_simple", ts_str)
title_vec = func.to_tsvector(
"public.unaccent_simple", func.coalesce(Video.title, "")
)
query = query.where(
or_(
title_vec.op("@@")(tsq),
func.unaccent(Channel.title).ilike(func.unaccent(f"%{q}%")),
)
)
# Scale ts_rank (a float4) to an integer score so the keyset cursor compares it
# EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and
# breaks paging (the same page repeats). 1e6 granularity is ample; ties break on
# published_at then id.
rank_expr = cast(func.ts_rank(title_vec, tsq) * 1000000, BigInteger)
else:
# No usable tokens (e.g. only punctuation): fall back to a plain substring match.
like = func.unaccent(f"%{q}%")
query = query.where(
or_(
@ -271,7 +311,19 @@ def _filtered_query(
else: # all
query = query.where(status_expr != "hidden")
return query, status_expr
return query, status_expr, rank_expr
def _to_tsquery_str(q: str) -> str | None:
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
prefix-matched (`:*`) for as-you-type search e.g. "ti amo magyar" "ti & amo & magyar:*".
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
(the string is built from sanitised tokens, never the raw query)."""
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
if not tokens:
return None
tokens[-1] = tokens[-1] + ":*"
return " & ".join(tokens)
# Shared query parameters for /feed and /feed/count.
@ -320,7 +372,7 @@ def _feed_params(
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
# (OFFSET would shift and duplicate/skip rows).
def _sort_keys(sort: str, seed: int) -> list[dict]:
def _sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"}
table: dict[str, list[dict]] = {
"newest": [pub],
@ -341,6 +393,13 @@ def _sort_keys(sort: str, seed: int) -> list[dict]:
# the cursor stays valid across pages of one shuffled run.
"shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
}
# Relevance (search-result ranking) only exists when there's a query to rank against;
# without `rank_expr` it falls through to newest. Ties break by newest, then id.
if rank_expr is not None:
table["relevance"] = [
{"expr": rank_expr, "asc": False, "nullable": False, "kind": "num"},
pub,
]
return table.get(sort) or table["newest"]
@ -409,8 +468,8 @@ def get_feed(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
query, _status = _filtered_query(db, user, **params)
keys = _sort_keys(sort, seed)
query, _status, rank_expr = _filtered_query(db, user, **params)
keys = _sort_keys(sort, seed, rank_expr)
# Expose each key column on the row so we can build the next cursor from the last item.
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
@ -439,7 +498,7 @@ def get_feed_count(
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
query, _status = _filtered_query(db, user, **params)
query, _status, _rank = _filtered_query(db, user, **params)
total = db.scalar(select(func.count()).select_from(query.subquery()))
return {"count": total or 0}
@ -467,7 +526,7 @@ def get_facets(
# keeps them applied, so each remaining chip narrows to channels that ALSO have all
# already-selected topics — and tags that can't co-occur drop to zero (hidden).
conjunctive = category == "topic" and params.get("tag_mode") == "and"
base, _status = _filtered_query(
base, _status, _rank = _filtered_query(
db,
user,
**{**params, "exclude_tag_category": None if conjunctive else category},

View file

@ -17,12 +17,13 @@ from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import and_, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session, aliased
from app import quota, sysconfig
from app.auth import require_human
from app.db import get_db
from app.models import Channel, User, Video, VideoState
from app.models import Channel, SearchFind, User, Video, VideoState
from app.routes.feed import _serialize, feed_columns
from app.sync.subscriptions import apply_channel_details
from app.sync.videos import _insert_stubs, apply_video_details, parse_dt
@ -213,6 +214,17 @@ def search_youtube(
v.id for v in videos if not v.is_short and v.live_status not in _LIVE_HIDDEN
}
ordered = [vid for vid in ids if vid in keep]
# 6) Record these as THIS user's search finds so they surface in their own Mine feed
# (and the Mine Source filter), independent of the global via_search flag. Idempotent.
if ordered:
db.execute(
pg_insert(SearchFind)
.values([{"user_id": user.id, "video_id": vid} for vid in ordered])
.on_conflict_do_nothing(index_elements=["user_id", "video_id"])
)
db.commit()
return {
"items": _serialize_ids(db, user, ordered),
"next_cursor": page["next_page_token"],