feat(feed): per-user search finds in Mine scope + full-text relevance search

Two related search improvements:

1) Your own live-search results now belong to your Mine feed. A new per-user
   search_finds table (migration 0030) records each video you surface via your
   YouTube search (the route inserts them idempotently). The Mine feed becomes
   'your non-hidden subscriptions OR your search finds', and the Source filter
   now applies in Mine too: organic = subscriptions, search = your search finds,
   all = both (default stays organic, so the main feed is unchanged). The shared
   Library keeps using the global via_search flag.

2) Feed search ranks by relevance instead of a whole-phrase substring. A custom
   unaccent_simple text-search config + GIN index (migration 0031) back a
   YouTube-like fuzzy match: word-order-independent, multi-word AND, prefix on
   the word being typed, accent-insensitive. A new 'relevance' sort orders by
   ts_rank; the channel name still matches as a substring. The rank is scaled to
   an integer so the keyset cursor pages it exactly (a raw float4 breaks paging).
   _filtered_query returns the rank expr so only the feed list uses it.
This commit is contained in:
npeter83 2026-06-30 00:39:09 +02:00
parent a4abfa402f
commit 6cef826ecc
5 changed files with 208 additions and 22 deletions

View file

@ -0,0 +1,47 @@
"""per-user search finds
Revision ID: 0030_search_finds
Revises: 0029_unaccent_search
Create Date: 2026-06-30
Adds the `search_finds` table: a per-user record of which videos a user surfaced via their own
live YouTube search. This lets "videos I searched for" appear in that user's Mine feed (and the
Mine Source filter), independent of the global `videos.via_search` flag (which marks a video as
search-discovered by anyone, for the shared Library's Source filter).
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0030_search_finds"
down_revision: Union[str, None] = "0029_unaccent_search"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"search_finds",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("user_id", sa.Integer(), nullable=False),
sa.Column("video_id", sa.String(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["video_id"], ["videos.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),
)
op.create_index("ix_search_finds_user_id", "search_finds", ["user_id"])
op.create_index("ix_search_finds_video_id", "search_finds", ["video_id"])
def downgrade() -> None:
op.drop_index("ix_search_finds_video_id", table_name="search_finds")
op.drop_index("ix_search_finds_user_id", table_name="search_finds")
op.drop_table("search_finds")

View file

@ -0,0 +1,49 @@
"""full-text relevance search on video titles
Revision ID: 0031_title_fts
Revises: 0030_search_finds
Create Date: 2026-06-30
Adds a PostgreSQL full-text search index over video titles so the feed search box can rank by
relevance (a YouTube-like "fuzzy" search: word-order-independent, multi-word AND, prefix on the
word being typed) instead of a whole-phrase substring match.
A custom text-search configuration `unaccent_simple` = the `simple` config (no stemming, no
stopwords right for a multilingual HU/EN/DE catalog) plus the `unaccent` dictionary, so the
search stays accent-insensitive ("tiesto" matches "Tiësto"). A GIN expression index on
`to_tsvector('public.unaccent_simple', coalesce(title,''))` makes the @@ match fast; the feed
query uses the exact same expression so the planner picks the index. `unaccent` is already
enabled (migration 0029). The config-based 2-arg to_tsvector is IMMUTABLE, so it's index-safe.
"""
from typing import Sequence, Union
from alembic import op
revision: str = "0031_title_fts"
down_revision: Union[str, None] = "0030_search_finds"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute(
"CREATE TEXT SEARCH CONFIGURATION public.unaccent_simple (COPY = pg_catalog.simple)"
)
op.execute(
"""
ALTER TEXT SEARCH CONFIGURATION public.unaccent_simple
ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, word, hword, hword_part
WITH unaccent, simple
"""
)
op.execute(
"""
CREATE INDEX ix_videos_title_fts ON videos
USING gin (to_tsvector('public.unaccent_simple', coalesce(title, '')))
"""
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_videos_title_fts")
op.execute("DROP TEXT SEARCH CONFIGURATION IF EXISTS public.unaccent_simple")

View file

@ -344,6 +344,25 @@ class VideoState(Base, UpdatedAtMixin):
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class SearchFind(Base, TimestampMixin):
"""Per-user record of a video the user surfaced via their own live YouTube search.
Lets "videos I searched for" appear in that user's own (Mine) feed and the Mine Source
filter distinct from the GLOBAL `Video.via_search` provenance, which marks a video as
search-discovered by *anyone* and drives the shared Library's Source filter."""
__tablename__ = "search_finds"
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
video_id: Mapped[str] = mapped_column(
ForeignKey("videos.id", ondelete="CASCADE"), index=True
)
class ApiQuotaUsage(Base): class ApiQuotaUsage(Base):
"""Tracks YouTube Data API units spent per Pacific-time day (the quota resets at """Tracks YouTube Data API units spent per Pacific-time day (the quota resets at
midnight Pacific). The whole app shares one daily budget.""" midnight Pacific). The whole app shares one daily budget."""

View file

@ -1,10 +1,11 @@
import base64 import base64
import binascii import binascii
import json import json
import re
from datetime import date, datetime, timedelta, timezone from datetime import date, datetime, timedelta, timezone
from fastapi import APIRouter, Depends, HTTPException, Query from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import Select, and_, false, func, or_, select from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session, aliased from sqlalchemy.orm import Session, aliased
from sqlalchemy.orm.exc import StaleDataError from sqlalchemy.orm.exc import StaleDataError
@ -17,6 +18,7 @@ from app.models import (
ChannelTag, ChannelTag,
Playlist, Playlist,
PlaylistItem, PlaylistItem,
SearchFind,
Subscription, Subscription,
Tag, Tag,
User, User,
@ -153,14 +155,20 @@ def _filtered_query(
), ),
) )
else: else:
# Only channels this user is subscribed to (and hasn't hidden). # "Mine": the user's own videos = their non-hidden subscriptions OR videos they
query = query.join( # surfaced via their own live YouTube search. Both are LEFT-joined so the Source
# filter below can include either or both, and the subscription row stays available
# for the priority sort.
query = query.outerjoin(
Subscription, Subscription,
and_( and_(
Subscription.channel_id == Video.channel_id, Subscription.channel_id == Video.channel_id,
Subscription.user_id == user.id, Subscription.user_id == user.id,
Subscription.hidden.is_(False), Subscription.hidden.is_(False),
), ),
).outerjoin(
SearchFind,
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
) )
query = query.outerjoin( query = query.outerjoin(
@ -172,16 +180,25 @@ def _filtered_query(
# either way they shouldn't show in any feed view meanwhile. # either way they shouldn't show in any feed view meanwhile.
query = query.where(Video.unavailable_since.is_(None)) query = query.where(Video.unavailable_since.is_(None))
# Provenance filter for the Library: search-discovered videos are "noise" hidden by # Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag
# default ("organic"), can be mixed in ("all"), or shown exclusively ("search"). Only # (search-discovered by anyone). In "my" scope it's the user's OWN provenance:
# meaningful in "all" scope: "my" already restricts to subscribed channels, where a # "organic" = your subscriptions, "search" = videos you found via your own search,
# once-searched video is legitimately the user's. # "all" = both. Default "organic" keeps the Mine feed = subscriptions only.
if scope == "all": if scope == "all":
if library_source == "organic": if library_source == "organic":
query = query.where(Video.via_search.is_(False)) query = query.where(Video.via_search.is_(False))
elif library_source == "search": elif library_source == "search":
query = query.where(Video.via_search.is_(True)) query = query.where(Video.via_search.is_(True))
# "all" → both organic and search-discovered videos # "all" → both organic and search-discovered videos
else:
subscribed = Subscription.channel_id.isnot(None)
searched = SearchFind.video_id.isnot(None)
if library_source == "search":
query = query.where(searched)
elif library_source == "all":
query = query.where(or_(subscribed, searched))
else: # "organic" (default): subscriptions only
query = query.where(subscribed)
if channel_id: if channel_id:
query = query.where(Video.channel_id == channel_id) query = query.where(Video.channel_id == channel_id)
@ -204,9 +221,32 @@ def _filtered_query(
published_before, datetime.min.time(), tzinfo=timezone.utc published_before, datetime.min.time(), tzinfo=timezone.utc
) + timedelta(days=1) ) + timedelta(days=1)
query = query.where(Video.published_at < end) query = query.where(Video.published_at < end)
# Full-text relevance search on the title: YouTube-like "fuzzy" matching (word-order-
# independent, multi-word AND, prefix on the word being typed) + accent-insensitive via
# the unaccent_simple config. The channel name still matches as a substring so channel
# searches work. `rank_expr` (ts_rank) drives the optional "relevance" sort. The
# to_tsvector expression must match the GIN index (migration 0031) verbatim to be used.
rank_expr = None
if q: if q:
# Accent-insensitive: unaccent() both sides so "tiesto" matches "Tiësto" (ILIKE alone ts_str = _to_tsquery_str(q)
# is case- but not diacritic-insensitive). unaccent is enabled by migration 0029. if ts_str:
tsq = func.to_tsquery("public.unaccent_simple", ts_str)
title_vec = func.to_tsvector(
"public.unaccent_simple", func.coalesce(Video.title, "")
)
query = query.where(
or_(
title_vec.op("@@")(tsq),
func.unaccent(Channel.title).ilike(func.unaccent(f"%{q}%")),
)
)
# Scale ts_rank (a float4) to an integer score so the keyset cursor compares it
# EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and
# breaks paging (the same page repeats). 1e6 granularity is ample; ties break on
# published_at then id.
rank_expr = cast(func.ts_rank(title_vec, tsq) * 1000000, BigInteger)
else:
# No usable tokens (e.g. only punctuation): fall back to a plain substring match.
like = func.unaccent(f"%{q}%") like = func.unaccent(f"%{q}%")
query = query.where( query = query.where(
or_( or_(
@ -271,7 +311,19 @@ def _filtered_query(
else: # all else: # all
query = query.where(status_expr != "hidden") query = query.where(status_expr != "hidden")
return query, status_expr return query, status_expr, rank_expr
def _to_tsquery_str(q: str) -> str | None:
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
prefix-matched (`:*`) for as-you-type search e.g. "ti amo magyar" "ti & amo & magyar:*".
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
(the string is built from sanitised tokens, never the raw query)."""
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
if not tokens:
return None
tokens[-1] = tokens[-1] + ":*"
return " & ".join(tokens)
# Shared query parameters for /feed and /feed/count. # Shared query parameters for /feed and /feed/count.
@ -320,7 +372,7 @@ def _feed_params(
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows, # replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll # and the page boundary is stable even as the scheduler ingests new videos mid-scroll
# (OFFSET would shift and duplicate/skip rows). # (OFFSET would shift and duplicate/skip rows).
def _sort_keys(sort: str, seed: int) -> list[dict]: def _sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"} pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"}
table: dict[str, list[dict]] = { table: dict[str, list[dict]] = {
"newest": [pub], "newest": [pub],
@ -341,6 +393,13 @@ def _sort_keys(sort: str, seed: int) -> list[dict]:
# the cursor stays valid across pages of one shuffled run. # the cursor stays valid across pages of one shuffled run.
"shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}], "shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
} }
# Relevance (search-result ranking) only exists when there's a query to rank against;
# without `rank_expr` it falls through to newest. Ties break by newest, then id.
if rank_expr is not None:
table["relevance"] = [
{"expr": rank_expr, "asc": False, "nullable": False, "kind": "num"},
pub,
]
return table.get(sort) or table["newest"] return table.get(sort) or table["newest"]
@ -409,8 +468,8 @@ def get_feed(
user: User = Depends(current_user), user: User = Depends(current_user),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
query, _status = _filtered_query(db, user, **params) query, _status, rank_expr = _filtered_query(db, user, **params)
keys = _sort_keys(sort, seed) keys = _sort_keys(sort, seed, rank_expr)
# Expose each key column on the row so we can build the next cursor from the last item. # Expose each key column on the row so we can build the next cursor from the last item.
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)]) query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
@ -439,7 +498,7 @@ def get_feed_count(
user: User = Depends(current_user), user: User = Depends(current_user),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
query, _status = _filtered_query(db, user, **params) query, _status, _rank = _filtered_query(db, user, **params)
total = db.scalar(select(func.count()).select_from(query.subquery())) total = db.scalar(select(func.count()).select_from(query.subquery()))
return {"count": total or 0} return {"count": total or 0}
@ -467,7 +526,7 @@ def get_facets(
# keeps them applied, so each remaining chip narrows to channels that ALSO have all # keeps them applied, so each remaining chip narrows to channels that ALSO have all
# already-selected topics — and tags that can't co-occur drop to zero (hidden). # already-selected topics — and tags that can't co-occur drop to zero (hidden).
conjunctive = category == "topic" and params.get("tag_mode") == "and" conjunctive = category == "topic" and params.get("tag_mode") == "and"
base, _status = _filtered_query( base, _status, _rank = _filtered_query(
db, db,
user, user,
**{**params, "exclude_tag_category": None if conjunctive else category}, **{**params, "exclude_tag_category": None if conjunctive else category},

View file

@ -17,12 +17,13 @@ from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import and_, select from sqlalchemy import and_, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session, aliased from sqlalchemy.orm import Session, aliased
from app import quota, sysconfig from app import quota, sysconfig
from app.auth import require_human from app.auth import require_human
from app.db import get_db from app.db import get_db
from app.models import Channel, User, Video, VideoState from app.models import Channel, SearchFind, User, Video, VideoState
from app.routes.feed import _serialize, feed_columns from app.routes.feed import _serialize, feed_columns
from app.sync.subscriptions import apply_channel_details from app.sync.subscriptions import apply_channel_details
from app.sync.videos import _insert_stubs, apply_video_details, parse_dt from app.sync.videos import _insert_stubs, apply_video_details, parse_dt
@ -213,6 +214,17 @@ def search_youtube(
v.id for v in videos if not v.is_short and v.live_status not in _LIVE_HIDDEN v.id for v in videos if not v.is_short and v.live_status not in _LIVE_HIDDEN
} }
ordered = [vid for vid in ids if vid in keep] ordered = [vid for vid in ids if vid in keep]
# 6) Record these as THIS user's search finds so they surface in their own Mine feed
# (and the Mine Source filter), independent of the global via_search flag. Idempotent.
if ordered:
db.execute(
pg_insert(SearchFind)
.values([{"user_id": user.id, "video_id": vid} for vid in ordered])
.on_conflict_do_nothing(index_elements=["user_id", "video_id"])
)
db.commit()
return { return {
"items": _serialize_ids(db, user, ordered), "items": _serialize_ids(db, user, ordered),
"next_cursor": page["next_page_token"], "next_cursor": page["next_page_token"],