feat(feed): per-user search finds in Mine scope + full-text relevance search
Two related search improvements: 1) Your own live-search results now belong to your Mine feed. A new per-user search_finds table (migration 0030) records each video you surface via your YouTube search (the route inserts them idempotently). The Mine feed becomes 'your non-hidden subscriptions OR your search finds', and the Source filter now applies in Mine too: organic = subscriptions, search = your search finds, all = both (default stays organic, so the main feed is unchanged). The shared Library keeps using the global via_search flag. 2) Feed search ranks by relevance instead of a whole-phrase substring. A custom unaccent_simple text-search config + GIN index (migration 0031) back a YouTube-like fuzzy match: word-order-independent, multi-word AND, prefix on the word being typed, accent-insensitive. A new 'relevance' sort orders by ts_rank; the channel name still matches as a substring. The rank is scaled to an integer so the keyset cursor pages it exactly (a raw float4 breaks paging). _filtered_query returns the rank expr so only the feed list uses it.
This commit is contained in:
parent
a4abfa402f
commit
6cef826ecc
5 changed files with 208 additions and 22 deletions
47
backend/alembic/versions/0030_search_finds.py
Normal file
47
backend/alembic/versions/0030_search_finds.py
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
"""per-user search finds
|
||||
|
||||
Revision ID: 0030_search_finds
|
||||
Revises: 0029_unaccent_search
|
||||
Create Date: 2026-06-30
|
||||
|
||||
Adds the `search_finds` table: a per-user record of which videos a user surfaced via their own
|
||||
live YouTube search. This lets "videos I searched for" appear in that user's Mine feed (and the
|
||||
Mine Source filter), independent of the global `videos.via_search` flag (which marks a video as
|
||||
search-discovered by anyone, for the shared Library's Source filter).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0030_search_finds"
|
||||
down_revision: Union[str, None] = "0029_unaccent_search"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"search_finds",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("user_id", sa.Integer(), nullable=False),
|
||||
sa.Column("video_id", sa.String(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(["video_id"], ["videos.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),
|
||||
)
|
||||
op.create_index("ix_search_finds_user_id", "search_finds", ["user_id"])
|
||||
op.create_index("ix_search_finds_video_id", "search_finds", ["video_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_search_finds_video_id", table_name="search_finds")
|
||||
op.drop_index("ix_search_finds_user_id", table_name="search_finds")
|
||||
op.drop_table("search_finds")
|
||||
49
backend/alembic/versions/0031_title_fts.py
Normal file
49
backend/alembic/versions/0031_title_fts.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""full-text relevance search on video titles
|
||||
|
||||
Revision ID: 0031_title_fts
|
||||
Revises: 0030_search_finds
|
||||
Create Date: 2026-06-30
|
||||
|
||||
Adds a PostgreSQL full-text search index over video titles so the feed search box can rank by
|
||||
relevance (a YouTube-like "fuzzy" search: word-order-independent, multi-word AND, prefix on the
|
||||
word being typed) instead of a whole-phrase substring match.
|
||||
|
||||
A custom text-search configuration `unaccent_simple` = the `simple` config (no stemming, no
|
||||
stopwords — right for a multilingual HU/EN/DE catalog) plus the `unaccent` dictionary, so the
|
||||
search stays accent-insensitive ("tiesto" matches "Tiësto"). A GIN expression index on
|
||||
`to_tsvector('public.unaccent_simple', coalesce(title,''))` makes the @@ match fast; the feed
|
||||
query uses the exact same expression so the planner picks the index. `unaccent` is already
|
||||
enabled (migration 0029). The config-based 2-arg to_tsvector is IMMUTABLE, so it's index-safe.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0031_title_fts"
|
||||
down_revision: Union[str, None] = "0030_search_finds"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"CREATE TEXT SEARCH CONFIGURATION public.unaccent_simple (COPY = pg_catalog.simple)"
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TEXT SEARCH CONFIGURATION public.unaccent_simple
|
||||
ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, word, hword, hword_part
|
||||
WITH unaccent, simple
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE INDEX ix_videos_title_fts ON videos
|
||||
USING gin (to_tsvector('public.unaccent_simple', coalesce(title, '')))
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_videos_title_fts")
|
||||
op.execute("DROP TEXT SEARCH CONFIGURATION IF EXISTS public.unaccent_simple")
|
||||
|
|
@ -344,6 +344,25 @@ class VideoState(Base, UpdatedAtMixin):
|
|||
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class SearchFind(Base, TimestampMixin):
|
||||
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
||||
|
||||
Lets "videos I searched for" appear in that user's own (Mine) feed and the Mine Source
|
||||
filter — distinct from the GLOBAL `Video.via_search` provenance, which marks a video as
|
||||
search-discovered by *anyone* and drives the shared Library's Source filter."""
|
||||
|
||||
__tablename__ = "search_finds"
|
||||
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
video_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("videos.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
|
||||
class ApiQuotaUsage(Base):
|
||||
"""Tracks YouTube Data API units spent per Pacific-time day (the quota resets at
|
||||
midnight Pacific). The whole app shares one daily budget."""
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import re
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import Select, and_, false, func, or_, select
|
||||
from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
from sqlalchemy.orm.exc import StaleDataError
|
||||
|
|
@ -17,6 +18,7 @@ from app.models import (
|
|||
ChannelTag,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
SearchFind,
|
||||
Subscription,
|
||||
Tag,
|
||||
User,
|
||||
|
|
@ -153,14 +155,20 @@ def _filtered_query(
|
|||
),
|
||||
)
|
||||
else:
|
||||
# Only channels this user is subscribed to (and hasn't hidden).
|
||||
query = query.join(
|
||||
# "Mine": the user's own videos = their non-hidden subscriptions OR videos they
|
||||
# surfaced via their own live YouTube search. Both are LEFT-joined so the Source
|
||||
# filter below can include either or both, and the subscription row stays available
|
||||
# for the priority sort.
|
||||
query = query.outerjoin(
|
||||
Subscription,
|
||||
and_(
|
||||
Subscription.channel_id == Video.channel_id,
|
||||
Subscription.user_id == user.id,
|
||||
Subscription.hidden.is_(False),
|
||||
),
|
||||
).outerjoin(
|
||||
SearchFind,
|
||||
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
|
||||
)
|
||||
|
||||
query = query.outerjoin(
|
||||
|
|
@ -172,16 +180,25 @@ def _filtered_query(
|
|||
# either way they shouldn't show in any feed view meanwhile.
|
||||
query = query.where(Video.unavailable_since.is_(None))
|
||||
|
||||
# Provenance filter for the Library: search-discovered videos are "noise" hidden by
|
||||
# default ("organic"), can be mixed in ("all"), or shown exclusively ("search"). Only
|
||||
# meaningful in "all" scope: "my" already restricts to subscribed channels, where a
|
||||
# once-searched video is legitimately the user's.
|
||||
# Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag
|
||||
# (search-discovered by anyone). In "my" scope it's the user's OWN provenance:
|
||||
# "organic" = your subscriptions, "search" = videos you found via your own search,
|
||||
# "all" = both. Default "organic" keeps the Mine feed = subscriptions only.
|
||||
if scope == "all":
|
||||
if library_source == "organic":
|
||||
query = query.where(Video.via_search.is_(False))
|
||||
elif library_source == "search":
|
||||
query = query.where(Video.via_search.is_(True))
|
||||
# "all" → both organic and search-discovered videos
|
||||
else:
|
||||
subscribed = Subscription.channel_id.isnot(None)
|
||||
searched = SearchFind.video_id.isnot(None)
|
||||
if library_source == "search":
|
||||
query = query.where(searched)
|
||||
elif library_source == "all":
|
||||
query = query.where(or_(subscribed, searched))
|
||||
else: # "organic" (default): subscriptions only
|
||||
query = query.where(subscribed)
|
||||
|
||||
if channel_id:
|
||||
query = query.where(Video.channel_id == channel_id)
|
||||
|
|
@ -204,9 +221,32 @@ def _filtered_query(
|
|||
published_before, datetime.min.time(), tzinfo=timezone.utc
|
||||
) + timedelta(days=1)
|
||||
query = query.where(Video.published_at < end)
|
||||
# Full-text relevance search on the title: YouTube-like "fuzzy" matching (word-order-
|
||||
# independent, multi-word AND, prefix on the word being typed) + accent-insensitive via
|
||||
# the unaccent_simple config. The channel name still matches as a substring so channel
|
||||
# searches work. `rank_expr` (ts_rank) drives the optional "relevance" sort. The
|
||||
# to_tsvector expression must match the GIN index (migration 0031) verbatim to be used.
|
||||
rank_expr = None
|
||||
if q:
|
||||
# Accent-insensitive: unaccent() both sides so "tiesto" matches "Tiësto" (ILIKE alone
|
||||
# is case- but not diacritic-insensitive). unaccent is enabled by migration 0029.
|
||||
ts_str = _to_tsquery_str(q)
|
||||
if ts_str:
|
||||
tsq = func.to_tsquery("public.unaccent_simple", ts_str)
|
||||
title_vec = func.to_tsvector(
|
||||
"public.unaccent_simple", func.coalesce(Video.title, "")
|
||||
)
|
||||
query = query.where(
|
||||
or_(
|
||||
title_vec.op("@@")(tsq),
|
||||
func.unaccent(Channel.title).ilike(func.unaccent(f"%{q}%")),
|
||||
)
|
||||
)
|
||||
# Scale ts_rank (a float4) to an integer score so the keyset cursor compares it
|
||||
# EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and
|
||||
# breaks paging (the same page repeats). 1e6 granularity is ample; ties break on
|
||||
# published_at then id.
|
||||
rank_expr = cast(func.ts_rank(title_vec, tsq) * 1000000, BigInteger)
|
||||
else:
|
||||
# No usable tokens (e.g. only punctuation): fall back to a plain substring match.
|
||||
like = func.unaccent(f"%{q}%")
|
||||
query = query.where(
|
||||
or_(
|
||||
|
|
@ -271,7 +311,19 @@ def _filtered_query(
|
|||
else: # all
|
||||
query = query.where(status_expr != "hidden")
|
||||
|
||||
return query, status_expr
|
||||
return query, status_expr, rank_expr
|
||||
|
||||
|
||||
def _to_tsquery_str(q: str) -> str | None:
|
||||
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
|
||||
prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*".
|
||||
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
|
||||
(the string is built from sanitised tokens, never the raw query)."""
|
||||
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
|
||||
if not tokens:
|
||||
return None
|
||||
tokens[-1] = tokens[-1] + ":*"
|
||||
return " & ".join(tokens)
|
||||
|
||||
|
||||
# Shared query parameters for /feed and /feed/count.
|
||||
|
|
@ -320,7 +372,7 @@ def _feed_params(
|
|||
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
|
||||
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
||||
# (OFFSET would shift and duplicate/skip rows).
|
||||
def _sort_keys(sort: str, seed: int) -> list[dict]:
|
||||
def _sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||
pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"}
|
||||
table: dict[str, list[dict]] = {
|
||||
"newest": [pub],
|
||||
|
|
@ -341,6 +393,13 @@ def _sort_keys(sort: str, seed: int) -> list[dict]:
|
|||
# the cursor stays valid across pages of one shuffled run.
|
||||
"shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
|
||||
}
|
||||
# Relevance (search-result ranking) only exists when there's a query to rank against;
|
||||
# without `rank_expr` it falls through to newest. Ties break by newest, then id.
|
||||
if rank_expr is not None:
|
||||
table["relevance"] = [
|
||||
{"expr": rank_expr, "asc": False, "nullable": False, "kind": "num"},
|
||||
pub,
|
||||
]
|
||||
return table.get(sort) or table["newest"]
|
||||
|
||||
|
||||
|
|
@ -409,8 +468,8 @@ def get_feed(
|
|||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
query, _status = _filtered_query(db, user, **params)
|
||||
keys = _sort_keys(sort, seed)
|
||||
query, _status, rank_expr = _filtered_query(db, user, **params)
|
||||
keys = _sort_keys(sort, seed, rank_expr)
|
||||
|
||||
# Expose each key column on the row so we can build the next cursor from the last item.
|
||||
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
|
||||
|
|
@ -439,7 +498,7 @@ def get_feed_count(
|
|||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
query, _status = _filtered_query(db, user, **params)
|
||||
query, _status, _rank = _filtered_query(db, user, **params)
|
||||
total = db.scalar(select(func.count()).select_from(query.subquery()))
|
||||
return {"count": total or 0}
|
||||
|
||||
|
|
@ -467,7 +526,7 @@ def get_facets(
|
|||
# keeps them applied, so each remaining chip narrows to channels that ALSO have all
|
||||
# already-selected topics — and tags that can't co-occur drop to zero (hidden).
|
||||
conjunctive = category == "topic" and params.get("tag_mode") == "and"
|
||||
base, _status = _filtered_query(
|
||||
base, _status, _rank = _filtered_query(
|
||||
db,
|
||||
user,
|
||||
**{**params, "exclude_tag_category": None if conjunctive else category},
|
||||
|
|
|
|||
|
|
@ -17,12 +17,13 @@ from datetime import datetime, timezone
|
|||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy import and_, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app import quota, sysconfig
|
||||
from app.auth import require_human
|
||||
from app.db import get_db
|
||||
from app.models import Channel, User, Video, VideoState
|
||||
from app.models import Channel, SearchFind, User, Video, VideoState
|
||||
from app.routes.feed import _serialize, feed_columns
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.sync.videos import _insert_stubs, apply_video_details, parse_dt
|
||||
|
|
@ -213,6 +214,17 @@ def search_youtube(
|
|||
v.id for v in videos if not v.is_short and v.live_status not in _LIVE_HIDDEN
|
||||
}
|
||||
ordered = [vid for vid in ids if vid in keep]
|
||||
|
||||
# 6) Record these as THIS user's search finds so they surface in their own Mine feed
|
||||
# (and the Mine Source filter), independent of the global via_search flag. Idempotent.
|
||||
if ordered:
|
||||
db.execute(
|
||||
pg_insert(SearchFind)
|
||||
.values([{"user_id": user.id, "video_id": vid} for vid in ordered])
|
||||
.on_conflict_do_nothing(index_elements=["user_id", "video_id"])
|
||||
)
|
||||
db.commit()
|
||||
|
||||
return {
|
||||
"items": _serialize_ids(db, user, ordered),
|
||||
"next_cursor": page["next_page_token"],
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue