From 6cef826ecc3f9ec55d71d10ec4c4f310d30bb37e Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 30 Jun 2026 00:39:09 +0200 Subject: [PATCH 1/4] feat(feed): per-user search finds in Mine scope + full-text relevance search Two related search improvements: 1) Your own live-search results now belong to your Mine feed. A new per-user search_finds table (migration 0030) records each video you surface via your YouTube search (the route inserts them idempotently). The Mine feed becomes 'your non-hidden subscriptions OR your search finds', and the Source filter now applies in Mine too: organic = subscriptions, search = your search finds, all = both (default stays organic, so the main feed is unchanged). The shared Library keeps using the global via_search flag. 2) Feed search ranks by relevance instead of a whole-phrase substring. A custom unaccent_simple text-search config + GIN index (migration 0031) back a YouTube-like fuzzy match: word-order-independent, multi-word AND, prefix on the word being typed, accent-insensitive. A new 'relevance' sort orders by ts_rank; the channel name still matches as a substring. The rank is scaled to an integer so the keyset cursor pages it exactly (a raw float4 breaks paging). _filtered_query returns the rank expr so only the feed list uses it. --- backend/alembic/versions/0030_search_finds.py | 47 ++++++++ backend/alembic/versions/0031_title_fts.py | 49 +++++++++ backend/app/models.py | 19 ++++ backend/app/routes/feed.py | 101 ++++++++++++++---- backend/app/routes/search.py | 14 ++- 5 files changed, 208 insertions(+), 22 deletions(-) create mode 100644 backend/alembic/versions/0030_search_finds.py create mode 100644 backend/alembic/versions/0031_title_fts.py diff --git a/backend/alembic/versions/0030_search_finds.py b/backend/alembic/versions/0030_search_finds.py new file mode 100644 index 0000000..975027a --- /dev/null +++ b/backend/alembic/versions/0030_search_finds.py @@ -0,0 +1,47 @@ +"""per-user search finds + +Revision ID: 0030_search_finds +Revises: 0029_unaccent_search +Create Date: 2026-06-30 + +Adds the `search_finds` table: a per-user record of which videos a user surfaced via their own +live YouTube search. This lets "videos I searched for" appear in that user's Mine feed (and the +Mine Source filter), independent of the global `videos.via_search` flag (which marks a video as +search-discovered by anyone, for the shared Library's Source filter). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0030_search_finds" +down_revision: Union[str, None] = "0029_unaccent_search" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "search_finds", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("video_id", sa.String(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["video_id"], ["videos.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "video_id", name="uq_user_search_video"), + ) + op.create_index("ix_search_finds_user_id", "search_finds", ["user_id"]) + op.create_index("ix_search_finds_video_id", "search_finds", ["video_id"]) + + +def downgrade() -> None: + op.drop_index("ix_search_finds_video_id", table_name="search_finds") + op.drop_index("ix_search_finds_user_id", table_name="search_finds") + op.drop_table("search_finds") diff --git a/backend/alembic/versions/0031_title_fts.py b/backend/alembic/versions/0031_title_fts.py new file mode 100644 index 0000000..ea16ac9 --- /dev/null +++ b/backend/alembic/versions/0031_title_fts.py @@ -0,0 +1,49 @@ +"""full-text relevance search on video titles + +Revision ID: 0031_title_fts +Revises: 0030_search_finds +Create Date: 2026-06-30 + +Adds a PostgreSQL full-text search index over video titles so the feed search box can rank by +relevance (a YouTube-like "fuzzy" search: word-order-independent, multi-word AND, prefix on the +word being typed) instead of a whole-phrase substring match. + +A custom text-search configuration `unaccent_simple` = the `simple` config (no stemming, no +stopwords — right for a multilingual HU/EN/DE catalog) plus the `unaccent` dictionary, so the +search stays accent-insensitive ("tiesto" matches "Tiësto"). A GIN expression index on +`to_tsvector('public.unaccent_simple', coalesce(title,''))` makes the @@ match fast; the feed +query uses the exact same expression so the planner picks the index. `unaccent` is already +enabled (migration 0029). The config-based 2-arg to_tsvector is IMMUTABLE, so it's index-safe. +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0031_title_fts" +down_revision: Union[str, None] = "0030_search_finds" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + "CREATE TEXT SEARCH CONFIGURATION public.unaccent_simple (COPY = pg_catalog.simple)" + ) + op.execute( + """ + ALTER TEXT SEARCH CONFIGURATION public.unaccent_simple + ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, word, hword, hword_part + WITH unaccent, simple + """ + ) + op.execute( + """ + CREATE INDEX ix_videos_title_fts ON videos + USING gin (to_tsvector('public.unaccent_simple', coalesce(title, ''))) + """ + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_videos_title_fts") + op.execute("DROP TEXT SEARCH CONFIGURATION IF EXISTS public.unaccent_simple") diff --git a/backend/app/models.py b/backend/app/models.py index f40653d..fac6fab 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -344,6 +344,25 @@ class VideoState(Base, UpdatedAtMixin): progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) +class SearchFind(Base, TimestampMixin): + """Per-user record of a video the user surfaced via their own live YouTube search. + + Lets "videos I searched for" appear in that user's own (Mine) feed and the Mine Source + filter — distinct from the GLOBAL `Video.via_search` provenance, which marks a video as + search-discovered by *anyone* and drives the shared Library's Source filter.""" + + __tablename__ = "search_finds" + __table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + video_id: Mapped[str] = mapped_column( + ForeignKey("videos.id", ondelete="CASCADE"), index=True + ) + + class ApiQuotaUsage(Base): """Tracks YouTube Data API units spent per Pacific-time day (the quota resets at midnight Pacific). The whole app shares one daily budget.""" diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 09648d5..0659c0d 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -1,10 +1,11 @@ import base64 import binascii import json +import re from datetime import date, datetime, timedelta, timezone from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy import Select, and_, false, func, or_, select +from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session, aliased from sqlalchemy.orm.exc import StaleDataError @@ -17,6 +18,7 @@ from app.models import ( ChannelTag, Playlist, PlaylistItem, + SearchFind, Subscription, Tag, User, @@ -153,14 +155,20 @@ def _filtered_query( ), ) else: - # Only channels this user is subscribed to (and hasn't hidden). - query = query.join( + # "Mine": the user's own videos = their non-hidden subscriptions OR videos they + # surfaced via their own live YouTube search. Both are LEFT-joined so the Source + # filter below can include either or both, and the subscription row stays available + # for the priority sort. + query = query.outerjoin( Subscription, and_( Subscription.channel_id == Video.channel_id, Subscription.user_id == user.id, Subscription.hidden.is_(False), ), + ).outerjoin( + SearchFind, + and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id), ) query = query.outerjoin( @@ -172,16 +180,25 @@ def _filtered_query( # either way they shouldn't show in any feed view meanwhile. query = query.where(Video.unavailable_since.is_(None)) - # Provenance filter for the Library: search-discovered videos are "noise" hidden by - # default ("organic"), can be mixed in ("all"), or shown exclusively ("search"). Only - # meaningful in "all" scope: "my" already restricts to subscribed channels, where a - # once-searched video is legitimately the user's. + # Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag + # (search-discovered by anyone). In "my" scope it's the user's OWN provenance: + # "organic" = your subscriptions, "search" = videos you found via your own search, + # "all" = both. Default "organic" keeps the Mine feed = subscriptions only. if scope == "all": if library_source == "organic": query = query.where(Video.via_search.is_(False)) elif library_source == "search": query = query.where(Video.via_search.is_(True)) # "all" → both organic and search-discovered videos + else: + subscribed = Subscription.channel_id.isnot(None) + searched = SearchFind.video_id.isnot(None) + if library_source == "search": + query = query.where(searched) + elif library_source == "all": + query = query.where(or_(subscribed, searched)) + else: # "organic" (default): subscriptions only + query = query.where(subscribed) if channel_id: query = query.where(Video.channel_id == channel_id) @@ -204,16 +221,39 @@ def _filtered_query( published_before, datetime.min.time(), tzinfo=timezone.utc ) + timedelta(days=1) query = query.where(Video.published_at < end) + # Full-text relevance search on the title: YouTube-like "fuzzy" matching (word-order- + # independent, multi-word AND, prefix on the word being typed) + accent-insensitive via + # the unaccent_simple config. The channel name still matches as a substring so channel + # searches work. `rank_expr` (ts_rank) drives the optional "relevance" sort. The + # to_tsvector expression must match the GIN index (migration 0031) verbatim to be used. + rank_expr = None if q: - # Accent-insensitive: unaccent() both sides so "tiesto" matches "Tiësto" (ILIKE alone - # is case- but not diacritic-insensitive). unaccent is enabled by migration 0029. - like = func.unaccent(f"%{q}%") - query = query.where( - or_( - func.unaccent(Video.title).ilike(like), - func.unaccent(Channel.title).ilike(like), + ts_str = _to_tsquery_str(q) + if ts_str: + tsq = func.to_tsquery("public.unaccent_simple", ts_str) + title_vec = func.to_tsvector( + "public.unaccent_simple", func.coalesce(Video.title, "") + ) + query = query.where( + or_( + title_vec.op("@@")(tsq), + func.unaccent(Channel.title).ilike(func.unaccent(f"%{q}%")), + ) + ) + # Scale ts_rank (a float4) to an integer score so the keyset cursor compares it + # EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and + # breaks paging (the same page repeats). 1e6 granularity is ample; ties break on + # published_at then id. + rank_expr = cast(func.ts_rank(title_vec, tsq) * 1000000, BigInteger) + else: + # No usable tokens (e.g. only punctuation): fall back to a plain substring match. + like = func.unaccent(f"%{q}%") + query = query.where( + or_( + func.unaccent(Video.title).ilike(like), + func.unaccent(Channel.title).ilike(like), + ) ) - ) if tags: # AND across tag categories (e.g. language AND topic narrows), OR within a @@ -271,7 +311,19 @@ def _filtered_query( else: # all query = query.where(status_expr != "hidden") - return query, status_expr + return query, status_expr, rank_expr + + +def _to_tsquery_str(q: str) -> str | None: + """Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word + prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*". + Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in + (the string is built from sanitised tokens, never the raw query).""" + tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)] + if not tokens: + return None + tokens[-1] = tokens[-1] + ":*" + return " & ".join(tokens) # Shared query parameters for /feed and /feed/count. @@ -320,7 +372,7 @@ def _feed_params( # replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows, # and the page boundary is stable even as the scheduler ingests new videos mid-scroll # (OFFSET would shift and duplicate/skip rows). -def _sort_keys(sort: str, seed: int) -> list[dict]: +def _sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]: pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"} table: dict[str, list[dict]] = { "newest": [pub], @@ -341,6 +393,13 @@ def _sort_keys(sort: str, seed: int) -> list[dict]: # the cursor stays valid across pages of one shuffled run. "shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}], } + # Relevance (search-result ranking) only exists when there's a query to rank against; + # without `rank_expr` it falls through to newest. Ties break by newest, then id. + if rank_expr is not None: + table["relevance"] = [ + {"expr": rank_expr, "asc": False, "nullable": False, "kind": "num"}, + pub, + ] return table.get(sort) or table["newest"] @@ -409,8 +468,8 @@ def get_feed( user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: - query, _status = _filtered_query(db, user, **params) - keys = _sort_keys(sort, seed) + query, _status, rank_expr = _filtered_query(db, user, **params) + keys = _sort_keys(sort, seed, rank_expr) # Expose each key column on the row so we can build the next cursor from the last item. query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)]) @@ -439,7 +498,7 @@ def get_feed_count( user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: - query, _status = _filtered_query(db, user, **params) + query, _status, _rank = _filtered_query(db, user, **params) total = db.scalar(select(func.count()).select_from(query.subquery())) return {"count": total or 0} @@ -467,7 +526,7 @@ def get_facets( # keeps them applied, so each remaining chip narrows to channels that ALSO have all # already-selected topics — and tags that can't co-occur drop to zero (hidden). conjunctive = category == "topic" and params.get("tag_mode") == "and" - base, _status = _filtered_query( + base, _status, _rank = _filtered_query( db, user, **{**params, "exclude_tag_category": None if conjunctive else category}, diff --git a/backend/app/routes/search.py b/backend/app/routes/search.py index efba64f..23ad271 100644 --- a/backend/app/routes/search.py +++ b/backend/app/routes/search.py @@ -17,12 +17,13 @@ from datetime import datetime, timezone from fastapi import APIRouter, Depends, HTTPException from sqlalchemy import and_, select +from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session, aliased from app import quota, sysconfig from app.auth import require_human from app.db import get_db -from app.models import Channel, User, Video, VideoState +from app.models import Channel, SearchFind, User, Video, VideoState from app.routes.feed import _serialize, feed_columns from app.sync.subscriptions import apply_channel_details from app.sync.videos import _insert_stubs, apply_video_details, parse_dt @@ -213,6 +214,17 @@ def search_youtube( v.id for v in videos if not v.is_short and v.live_status not in _LIVE_HIDDEN } ordered = [vid for vid in ids if vid in keep] + + # 6) Record these as THIS user's search finds so they surface in their own Mine feed + # (and the Mine Source filter), independent of the global via_search flag. Idempotent. + if ordered: + db.execute( + pg_insert(SearchFind) + .values([{"user_id": user.id, "video_id": vid} for vid in ordered]) + .on_conflict_do_nothing(index_elements=["user_id", "video_id"]) + ) + db.commit() + return { "items": _serialize_ids(db, user, ordered), "next_cursor": page["next_page_token"], From 2e5919399c9e4c013681414863a5433d9d9fce5f Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 30 Jun 2026 00:39:21 +0200 Subject: [PATCH 2/4] feat(feed): Source filter in Mine, Relevance sort, auto search-source Frontend for the Mine search finds + relevance search: - The Source filter (organic / include search / search-only) now shows in Mine scope too, not just the Library. - Returning from a YouTube search via 'Back to feed' switches the Source filter to 'search' so your just-found videos show in the feed you land on (filtered by the kept term). - New 'Relevance' sort, offered while a search term is present and auto-selected when you start searching (reverts to newest when you clear it). EN/HU/DE. --- frontend/src/components/Feed.tsx | 43 ++++++++++++++++++++------ frontend/src/i18n/locales/de/feed.json | 5 +-- frontend/src/i18n/locales/en/feed.json | 5 +-- frontend/src/i18n/locales/hu/feed.json | 5 +-- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/frontend/src/components/Feed.tsx b/frontend/src/components/Feed.tsx index 0223735..13ac40a 100644 --- a/frontend/src/components/Feed.tsx +++ b/frontend/src/components/Feed.tsx @@ -23,9 +23,11 @@ const rollSeed = () => Math.floor(Math.random() * 1_000_000_000); // Ordering = a key + direction (like the Playlists page), mapped to the backend sort strings // (which encode both). One entry per concept; a single arrow flips the direction. -type SortKey = "date" | "popular" | "duration" | "title" | "subscribers" | "priority" | "shuffle"; +// "relevance" (full-text search ranking) and "shuffle" are directionless single-string sorts. +type SortKey = "date" | "popular" | "duration" | "title" | "subscribers" | "priority" | "shuffle" | "relevance"; const SORT_KEYS: SortKey[] = ["date", "popular", "duration", "title", "subscribers", "priority", "shuffle"]; -const SORT_MAP: Record, { asc: string; desc: string }> = { +const DIRECTIONLESS: SortKey[] = ["shuffle", "relevance"]; +const SORT_MAP: Record, { asc: string; desc: string }> = { date: { desc: "newest", asc: "oldest" }, popular: { desc: "views", asc: "views_asc" }, duration: { desc: "duration_desc", asc: "duration_asc" }, @@ -33,11 +35,11 @@ const SORT_MAP: Record, { asc: string; desc: string subscribers: { desc: "subscribers", asc: "subscribers_asc" }, priority: { desc: "priority", asc: "priority_asc" }, }; -const SORT_DEFAULT_DIR: Record, "asc" | "desc"> = { +const SORT_DEFAULT_DIR: Record, "asc" | "desc"> = { date: "desc", popular: "desc", duration: "desc", title: "asc", subscribers: "desc", priority: "desc", }; function parseSort(s: string): { key: SortKey; dir: "asc" | "desc" } { - if (s === "shuffle") return { key: "shuffle", dir: "desc" }; + if (s === "shuffle" || s === "relevance") return { key: s, dir: "desc" }; for (const k of Object.keys(SORT_MAP) as (keyof typeof SORT_MAP)[]) { if (SORT_MAP[k].asc === s) return { key: k, dir: "asc" }; if (SORT_MAP[k].desc === s) return { key: k, dir: "desc" }; @@ -45,7 +47,8 @@ function parseSort(s: string): { key: SortKey; dir: "asc" | "desc" } { return { key: "date", dir: "desc" }; } function buildSort(key: SortKey, dir: "asc" | "desc"): string { - return key === "shuffle" ? "shuffle" : SORT_MAP[key][dir]; + if (key === "shuffle" || key === "relevance") return key; + return SORT_MAP[key][dir]; } function matchesView(status: string, show: string): boolean { @@ -134,6 +137,22 @@ export default function Feed({ retry: false, }); + // When a search term first appears, rank the local feed by relevance (YouTube-like); when + // it's cleared, fall back to the default newest sort. Only fires on the empty↔non-empty + // transition, so it never overrides a sort the user picks while a query stays active. + const qEmpty = !filters.q.trim(); + const prevQEmptyRef = useRef(qEmpty); + useEffect(() => { + const sortKeyNow = parseSort(filters.sort).key; + if (prevQEmptyRef.current && !qEmpty && sortKeyNow !== "relevance") { + setFilters({ ...filters, sort: "relevance" }); + } else if (!prevQEmptyRef.current && qEmpty && sortKeyNow === "relevance") { + setFilters({ ...filters, sort: buildSort("date", "desc") }); + } + prevQEmptyRef.current = qEmpty; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [qEmpty]); + // Drop optimistic status overrides when filters change OR fresh server data // arrives — after a refetch the server is authoritative, so a stale override // (e.g. a video reverted to "new" from the notification center) won't keep it @@ -296,6 +315,9 @@ export default function Feed({ // The search just ingested new catalog videos, so the normal feed (which was // disabled and still holds its pre-search cache) is stale. Drop those caches so // it re-fetches fresh on return instead of showing the old (often empty) result. + // Surface the just-searched videos in the feed you return to: switch the Source + // filter to "search" so your own search finds show (filtered by the kept term). + setFilters({ ...filters, librarySource: "search" }); onExitYtSearch(); qc.removeQueries({ queryKey: ["feed"] }); qc.removeQueries({ queryKey: ["feed-count"] }); @@ -445,7 +467,7 @@ export default function Feed({ ); })} - {filters.scope === "all" && ( + {( <>