From 4395afc21003322aeb5d8393a972051e29c36246 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 30 Jun 2026 02:00:38 +0200 Subject: [PATCH] feat(feed): broaden search to a weighted document (title + keywords + queries + description) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the local search behave more like YouTube's — finding videos by the uploader's own keywords or the query that surfaced them, not only words in the title. A DB-generated, weighted search_vector (migration 0032) replaces the title-only FTS index: - keywords: the creator's snippet.tags (free — already in the snippet we fetch), stored on enrich. - search_terms: distinct live-search queries that surfaced the video (across all users), appended by the search route — folds YouTube's relevance into local search (a video YT returned for a query becomes findable by it even without a title match), the user's own idea. - description (truncated) for broad recall on the existing catalog. Weighted title(A) > keywords+queries(B) > description(C) so ts_rank keeps title hits on top. A plain GIN index on the generated column guarantees index use (no expression/param matching). Verified on localdev: recall 146->213 for one query; 7 'eurovision' hits via the document but not the title; index scan confirmed. --- .../alembic/versions/0032_search_document.py | 65 +++++++++++++++++++ backend/app/models.py | 25 +++++++ backend/app/routes/feed.py | 17 ++--- backend/app/routes/search.py | 16 ++++- backend/app/sync/videos.py | 3 + 5 files changed, 115 insertions(+), 11 deletions(-) create mode 100644 backend/alembic/versions/0032_search_document.py diff --git a/backend/alembic/versions/0032_search_document.py b/backend/alembic/versions/0032_search_document.py new file mode 100644 index 0000000..3683df4 --- /dev/null +++ b/backend/alembic/versions/0032_search_document.py @@ -0,0 +1,65 @@ +"""richer full-text search document (title + keywords + search terms + description) + +Revision ID: 0032_search_document +Revises: 0031_title_fts +Create Date: 2026-06-30 + +Broadens the feed's full-text search from the title alone to a weighted search document, so the +local search behaves more like YouTube's — finding videos by the uploader's own keywords or by +the search query that surfaced them, not only by words in the title: + +- `keywords` — the creator's snippet.tags (already fetched with the snippet we request, so + zero extra quota), stored space-joined. +- `search_terms` — distinct live-search queries that surfaced the video (across all users), + appended by the search route. Folds YouTube's relevance judgement into local + search: a video YT returned for a query becomes findable by it locally. + +These plus a truncated description feed a STORED generated `search_vector` column, weighted +title (A) > keywords+search_terms (B) > description (C) so ts_rank ranks title hits highest. A +plain GIN index on the column guarantees the planner uses it (no expression/param matching). The +title-only expression index from 0031 is dropped (superseded). Existing rows get title + +description indexed immediately; keywords/search_terms fill in as videos are (re-)enriched and +searched. Uses the `unaccent_simple` config from 0031 (accent-insensitive). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0032_search_document" +down_revision: Union[str, None] = "0031_title_fts" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_SEARCH_VECTOR_EXPR = ( + "setweight(to_tsvector('public.unaccent_simple', coalesce(title, '')), 'A') || " + "setweight(to_tsvector('public.unaccent_simple', coalesce(keywords, '') || ' ' || " + "coalesce(search_terms, '')), 'B') || " + "setweight(to_tsvector('public.unaccent_simple', left(coalesce(description, ''), 1000)), 'C')" +) + + +def upgrade() -> None: + op.add_column("videos", sa.Column("keywords", sa.Text(), nullable=True)) + op.add_column("videos", sa.Column("search_terms", sa.Text(), nullable=True)) + # Replace the title-only index with a generated weighted search_vector + its GIN index. + op.execute("DROP INDEX IF EXISTS ix_videos_title_fts") + op.execute( + f"ALTER TABLE videos ADD COLUMN search_vector tsvector " + f"GENERATED ALWAYS AS ({_SEARCH_VECTOR_EXPR}) STORED" + ) + op.execute( + "CREATE INDEX ix_videos_search_vector ON videos USING gin (search_vector)" + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_videos_search_vector") + op.drop_column("videos", "search_vector") + op.drop_column("videos", "search_terms") + op.drop_column("videos", "keywords") + # Recreate the title-only FTS index from 0031. + op.execute( + "CREATE INDEX ix_videos_title_fts ON videos " + "USING gin (to_tsvector('public.unaccent_simple', coalesce(title, '')))" + ) diff --git a/backend/app/models.py b/backend/app/models.py index fac6fab..a35a10f 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -3,6 +3,7 @@ from datetime import date, datetime from sqlalchemy import ( BigInteger, Boolean, + Computed, Date, DateTime, Float, @@ -14,6 +15,7 @@ from sqlalchemy import ( UniqueConstraint, func, ) +from sqlalchemy.dialects.postgresql import TSVECTOR from sqlalchemy.orm import Mapped, mapped_column, relationship from app.db import Base @@ -237,6 +239,14 @@ class Video(Base, TimestampMixin): topic_categories: Mapped[list | None] = mapped_column(JSON) default_language: Mapped[str | None] = mapped_column(String(16)) detected_language: Mapped[str | None] = mapped_column(String(16)) + # Creator-supplied keyword tags (snippet.tags joined by spaces) — fetched for free with the + # snippet we already request. Indexed into search_vector so the feed search matches the + # uploader's own keywords, not just words in the title. + keywords: Mapped[str | None] = mapped_column(Text) + # Accumulated distinct live-search queries that surfaced this video (across all users) — + # folds YouTube's own relevance judgement into the local search: a video YouTube returned + # for "ti amo magyarul" becomes findable by that query locally even if its title lacks it. + search_terms: Mapped[str | None] = mapped_column(Text) is_short: Mapped[bool] = mapped_column( Boolean, default=False, server_default="false", index=True ) @@ -272,6 +282,21 @@ class Video(Base, TimestampMixin): ) unavailable_reason: Mapped[str | None] = mapped_column(String(24)) + # Weighted full-text search document (DB-generated, never written by the app): title (A) > + # creator keywords + search queries (B) > truncated description (C). ts_rank honours the + # weights so title matches outrank description matches. Backed by a GIN index (migration + # 0032). The accent-insensitive `unaccent_simple` config comes from migration 0031. + search_vector: Mapped[object | None] = mapped_column( + TSVECTOR, + Computed( + "setweight(to_tsvector('public.unaccent_simple', coalesce(title, '')), 'A') || " + "setweight(to_tsvector('public.unaccent_simple', coalesce(keywords, '') || ' ' || " + "coalesce(search_terms, '')), 'B') || " + "setweight(to_tsvector('public.unaccent_simple', left(coalesce(description, ''), 1000)), 'C')", + persisted=True, + ), + ) + channel: Mapped["Channel"] = relationship(back_populates="videos") diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 0659c0d..528a42e 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -221,22 +221,19 @@ def _filtered_query( published_before, datetime.min.time(), tzinfo=timezone.utc ) + timedelta(days=1) query = query.where(Video.published_at < end) - # Full-text relevance search on the title: YouTube-like "fuzzy" matching (word-order- - # independent, multi-word AND, prefix on the word being typed) + accent-insensitive via - # the unaccent_simple config. The channel name still matches as a substring so channel - # searches work. `rank_expr` (ts_rank) drives the optional "relevance" sort. The - # to_tsvector expression must match the GIN index (migration 0031) verbatim to be used. + # Full-text relevance search over the weighted search document (title > creator keywords + + # the queries that surfaced the video > description; see Video.search_vector). YouTube-like + # "fuzzy" matching: word-order-independent, multi-word AND, prefix on the word being typed, + # accent-insensitive (unaccent_simple). The channel name still matches as a substring so + # channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort. rank_expr = None if q: ts_str = _to_tsquery_str(q) if ts_str: tsq = func.to_tsquery("public.unaccent_simple", ts_str) - title_vec = func.to_tsvector( - "public.unaccent_simple", func.coalesce(Video.title, "") - ) query = query.where( or_( - title_vec.op("@@")(tsq), + Video.search_vector.op("@@")(tsq), func.unaccent(Channel.title).ilike(func.unaccent(f"%{q}%")), ) ) @@ -244,7 +241,7 @@ def _filtered_query( # EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and # breaks paging (the same page repeats). 1e6 granularity is ample; ties break on # published_at then id. - rank_expr = cast(func.ts_rank(title_vec, tsq) * 1000000, BigInteger) + rank_expr = cast(func.ts_rank(Video.search_vector, tsq) * 1000000, BigInteger) else: # No usable tokens (e.g. only punctuation): fall back to a plain substring match. like = func.unaccent(f"%{q}%") diff --git a/backend/app/routes/search.py b/backend/app/routes/search.py index 23ad271..0da0e61 100644 --- a/backend/app/routes/search.py +++ b/backend/app/routes/search.py @@ -16,7 +16,7 @@ from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy import and_, select +from sqlalchemy import and_, func, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session, aliased @@ -223,6 +223,20 @@ def search_youtube( .values([{"user_id": user.id, "video_id": vid} for vid in ordered]) .on_conflict_do_nothing(index_elements=["user_id", "video_id"]) ) + # 7) Fold the query into each result's search_terms (shared across users) so local + # full-text search inherits YouTube's relevance — a video YT returned for this query + # becomes findable by it even when its title doesn't contain the words. Skip rows + # that already include the term; the generated search_vector updates automatically. + db.execute( + update(Video) + .where(Video.id.in_(ordered)) + .where(func.coalesce(Video.search_terms, "").notilike(f"%{term}%")) + .values( + search_terms=func.trim( + func.coalesce(Video.search_terms, "").op("||")(" ").op("||")(term) + ) + ) + ) db.commit() return { diff --git a/backend/app/sync/videos.py b/backend/app/sync/videos.py index de0df77..a7a77d9 100644 --- a/backend/app/sync/videos.py +++ b/backend/app/sync/videos.py @@ -234,6 +234,9 @@ def apply_video_details(video: Video, item: dict) -> None: video.view_count = int(stats["viewCount"]) if stats.get("viewCount") else None video.like_count = int(stats["likeCount"]) if stats.get("likeCount") else None video.category_id = int(snippet["categoryId"]) if snippet.get("categoryId") else None + # Creator keyword tags (free with the snippet we already fetch) → search document. + tags = snippet.get("tags") + video.keywords = " ".join(tags) if tags else None video.topic_categories = topics.get("topicCategories") video.default_language = snippet.get("defaultLanguage") or snippet.get( "defaultAudioLanguage"