diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index caa9d87..e2f331a 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -1,3 +1,6 @@ +import base64 +import binascii +import json from datetime import date, datetime, timedelta, timezone from fastapi import APIRouter, Depends, HTTPException, Query @@ -278,21 +281,92 @@ def _feed_params( } -SORTS = { - "newest": Video.published_at.desc().nulls_last(), - "oldest": Video.published_at.asc().nulls_last(), - "views": Video.view_count.desc().nulls_last(), - "views_asc": Video.view_count.asc().nulls_last(), - "duration_desc": Video.duration_seconds.desc().nulls_last(), - "duration_asc": Video.duration_seconds.asc().nulls_last(), - "title": func.lower(Video.title).asc().nulls_last(), - "title_desc": func.lower(Video.title).desc().nulls_last(), - "subscribers": Channel.subscriber_count.desc().nulls_last(), - "subscribers_asc": Channel.subscriber_count.asc().nulls_last(), - # Your per-channel priority (set in the channel manager), newest first within a tier. - # coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row. - "priority": func.coalesce(Subscription.priority, 0).desc(), -} +# --- Keyset (cursor) pagination --------------------------------------------- +# +# Each sort is described as an ordered list of key columns (expression + direction +# + whether it can be NULL + a JSON round-trip kind), always with Video.id as the +# final, unique tiebreaker. This drives BOTH the ORDER BY and the keyset WHERE so +# they can never drift apart — the prerequisite for correct cursor paging. Keyset +# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows, +# and the page boundary is stable even as the scheduler ingests new videos mid-scroll +# (OFFSET would shift and duplicate/skip rows). +def _sort_keys(sort: str, seed: int) -> list[dict]: + pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"} + table: dict[str, list[dict]] = { + "newest": [pub], + "oldest": [{**pub, "asc": True}], + "views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}], + "views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}], + "duration_desc": [{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}], + "duration_asc": [{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}], + "title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}], + "title_desc": [{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}], + "subscribers": [{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}], + "subscribers_asc": [{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}], + # Your per-channel priority (set in the channel manager), newest first within a tier. + # coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row. + "priority": [{"expr": func.coalesce(Subscription.priority, 0), "asc": False, "nullable": False, "kind": "num"}, pub], + "priority_asc": [{"expr": func.coalesce(Subscription.priority, 0), "asc": True, "nullable": False, "kind": "num"}, pub], + # Deterministic per-seed shuffle: the same seed yields the same total order, so + # the cursor stays valid across pages of one shuffled run. + "shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}], + } + return table.get(sort) or table["newest"] + + +def _order_by(keys: list[dict]): + cols = [] + for k in keys: + c = k["expr"].asc() if k["asc"] else k["expr"].desc() + cols.append(c.nulls_last() if k["nullable"] else c) + cols.append(Video.id.asc()) # unique final tiebreaker + return cols + + +def _keyset_predicate(keys: list[dict], cursor_id: str): + """WHERE clause selecting rows strictly AFTER the cursor in the sort order. + + Lexicographic over the key columns then id, NULL-aware for `nulls_last`: a NULL + sorts after every non-NULL value, so a cursor sitting on a non-NULL value must + also pull in the NULL group, and equality treats NULL == NULL.""" + def eq(k): + return k["expr"].is_(None) if k["value"] is None else k["expr"] == k["value"] + + def after(k): + if k["value"] is None: + return false() # nulls are last; nothing sorts strictly after them + cmp = k["expr"] > k["value"] if k["asc"] else k["expr"] < k["value"] + return or_(cmp, k["expr"].is_(None)) if k["nullable"] else cmp + + branches = [] + for j in range(len(keys)): + branches.append(and_(*[eq(keys[i]) for i in range(j)], after(keys[j]))) + branches.append(and_(*[eq(k) for k in keys], Video.id > cursor_id)) + return or_(*branches) + + +def _encode_cursor(keys: list[dict], row) -> str: + vals = [] + for i, k in enumerate(keys): + v = getattr(row, f"_k{i}") + vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v) + payload = json.dumps({"k": vals, "id": row.id}) + return base64.urlsafe_b64encode(payload.encode()).decode() + + +def _decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str]: + try: + payload = json.loads(base64.urlsafe_b64decode(cursor.encode())) + raw = payload["k"] + if len(raw) != len(keys): + raise ValueError("key arity mismatch") + vals = [ + datetime.fromisoformat(v) if (v is not None and k["kind"] == "dt") else v + for k, v in zip(keys, raw) + ] + return vals, payload["id"] + except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError): + raise HTTPException(status_code=400, detail="Invalid feed cursor") @router.get("/feed") @@ -301,29 +375,30 @@ def get_feed( sort: str = "newest", seed: int = 0, limit: int = Query(default=60, le=200), - offset: int = 0, + cursor: str | None = None, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: query, _status = _filtered_query(db, user, **params) - if sort in ("priority", "priority_asc"): - prio = func.coalesce(Subscription.priority, 0) - query = query.order_by( - prio.asc() if sort == "priority_asc" else prio.desc(), - Video.published_at.desc().nulls_last(), - ) - else: - order = SORTS.get(sort) - if order is None and sort == "shuffle": - order = func.md5(func.concat(Video.id, str(seed))) - query = query.order_by(order if order is not None else SORTS["newest"]) + keys = _sort_keys(sort, seed) - rows = db.execute(query.offset(offset).limit(limit + 1)).all() + # Expose each key column on the row so we can build the next cursor from the last item. + query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)]) + query = query.order_by(*_order_by(keys)) + + if cursor: + vals, cursor_id = _decode_cursor(keys, cursor) + for k, v in zip(keys, vals): + k["value"] = v + query = query.where(_keyset_predicate(keys, cursor_id)) + + rows = db.execute(query.limit(limit + 1)).all() has_more = len(rows) > limit + page = rows[:limit] return { - "items": [_serialize(r) for r in rows[:limit]], + "items": [_serialize(r) for r in page], + "next_cursor": _encode_cursor(keys, page[-1]) if (has_more and page) else None, "has_more": has_more, - "offset": offset, "limit": limit, } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 00661c5..33392a9 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -10,6 +10,7 @@ "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", "@tanstack/react-query": "^5.51.0", + "@tanstack/react-virtual": "^3.14.3", "clsx": "^2.1.1", "i18next": "^23.11.5", "lucide-react": "^0.408.0", @@ -1247,6 +1248,33 @@ "react": "^18 || ^19" } }, + "node_modules/@tanstack/react-virtual": { + "version": "3.14.3", + "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.14.3.tgz", + "integrity": "sha512-k/cnHPVaOfn46hSbiY6n4Dzf4QjCGWSF40zR5QIIYUqPAjpA6TN7InfYmcMiDVQGP2iUn9xsRbAl8u1v3UmeVQ==", + "license": "MIT", + "dependencies": { + "@tanstack/virtual-core": "3.17.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/@tanstack/virtual-core": { + "version": "3.17.1", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.1.tgz", + "integrity": "sha512-VZyW2Uiml5tmBZwPGrSD3Sz73OxzljQMCmzYHsUTPEuTsERf5xwa+uWb01xEzkz3ZSYTjj8NEb/mKHvgKxyZdA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", diff --git a/frontend/package.json b/frontend/package.json index 8567f4f..a2cfc6a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,6 +12,7 @@ "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", "@tanstack/react-query": "^5.51.0", + "@tanstack/react-virtual": "^3.14.3", "clsx": "^2.1.1", "i18next": "^23.11.5", "lucide-react": "^0.408.0", diff --git a/frontend/src/components/Feed.tsx b/frontend/src/components/Feed.tsx index f66f61c..300b7f9 100644 --- a/frontend/src/components/Feed.tsx +++ b/frontend/src/components/Feed.tsx @@ -5,7 +5,7 @@ import { ArrowDown, ArrowUp, RefreshCw } from "lucide-react"; import { api, type FeedFilters, type Video } from "../lib/api"; import i18n from "../i18n"; import { notify, resolveVideo } from "../lib/notifications"; -import VideoCard from "./VideoCard"; +import VirtualFeed from "./VirtualFeed"; import PlayerModal from "./PlayerModal"; const PAGE = 60; @@ -94,9 +94,9 @@ export default function Feed({ const query = useInfiniteQuery({ queryKey: ["feed", filters], - queryFn: ({ pageParam }) => api.feed(filters, pageParam as number, PAGE), - initialPageParam: 0, - getNextPageParam: (last) => (last.has_more ? last.offset + last.limit : undefined), + queryFn: ({ pageParam }) => api.feed(filters, pageParam as string | null, PAGE), + initialPageParam: null as string | null, + getNextPageParam: (last) => last.next_cursor ?? undefined, }); // Drop optimistic status overrides when filters change OR fresh server data @@ -118,22 +118,7 @@ export default function Feed({ staleTime: 30_000, }); - const sentinel = useRef(null); const { hasNextPage, isFetchingNextPage, fetchNextPage } = query; - useEffect(() => { - const el = sentinel.current; - if (!el) return; - const io = new IntersectionObserver( - (entries) => { - if (entries[0].isIntersecting && hasNextPage && !isFetchingNextPage) { - fetchNextPage(); - } - }, - { rootMargin: "1500px" } // prefetch the next page well before the bottom is in view - ); - io.observe(el); - return () => io.disconnect(); - }, [hasNextPage, isFetchingNextPage, fetchNextPage]); // Keep the loaded videos in a ref so onState can stay referentially stable // (it needs the list only to look up a title for the notification). A stable @@ -388,36 +373,19 @@ export default function Feed({ {toolbar} {items.length === 0 ? (
{t("feed.noMatches")}
- ) : view === "grid" ? ( -
- {items.map((v) => ( - - ))} -
) : ( -
- {items.map((v) => ( - - ))} -
+ )} {activeVideo && ( )} -
{isFetchingNextPage && (
{t("feed.loadingMore")}
)} diff --git a/frontend/src/components/VirtualFeed.tsx b/frontend/src/components/VirtualFeed.tsx new file mode 100644 index 0000000..2048cd5 --- /dev/null +++ b/frontend/src/components/VirtualFeed.tsx @@ -0,0 +1,176 @@ +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; +import { useVirtualizer } from "@tanstack/react-virtual"; +import type { Video } from "../lib/api"; +import VideoCard from "./VideoCard"; + +// Grid column sizing mirrors the CSS the feed used before virtualization +// (grid-cols-[repeat(auto-fill,minmax(260px,1fr))] with a gap-4 / 1rem gutter). +const GRID_MIN_COL = 260; +const GRID_GAP = 16; +const LIST_GAP = 4; +// Estimated row heights before measurement; real heights are measured per-row so +// these only affect the very first paint and the scrollbar's initial guess. +const GRID_ROW_EST = 340; +const LIST_ROW_EST = 96; +// Start fetching the next page this many rows before the end scrolls into view +// (keeps the old IntersectionObserver's generous prefetch feel). +const PREFETCH_ROWS = 4; + +function chunk(arr: T[], size: number): T[][] { + if (size <= 1) return arr.map((x) => [x]); + const out: T[][] = []; + for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size)); + return out; +} + +// The feed scrolls inside the app's
, not the window, so the +// virtualizer needs that element. Walk up to the nearest scrollable ancestor. +function getScrollParent(node: HTMLElement | null): HTMLElement { + let el = node?.parentElement ?? null; + while (el) { + const oy = getComputedStyle(el).overflowY; + if (oy === "auto" || oy === "scroll") return el; + el = el.parentElement; + } + return document.documentElement; +} + +export interface VirtualFeedProps { + items: Video[]; + view: "grid" | "list"; + onState: (id: string, status: string) => void; + onToggleSave: (id: string, saved: boolean) => void; + onChannelFilter: (channelId: string, channelName: string) => void; + onResetState: (id: string) => void; + onOpen: (v: Video, startAt?: number | null) => void; + hasNextPage: boolean; + isFetchingNextPage: boolean; + fetchNextPage: () => void; +} + +export default function VirtualFeed({ + items, + view, + onState, + onToggleSave, + onChannelFilter, + onResetState, + onOpen, + hasNextPage, + isFetchingNextPage, + fetchNextPage, +}: VirtualFeedProps) { + const listRef = useRef(null); + const [scrollEl, setScrollEl] = useState(null); + const [colCount, setColCount] = useState(view === "list" ? 1 : 4); + // Distance from the scroll element's content top to where this list starts (the + // feed toolbar sits above it inside the same scroll area). + const [scrollMargin, setScrollMargin] = useState(0); + + useLayoutEffect(() => { + if (listRef.current) setScrollEl(getScrollParent(listRef.current)); + }, []); + + // Re-measure the column count and start offset on container/scroll-area resize. + useLayoutEffect(() => { + const node = listRef.current; + if (!node) return; + const measure = () => { + if (scrollEl) { + const m = + node.getBoundingClientRect().top - + scrollEl.getBoundingClientRect().top + + scrollEl.scrollTop; + setScrollMargin(m); + } + setColCount( + view === "list" + ? 1 + : Math.max(1, Math.floor((node.clientWidth + GRID_GAP) / (GRID_MIN_COL + GRID_GAP))) + ); + }; + measure(); + const ro = new ResizeObserver(measure); + ro.observe(node); + if (scrollEl) ro.observe(scrollEl); + return () => ro.disconnect(); + }, [scrollEl, view]); + + const rows = useMemo(() => chunk(items, colCount), [items, colCount]); + + const virtualizer = useVirtualizer({ + count: rows.length, + getScrollElement: () => scrollEl, + estimateSize: () => (view === "grid" ? GRID_ROW_EST : LIST_ROW_EST), + overscan: 4, + gap: view === "grid" ? GRID_GAP : LIST_GAP, + scrollMargin, + }); + + const virtualRows = virtualizer.getVirtualItems(); + + // Infinite-scroll trigger: once the last rendered row is within PREFETCH_ROWS of + // the end, pull the next keyset page. + useEffect(() => { + const last = virtualRows[virtualRows.length - 1]; + if (!last) return; + if (last.index >= rows.length - 1 - PREFETCH_ROWS && hasNextPage && !isFetchingNextPage) { + fetchNextPage(); + } + }, [virtualRows, rows.length, hasNextPage, isFetchingNextPage, fetchNextPage]); + + return ( +
+ {virtualRows.map((vr) => { + const row = rows[vr.index]; + if (!row) return null; + return ( +
+ {view === "grid" ? ( +
+ {row.map((v) => ( + + ))} +
+ ) : ( +
+ +
+ )} +
+ ); + })} +
+ ); +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 2251e59..e5f130c 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -79,7 +79,8 @@ export interface VideoDetail { export interface FeedResponse { items: Video[]; has_more: boolean; - offset: number; + // Opaque keyset cursor for the next page, or null when this is the last page. + next_cursor: string | null; limit: number; } @@ -308,11 +309,11 @@ function filterParams(f: FeedFilters): URLSearchParams { return p; } -function feedQuery(f: FeedFilters, offset: number, limit: number): string { +function feedQuery(f: FeedFilters, cursor: string | null, limit: number): string { const p = filterParams(f); p.set("sort", f.sort); if (f.sort === "shuffle" && f.seed) p.set("seed", String(f.seed)); - p.set("offset", String(offset)); + if (cursor) p.set("cursor", cursor); p.set("limit", String(limit)); return p.toString(); } @@ -511,10 +512,10 @@ export const api = { version: (): Promise => req("/api/version"), tags: (): Promise => req("/api/tags"), status: (): Promise => req("/api/sync/status"), - feed: (f: FeedFilters, offset: number, limit: number): Promise => - req(`/api/feed?${feedQuery(f, offset, limit)}`), + feed: (f: FeedFilters, cursor: string | null, limit: number): Promise => + req(`/api/feed?${feedQuery(f, cursor, limit)}`), feedCount: (f: FeedFilters): Promise<{ count: number }> => - req(`/api/feed/count?${feedQuery(f, 0, 0)}`), + req(`/api/feed/count?${filterParams(f).toString()}`), facets: (f: FeedFilters): Promise<{ counts: Record }> => req(`/api/facets?${filterParams(f).toString()}`), // idempotent: setting a state / saving a progress checkpoint is safe to replay, so these