Merge feature/feed-virtualization-keyset — feed virtualization + keyset pagination (epic 7)

backend/app/routes/feed.py

@@ -1,3 +1,6 @@
+import base64
+import binascii
+import json
 from datetime import date, datetime, timedelta, timezone
 
 from fastapi import APIRouter, Depends, HTTPException, Query
@@ -278,21 +281,92 @@ def _feed_params(
     }
 
 
-SORTS = {
+# --- Keyset (cursor) pagination ---------------------------------------------
-    "newest": Video.published_at.desc().nulls_last(),
+#
-    "oldest": Video.published_at.asc().nulls_last(),
+# Each sort is described as an ordered list of key columns (expression + direction
-    "views": Video.view_count.desc().nulls_last(),
+# + whether it can be NULL + a JSON round-trip kind), always with Video.id as the
-    "views_asc": Video.view_count.asc().nulls_last(),
+# final, unique tiebreaker. This drives BOTH the ORDER BY and the keyset WHERE so
-    "duration_desc": Video.duration_seconds.desc().nulls_last(),
+# they can never drift apart — the prerequisite for correct cursor paging. Keyset
-    "duration_asc": Video.duration_seconds.asc().nulls_last(),
+# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
-    "title": func.lower(Video.title).asc().nulls_last(),
+# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
-    "title_desc": func.lower(Video.title).desc().nulls_last(),
+# (OFFSET would shift and duplicate/skip rows).
-    "subscribers": Channel.subscriber_count.desc().nulls_last(),
+def _sort_keys(sort: str, seed: int) -> list[dict]:
-    "subscribers_asc": Channel.subscriber_count.asc().nulls_last(),
+    pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"}
-    # Your per-channel priority (set in the channel manager), newest first within a tier.
+    table: dict[str, list[dict]] = {
-    # coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
+        "newest": [pub],
-    "priority": func.coalesce(Subscription.priority, 0).desc(),
+        "oldest": [{**pub, "asc": True}],
-}
+        "views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
+        "views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
+        "duration_desc": [{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}],
+        "duration_asc": [{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}],
+        "title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
+        "title_desc": [{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}],
+        "subscribers": [{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}],
+        "subscribers_asc": [{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}],
+        # Your per-channel priority (set in the channel manager), newest first within a tier.
+        # coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
+        "priority": [{"expr": func.coalesce(Subscription.priority, 0), "asc": False, "nullable": False, "kind": "num"}, pub],
+        "priority_asc": [{"expr": func.coalesce(Subscription.priority, 0), "asc": True, "nullable": False, "kind": "num"}, pub],
+        # Deterministic per-seed shuffle: the same seed yields the same total order, so
+        # the cursor stays valid across pages of one shuffled run.
+        "shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
+    }
+    return table.get(sort) or table["newest"]
+
+
+def _order_by(keys: list[dict]):
+    cols = []
+    for k in keys:
+        c = k["expr"].asc() if k["asc"] else k["expr"].desc()
+        cols.append(c.nulls_last() if k["nullable"] else c)
+    cols.append(Video.id.asc())  # unique final tiebreaker
+    return cols
+
+
+def _keyset_predicate(keys: list[dict], cursor_id: str):
+    """WHERE clause selecting rows strictly AFTER the cursor in the sort order.
+
+    Lexicographic over the key columns then id, NULL-aware for `nulls_last`: a NULL
+    sorts after every non-NULL value, so a cursor sitting on a non-NULL value must
+    also pull in the NULL group, and equality treats NULL == NULL."""
+    def eq(k):
+        return k["expr"].is_(None) if k["value"] is None else k["expr"] == k["value"]
+
+    def after(k):
+        if k["value"] is None:
+            return false()  # nulls are last; nothing sorts strictly after them
+        cmp = k["expr"] > k["value"] if k["asc"] else k["expr"] < k["value"]
+        return or_(cmp, k["expr"].is_(None)) if k["nullable"] else cmp
+
+    branches = []
+    for j in range(len(keys)):
+        branches.append(and_(*[eq(keys[i]) for i in range(j)], after(keys[j])))
+    branches.append(and_(*[eq(k) for k in keys], Video.id > cursor_id))
+    return or_(*branches)
+
+
+def _encode_cursor(keys: list[dict], row) -> str:
+    vals = []
+    for i, k in enumerate(keys):
+        v = getattr(row, f"_k{i}")
+        vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v)
+    payload = json.dumps({"k": vals, "id": row.id})
+    return base64.urlsafe_b64encode(payload.encode()).decode()
+
+
+def _decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str]:
+    try:
+        payload = json.loads(base64.urlsafe_b64decode(cursor.encode()))
+        raw = payload["k"]
+        if len(raw) != len(keys):
+            raise ValueError("key arity mismatch")
+        vals = [
+            datetime.fromisoformat(v) if (v is not None and k["kind"] == "dt") else v
+            for k, v in zip(keys, raw)
+        ]
+        return vals, payload["id"]
+    except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError):
+        raise HTTPException(status_code=400, detail="Invalid feed cursor")
 
 
 @router.get("/feed")
@@ -301,29 +375,30 @@ def get_feed(
     sort: str = "newest",
     seed: int = 0,
     limit: int = Query(default=60, le=200),
-    offset: int = 0,
+    cursor: str | None = None,
     user: User = Depends(current_user),
     db: Session = Depends(get_db),
 ) -> dict:
     query, _status = _filtered_query(db, user, **params)
-    if sort in ("priority", "priority_asc"):
+    keys = _sort_keys(sort, seed)
-        prio = func.coalesce(Subscription.priority, 0)
-        query = query.order_by(
-            prio.asc() if sort == "priority_asc" else prio.desc(),
-            Video.published_at.desc().nulls_last(),
-        )
-    else:
-        order = SORTS.get(sort)
-        if order is None and sort == "shuffle":
-            order = func.md5(func.concat(Video.id, str(seed)))
-        query = query.order_by(order if order is not None else SORTS["newest"])
 
-    rows = db.execute(query.offset(offset).limit(limit + 1)).all()
+    # Expose each key column on the row so we can build the next cursor from the last item.
+    query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
+    query = query.order_by(*_order_by(keys))
+
+    if cursor:
+        vals, cursor_id = _decode_cursor(keys, cursor)
+        for k, v in zip(keys, vals):
+            k["value"] = v
+        query = query.where(_keyset_predicate(keys, cursor_id))
+
+    rows = db.execute(query.limit(limit + 1)).all()
     has_more = len(rows) > limit
+    page = rows[:limit]
     return {
-        "items": [_serialize(r) for r in rows[:limit]],
+        "items": [_serialize(r) for r in page],
+        "next_cursor": _encode_cursor(keys, page[-1]) if (has_more and page) else None,
         "has_more": has_more,
-        "offset": offset,
         "limit": limit,
     }
 

frontend/package-lock.json

@@ -10,6 +10,7 @@
         "@dnd-kit/sortable": "^10.0.0",
         "@dnd-kit/utilities": "^3.2.2",
         "@tanstack/react-query": "^5.51.0",
+        "@tanstack/react-virtual": "^3.14.3",
         "clsx": "^2.1.1",
         "i18next": "^23.11.5",
         "lucide-react": "^0.408.0",
@@ -1247,6 +1248,33 @@
         "react": "^18 || ^19"
       }
     },
+    "node_modules/@tanstack/react-virtual": {
+      "version": "3.14.3",
+      "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.14.3.tgz",
+      "integrity": "sha512-k/cnHPVaOfn46hSbiY6n4Dzf4QjCGWSF40zR5QIIYUqPAjpA6TN7InfYmcMiDVQGP2iUn9xsRbAl8u1v3UmeVQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@tanstack/virtual-core": "3.17.1"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
+        "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@tanstack/virtual-core": {
+      "version": "3.17.1",
+      "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.1.tgz",
+      "integrity": "sha512-VZyW2Uiml5tmBZwPGrSD3Sz73OxzljQMCmzYHsUTPEuTsERf5xwa+uWb01xEzkz3ZSYTjj8NEb/mKHvgKxyZdA==",
+      "license": "MIT",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      }
+    },
     "node_modules/@types/babel__core": {
       "version": "7.20.5",
       "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz",

frontend/package.json

@@ -12,6 +12,7 @@
     "@dnd-kit/sortable": "^10.0.0",
     "@dnd-kit/utilities": "^3.2.2",
     "@tanstack/react-query": "^5.51.0",
+    "@tanstack/react-virtual": "^3.14.3",
     "clsx": "^2.1.1",
     "i18next": "^23.11.5",
     "lucide-react": "^0.408.0",

frontend/src/components/Feed.tsx

@@ -5,7 +5,7 @@ import { ArrowDown, ArrowUp, RefreshCw } from "lucide-react";
 import { api, type FeedFilters, type Video } from "../lib/api";
 import i18n from "../i18n";
 import { notify, resolveVideo } from "../lib/notifications";
-import VideoCard from "./VideoCard";
+import VirtualFeed from "./VirtualFeed";
 import PlayerModal from "./PlayerModal";
 
 const PAGE = 60;
@@ -94,9 +94,9 @@ export default function Feed({
 
   const query = useInfiniteQuery({
     queryKey: ["feed", filters],
-    queryFn: ({ pageParam }) => api.feed(filters, pageParam as number, PAGE),
+    queryFn: ({ pageParam }) => api.feed(filters, pageParam as string | null, PAGE),
-    initialPageParam: 0,
+    initialPageParam: null as string | null,
-    getNextPageParam: (last) => (last.has_more ? last.offset + last.limit : undefined),
+    getNextPageParam: (last) => last.next_cursor ?? undefined,
   });
 
   // Drop optimistic status overrides when filters change OR fresh server data
@@ -118,22 +118,7 @@ export default function Feed({
     staleTime: 30_000,
   });
 
-  const sentinel = useRef<HTMLDivElement | null>(null);
   const { hasNextPage, isFetchingNextPage, fetchNextPage } = query;
-  useEffect(() => {
-    const el = sentinel.current;
-    if (!el) return;
-    const io = new IntersectionObserver(
-      (entries) => {
-        if (entries[0].isIntersecting && hasNextPage && !isFetchingNextPage) {
-          fetchNextPage();
-        }
-      },
-      { rootMargin: "1500px" } // prefetch the next page well before the bottom is in view
-    );
-    io.observe(el);
-    return () => io.disconnect();
-  }, [hasNextPage, isFetchingNextPage, fetchNextPage]);
 
   // Keep the loaded videos in a ref so onState can stay referentially stable
   // (it needs the list only to look up a title for the notification). A stable
@@ -388,36 +373,19 @@ export default function Feed({
       {toolbar}
       {items.length === 0 ? (
         <div className="py-16 text-center text-muted">{t("feed.noMatches")}</div>
-      ) : view === "grid" ? (
-        <div className="grid gap-4 grid-cols-[repeat(auto-fill,minmax(260px,1fr))]">
-          {items.map((v) => (
-            <VideoCard
-              key={v.id}
-              video={v}
-              view="grid"
-              onState={onState}
-              onToggleSave={onToggleSave}
-              onChannelFilter={onChannelFilter}
-              onResetState={onResetState}
-              onOpen={openVideo}
-            />
-          ))}
-        </div>
       ) : (
-        <div className="max-w-4xl mx-auto flex flex-col gap-1">
+        <VirtualFeed
-          {items.map((v) => (
+          items={items}
-            <VideoCard
+          view={view}
-              key={v.id}
+          onState={onState}
-              video={v}
+          onToggleSave={onToggleSave}
-              view="list"
+          onChannelFilter={onChannelFilter}
-              onState={onState}
+          onResetState={onResetState}
-              onToggleSave={onToggleSave}
+          onOpen={openVideo}
-              onChannelFilter={onChannelFilter}
+          hasNextPage={!!hasNextPage}
-              onResetState={onResetState}
+          isFetchingNextPage={isFetchingNextPage}
-              onOpen={openVideo}
+          fetchNextPage={fetchNextPage}
-            />
+        />
-          ))}
-        </div>
       )}
       {activeVideo && (
         <PlayerModal
@@ -427,7 +395,6 @@ export default function Feed({
           onState={onState}
         />
       )}
-      <div ref={sentinel} className="h-10" />
       {isFetchingNextPage && (
         <div className="text-center text-muted py-4">{t("feed.loadingMore")}</div>
       )}

frontend/src/components/VirtualFeed.tsx

@@ -0,0 +1,176 @@
+import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
+import { useVirtualizer } from "@tanstack/react-virtual";
+import type { Video } from "../lib/api";
+import VideoCard from "./VideoCard";
+
+// Grid column sizing mirrors the CSS the feed used before virtualization
+// (grid-cols-[repeat(auto-fill,minmax(260px,1fr))] with a gap-4 / 1rem gutter).
+const GRID_MIN_COL = 260;
+const GRID_GAP = 16;
+const LIST_GAP = 4;
+// Estimated row heights before measurement; real heights are measured per-row so
+// these only affect the very first paint and the scrollbar's initial guess.
+const GRID_ROW_EST = 340;
+const LIST_ROW_EST = 96;
+// Start fetching the next page this many rows before the end scrolls into view
+// (keeps the old IntersectionObserver's generous prefetch feel).
+const PREFETCH_ROWS = 4;
+
+function chunk<T>(arr: T[], size: number): T[][] {
+  if (size <= 1) return arr.map((x) => [x]);
+  const out: T[][] = [];
+  for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));
+  return out;
+}
+
+// The feed scrolls inside the app's <main overflow-y-auto>, not the window, so the
+// virtualizer needs that element. Walk up to the nearest scrollable ancestor.
+function getScrollParent(node: HTMLElement | null): HTMLElement {
+  let el = node?.parentElement ?? null;
+  while (el) {
+    const oy = getComputedStyle(el).overflowY;
+    if (oy === "auto" || oy === "scroll") return el;
+    el = el.parentElement;
+  }
+  return document.documentElement;
+}
+
+export interface VirtualFeedProps {
+  items: Video[];
+  view: "grid" | "list";
+  onState: (id: string, status: string) => void;
+  onToggleSave: (id: string, saved: boolean) => void;
+  onChannelFilter: (channelId: string, channelName: string) => void;
+  onResetState: (id: string) => void;
+  onOpen: (v: Video, startAt?: number | null) => void;
+  hasNextPage: boolean;
+  isFetchingNextPage: boolean;
+  fetchNextPage: () => void;
+}
+
+export default function VirtualFeed({
+  items,
+  view,
+  onState,
+  onToggleSave,
+  onChannelFilter,
+  onResetState,
+  onOpen,
+  hasNextPage,
+  isFetchingNextPage,
+  fetchNextPage,
+}: VirtualFeedProps) {
+  const listRef = useRef<HTMLDivElement>(null);
+  const [scrollEl, setScrollEl] = useState<HTMLElement | null>(null);
+  const [colCount, setColCount] = useState(view === "list" ? 1 : 4);
+  // Distance from the scroll element's content top to where this list starts (the
+  // feed toolbar sits above it inside the same scroll area).
+  const [scrollMargin, setScrollMargin] = useState(0);
+
+  useLayoutEffect(() => {
+    if (listRef.current) setScrollEl(getScrollParent(listRef.current));
+  }, []);
+
+  // Re-measure the column count and start offset on container/scroll-area resize.
+  useLayoutEffect(() => {
+    const node = listRef.current;
+    if (!node) return;
+    const measure = () => {
+      if (scrollEl) {
+        const m =
+          node.getBoundingClientRect().top -
+          scrollEl.getBoundingClientRect().top +
+          scrollEl.scrollTop;
+        setScrollMargin(m);
+      }
+      setColCount(
+        view === "list"
+          ? 1
+          : Math.max(1, Math.floor((node.clientWidth + GRID_GAP) / (GRID_MIN_COL + GRID_GAP)))
+      );
+    };
+    measure();
+    const ro = new ResizeObserver(measure);
+    ro.observe(node);
+    if (scrollEl) ro.observe(scrollEl);
+    return () => ro.disconnect();
+  }, [scrollEl, view]);
+
+  const rows = useMemo(() => chunk(items, colCount), [items, colCount]);
+
+  const virtualizer = useVirtualizer({
+    count: rows.length,
+    getScrollElement: () => scrollEl,
+    estimateSize: () => (view === "grid" ? GRID_ROW_EST : LIST_ROW_EST),
+    overscan: 4,
+    gap: view === "grid" ? GRID_GAP : LIST_GAP,
+    scrollMargin,
+  });
+
+  const virtualRows = virtualizer.getVirtualItems();
+
+  // Infinite-scroll trigger: once the last rendered row is within PREFETCH_ROWS of
+  // the end, pull the next keyset page.
+  useEffect(() => {
+    const last = virtualRows[virtualRows.length - 1];
+    if (!last) return;
+    if (last.index >= rows.length - 1 - PREFETCH_ROWS && hasNextPage && !isFetchingNextPage) {
+      fetchNextPage();
+    }
+  }, [virtualRows, rows.length, hasNextPage, isFetchingNextPage, fetchNextPage]);
+
+  return (
+    <div ref={listRef} style={{ height: virtualizer.getTotalSize(), position: "relative" }}>
+      {virtualRows.map((vr) => {
+        const row = rows[vr.index];
+        if (!row) return null;
+        return (
+          <div
+            key={vr.key}
+            data-index={vr.index}
+            ref={virtualizer.measureElement}
+            style={{
+              position: "absolute",
+              top: 0,
+              left: 0,
+              width: "100%",
+              transform: `translateY(${vr.start - virtualizer.options.scrollMargin}px)`,
+            }}
+          >
+            {view === "grid" ? (
+              <div
+                className="grid gap-4"
+                style={{ gridTemplateColumns: `repeat(${colCount}, minmax(0, 1fr))` }}
+              >
+                {row.map((v) => (
+                  <VideoCard
+                    key={v.id}
+                    video={v}
+                    view="grid"
+                    onState={onState}
+                    onToggleSave={onToggleSave}
+                    onChannelFilter={onChannelFilter}
+                    onResetState={onResetState}
+                    onOpen={onOpen}
+                  />
+                ))}
+              </div>
+            ) : (
+              <div className="max-w-4xl mx-auto pb-1">
+                <VideoCard
+                  video={row[0]}
+                  view="list"
+                  onState={onState}
+                  onToggleSave={onToggleSave}
+                  onChannelFilter={onChannelFilter}
+                  onResetState={onResetState}
+                  onOpen={onOpen}
+                />
+              </div>
+            )}
+          </div>
+        );
+      })}
+    </div>
+  );
+}

frontend/src/lib/api.ts

@@ -79,7 +79,8 @@ export interface VideoDetail {
 export interface FeedResponse {
   items: Video[];
   has_more: boolean;
-  offset: number;
+  // Opaque keyset cursor for the next page, or null when this is the last page.
+  next_cursor: string | null;
   limit: number;
 }
 
@@ -308,11 +309,11 @@ function filterParams(f: FeedFilters): URLSearchParams {
   return p;
 }
 
-function feedQuery(f: FeedFilters, offset: number, limit: number): string {
+function feedQuery(f: FeedFilters, cursor: string | null, limit: number): string {
   const p = filterParams(f);
   p.set("sort", f.sort);
   if (f.sort === "shuffle" && f.seed) p.set("seed", String(f.seed));
-  p.set("offset", String(offset));
+  if (cursor) p.set("cursor", cursor);
   p.set("limit", String(limit));
   return p.toString();
 }
@@ -511,10 +512,10 @@ export const api = {
   version: (): Promise<VersionInfo> => req("/api/version"),
   tags: (): Promise<Tag[]> => req("/api/tags"),
   status: (): Promise<SyncStatus> => req("/api/sync/status"),
-  feed: (f: FeedFilters, offset: number, limit: number): Promise<FeedResponse> =>
+  feed: (f: FeedFilters, cursor: string | null, limit: number): Promise<FeedResponse> =>
-    req(`/api/feed?${feedQuery(f, offset, limit)}`),
+    req(`/api/feed?${feedQuery(f, cursor, limit)}`),
   feedCount: (f: FeedFilters): Promise<{ count: number }> =>
-    req(`/api/feed/count?${feedQuery(f, 0, 0)}`),
+    req(`/api/feed/count?${filterParams(f).toString()}`),
   facets: (f: FeedFilters): Promise<{ counts: Record<string, number> }> =>
     req(`/api/facets?${filterParams(f).toString()}`),
   // idempotent: setting a state / saving a progress checkpoint is safe to replay, so these