Merge feature/feed-virtualization-keyset — feed virtualization + keyset pagination (epic 7)

This commit is contained in:
npeter83 2026-06-25 20:03:44 +02:00
commit 64a57ed29e
6 changed files with 333 additions and 85 deletions

View file

@ -1,3 +1,6 @@
import base64
import binascii
import json
from datetime import date, datetime, timedelta, timezone
from fastapi import APIRouter, Depends, HTTPException, Query
@ -278,21 +281,92 @@ def _feed_params(
}
SORTS = {
"newest": Video.published_at.desc().nulls_last(),
"oldest": Video.published_at.asc().nulls_last(),
"views": Video.view_count.desc().nulls_last(),
"views_asc": Video.view_count.asc().nulls_last(),
"duration_desc": Video.duration_seconds.desc().nulls_last(),
"duration_asc": Video.duration_seconds.asc().nulls_last(),
"title": func.lower(Video.title).asc().nulls_last(),
"title_desc": func.lower(Video.title).desc().nulls_last(),
"subscribers": Channel.subscriber_count.desc().nulls_last(),
"subscribers_asc": Channel.subscriber_count.asc().nulls_last(),
# Your per-channel priority (set in the channel manager), newest first within a tier.
# coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
"priority": func.coalesce(Subscription.priority, 0).desc(),
}
# --- Keyset (cursor) pagination ---------------------------------------------
#
# Each sort is described as an ordered list of key columns (expression + direction
# + whether it can be NULL + a JSON round-trip kind), always with Video.id as the
# final, unique tiebreaker. This drives BOTH the ORDER BY and the keyset WHERE so
# they can never drift apart — the prerequisite for correct cursor paging. Keyset
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
# (OFFSET would shift and duplicate/skip rows).
def _sort_keys(sort: str, seed: int) -> list[dict]:
pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"}
table: dict[str, list[dict]] = {
"newest": [pub],
"oldest": [{**pub, "asc": True}],
"views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
"views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
"duration_desc": [{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}],
"duration_asc": [{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}],
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
"title_desc": [{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}],
"subscribers": [{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}],
"subscribers_asc": [{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}],
# Your per-channel priority (set in the channel manager), newest first within a tier.
# coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
"priority": [{"expr": func.coalesce(Subscription.priority, 0), "asc": False, "nullable": False, "kind": "num"}, pub],
"priority_asc": [{"expr": func.coalesce(Subscription.priority, 0), "asc": True, "nullable": False, "kind": "num"}, pub],
# Deterministic per-seed shuffle: the same seed yields the same total order, so
# the cursor stays valid across pages of one shuffled run.
"shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
}
return table.get(sort) or table["newest"]
def _order_by(keys: list[dict]):
cols = []
for k in keys:
c = k["expr"].asc() if k["asc"] else k["expr"].desc()
cols.append(c.nulls_last() if k["nullable"] else c)
cols.append(Video.id.asc()) # unique final tiebreaker
return cols
def _keyset_predicate(keys: list[dict], cursor_id: str):
"""WHERE clause selecting rows strictly AFTER the cursor in the sort order.
Lexicographic over the key columns then id, NULL-aware for `nulls_last`: a NULL
sorts after every non-NULL value, so a cursor sitting on a non-NULL value must
also pull in the NULL group, and equality treats NULL == NULL."""
def eq(k):
return k["expr"].is_(None) if k["value"] is None else k["expr"] == k["value"]
def after(k):
if k["value"] is None:
return false() # nulls are last; nothing sorts strictly after them
cmp = k["expr"] > k["value"] if k["asc"] else k["expr"] < k["value"]
return or_(cmp, k["expr"].is_(None)) if k["nullable"] else cmp
branches = []
for j in range(len(keys)):
branches.append(and_(*[eq(keys[i]) for i in range(j)], after(keys[j])))
branches.append(and_(*[eq(k) for k in keys], Video.id > cursor_id))
return or_(*branches)
def _encode_cursor(keys: list[dict], row) -> str:
vals = []
for i, k in enumerate(keys):
v = getattr(row, f"_k{i}")
vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v)
payload = json.dumps({"k": vals, "id": row.id})
return base64.urlsafe_b64encode(payload.encode()).decode()
def _decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str]:
try:
payload = json.loads(base64.urlsafe_b64decode(cursor.encode()))
raw = payload["k"]
if len(raw) != len(keys):
raise ValueError("key arity mismatch")
vals = [
datetime.fromisoformat(v) if (v is not None and k["kind"] == "dt") else v
for k, v in zip(keys, raw)
]
return vals, payload["id"]
except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError):
raise HTTPException(status_code=400, detail="Invalid feed cursor")
@router.get("/feed")
@ -301,29 +375,30 @@ def get_feed(
sort: str = "newest",
seed: int = 0,
limit: int = Query(default=60, le=200),
offset: int = 0,
cursor: str | None = None,
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
query, _status = _filtered_query(db, user, **params)
if sort in ("priority", "priority_asc"):
prio = func.coalesce(Subscription.priority, 0)
query = query.order_by(
prio.asc() if sort == "priority_asc" else prio.desc(),
Video.published_at.desc().nulls_last(),
)
else:
order = SORTS.get(sort)
if order is None and sort == "shuffle":
order = func.md5(func.concat(Video.id, str(seed)))
query = query.order_by(order if order is not None else SORTS["newest"])
keys = _sort_keys(sort, seed)
rows = db.execute(query.offset(offset).limit(limit + 1)).all()
# Expose each key column on the row so we can build the next cursor from the last item.
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
query = query.order_by(*_order_by(keys))
if cursor:
vals, cursor_id = _decode_cursor(keys, cursor)
for k, v in zip(keys, vals):
k["value"] = v
query = query.where(_keyset_predicate(keys, cursor_id))
rows = db.execute(query.limit(limit + 1)).all()
has_more = len(rows) > limit
page = rows[:limit]
return {
"items": [_serialize(r) for r in rows[:limit]],
"items": [_serialize(r) for r in page],
"next_cursor": _encode_cursor(keys, page[-1]) if (has_more and page) else None,
"has_more": has_more,
"offset": offset,
"limit": limit,
}

View file

@ -10,6 +10,7 @@
"@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2",
"@tanstack/react-query": "^5.51.0",
"@tanstack/react-virtual": "^3.14.3",
"clsx": "^2.1.1",
"i18next": "^23.11.5",
"lucide-react": "^0.408.0",
@ -1247,6 +1248,33 @@
"react": "^18 || ^19"
}
},
"node_modules/@tanstack/react-virtual": {
"version": "3.14.3",
"resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.14.3.tgz",
"integrity": "sha512-k/cnHPVaOfn46hSbiY6n4Dzf4QjCGWSF40zR5QIIYUqPAjpA6TN7InfYmcMiDVQGP2iUn9xsRbAl8u1v3UmeVQ==",
"license": "MIT",
"dependencies": {
"@tanstack/virtual-core": "3.17.1"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/tannerlinsley"
},
"peerDependencies": {
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
"react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
}
},
"node_modules/@tanstack/virtual-core": {
"version": "3.17.1",
"resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.1.tgz",
"integrity": "sha512-VZyW2Uiml5tmBZwPGrSD3Sz73OxzljQMCmzYHsUTPEuTsERf5xwa+uWb01xEzkz3ZSYTjj8NEb/mKHvgKxyZdA==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/tannerlinsley"
}
},
"node_modules/@types/babel__core": {
"version": "7.20.5",
"resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz",

View file

@ -12,6 +12,7 @@
"@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2",
"@tanstack/react-query": "^5.51.0",
"@tanstack/react-virtual": "^3.14.3",
"clsx": "^2.1.1",
"i18next": "^23.11.5",
"lucide-react": "^0.408.0",

View file

@ -5,7 +5,7 @@ import { ArrowDown, ArrowUp, RefreshCw } from "lucide-react";
import { api, type FeedFilters, type Video } from "../lib/api";
import i18n from "../i18n";
import { notify, resolveVideo } from "../lib/notifications";
import VideoCard from "./VideoCard";
import VirtualFeed from "./VirtualFeed";
import PlayerModal from "./PlayerModal";
const PAGE = 60;
@ -94,9 +94,9 @@ export default function Feed({
const query = useInfiniteQuery({
queryKey: ["feed", filters],
queryFn: ({ pageParam }) => api.feed(filters, pageParam as number, PAGE),
initialPageParam: 0,
getNextPageParam: (last) => (last.has_more ? last.offset + last.limit : undefined),
queryFn: ({ pageParam }) => api.feed(filters, pageParam as string | null, PAGE),
initialPageParam: null as string | null,
getNextPageParam: (last) => last.next_cursor ?? undefined,
});
// Drop optimistic status overrides when filters change OR fresh server data
@ -118,22 +118,7 @@ export default function Feed({
staleTime: 30_000,
});
const sentinel = useRef<HTMLDivElement | null>(null);
const { hasNextPage, isFetchingNextPage, fetchNextPage } = query;
useEffect(() => {
const el = sentinel.current;
if (!el) return;
const io = new IntersectionObserver(
(entries) => {
if (entries[0].isIntersecting && hasNextPage && !isFetchingNextPage) {
fetchNextPage();
}
},
{ rootMargin: "1500px" } // prefetch the next page well before the bottom is in view
);
io.observe(el);
return () => io.disconnect();
}, [hasNextPage, isFetchingNextPage, fetchNextPage]);
// Keep the loaded videos in a ref so onState can stay referentially stable
// (it needs the list only to look up a title for the notification). A stable
@ -388,36 +373,19 @@ export default function Feed({
{toolbar}
{items.length === 0 ? (
<div className="py-16 text-center text-muted">{t("feed.noMatches")}</div>
) : view === "grid" ? (
<div className="grid gap-4 grid-cols-[repeat(auto-fill,minmax(260px,1fr))]">
{items.map((v) => (
<VideoCard
key={v.id}
video={v}
view="grid"
onState={onState}
onToggleSave={onToggleSave}
onChannelFilter={onChannelFilter}
onResetState={onResetState}
onOpen={openVideo}
/>
))}
</div>
) : (
<div className="max-w-4xl mx-auto flex flex-col gap-1">
{items.map((v) => (
<VideoCard
key={v.id}
video={v}
view="list"
onState={onState}
onToggleSave={onToggleSave}
onChannelFilter={onChannelFilter}
onResetState={onResetState}
onOpen={openVideo}
/>
))}
</div>
<VirtualFeed
items={items}
view={view}
onState={onState}
onToggleSave={onToggleSave}
onChannelFilter={onChannelFilter}
onResetState={onResetState}
onOpen={openVideo}
hasNextPage={!!hasNextPage}
isFetchingNextPage={isFetchingNextPage}
fetchNextPage={fetchNextPage}
/>
)}
{activeVideo && (
<PlayerModal
@ -427,7 +395,6 @@ export default function Feed({
onState={onState}
/>
)}
<div ref={sentinel} className="h-10" />
{isFetchingNextPage && (
<div className="text-center text-muted py-4">{t("feed.loadingMore")}</div>
)}

View file

@ -0,0 +1,176 @@
import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
import { useVirtualizer } from "@tanstack/react-virtual";
import type { Video } from "../lib/api";
import VideoCard from "./VideoCard";
// Grid column sizing mirrors the CSS the feed used before virtualization
// (grid-cols-[repeat(auto-fill,minmax(260px,1fr))] with a gap-4 / 1rem gutter).
const GRID_MIN_COL = 260;
const GRID_GAP = 16;
const LIST_GAP = 4;
// Estimated row heights before measurement; real heights are measured per-row so
// these only affect the very first paint and the scrollbar's initial guess.
const GRID_ROW_EST = 340;
const LIST_ROW_EST = 96;
// Start fetching the next page this many rows before the end scrolls into view
// (keeps the old IntersectionObserver's generous prefetch feel).
const PREFETCH_ROWS = 4;
function chunk<T>(arr: T[], size: number): T[][] {
if (size <= 1) return arr.map((x) => [x]);
const out: T[][] = [];
for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));
return out;
}
// The feed scrolls inside the app's <main overflow-y-auto>, not the window, so the
// virtualizer needs that element. Walk up to the nearest scrollable ancestor.
function getScrollParent(node: HTMLElement | null): HTMLElement {
let el = node?.parentElement ?? null;
while (el) {
const oy = getComputedStyle(el).overflowY;
if (oy === "auto" || oy === "scroll") return el;
el = el.parentElement;
}
return document.documentElement;
}
export interface VirtualFeedProps {
items: Video[];
view: "grid" | "list";
onState: (id: string, status: string) => void;
onToggleSave: (id: string, saved: boolean) => void;
onChannelFilter: (channelId: string, channelName: string) => void;
onResetState: (id: string) => void;
onOpen: (v: Video, startAt?: number | null) => void;
hasNextPage: boolean;
isFetchingNextPage: boolean;
fetchNextPage: () => void;
}
export default function VirtualFeed({
items,
view,
onState,
onToggleSave,
onChannelFilter,
onResetState,
onOpen,
hasNextPage,
isFetchingNextPage,
fetchNextPage,
}: VirtualFeedProps) {
const listRef = useRef<HTMLDivElement>(null);
const [scrollEl, setScrollEl] = useState<HTMLElement | null>(null);
const [colCount, setColCount] = useState(view === "list" ? 1 : 4);
// Distance from the scroll element's content top to where this list starts (the
// feed toolbar sits above it inside the same scroll area).
const [scrollMargin, setScrollMargin] = useState(0);
useLayoutEffect(() => {
if (listRef.current) setScrollEl(getScrollParent(listRef.current));
}, []);
// Re-measure the column count and start offset on container/scroll-area resize.
useLayoutEffect(() => {
const node = listRef.current;
if (!node) return;
const measure = () => {
if (scrollEl) {
const m =
node.getBoundingClientRect().top -
scrollEl.getBoundingClientRect().top +
scrollEl.scrollTop;
setScrollMargin(m);
}
setColCount(
view === "list"
? 1
: Math.max(1, Math.floor((node.clientWidth + GRID_GAP) / (GRID_MIN_COL + GRID_GAP)))
);
};
measure();
const ro = new ResizeObserver(measure);
ro.observe(node);
if (scrollEl) ro.observe(scrollEl);
return () => ro.disconnect();
}, [scrollEl, view]);
const rows = useMemo(() => chunk(items, colCount), [items, colCount]);
const virtualizer = useVirtualizer({
count: rows.length,
getScrollElement: () => scrollEl,
estimateSize: () => (view === "grid" ? GRID_ROW_EST : LIST_ROW_EST),
overscan: 4,
gap: view === "grid" ? GRID_GAP : LIST_GAP,
scrollMargin,
});
const virtualRows = virtualizer.getVirtualItems();
// Infinite-scroll trigger: once the last rendered row is within PREFETCH_ROWS of
// the end, pull the next keyset page.
useEffect(() => {
const last = virtualRows[virtualRows.length - 1];
if (!last) return;
if (last.index >= rows.length - 1 - PREFETCH_ROWS && hasNextPage && !isFetchingNextPage) {
fetchNextPage();
}
}, [virtualRows, rows.length, hasNextPage, isFetchingNextPage, fetchNextPage]);
return (
<div ref={listRef} style={{ height: virtualizer.getTotalSize(), position: "relative" }}>
{virtualRows.map((vr) => {
const row = rows[vr.index];
if (!row) return null;
return (
<div
key={vr.key}
data-index={vr.index}
ref={virtualizer.measureElement}
style={{
position: "absolute",
top: 0,
left: 0,
width: "100%",
transform: `translateY(${vr.start - virtualizer.options.scrollMargin}px)`,
}}
>
{view === "grid" ? (
<div
className="grid gap-4"
style={{ gridTemplateColumns: `repeat(${colCount}, minmax(0, 1fr))` }}
>
{row.map((v) => (
<VideoCard
key={v.id}
video={v}
view="grid"
onState={onState}
onToggleSave={onToggleSave}
onChannelFilter={onChannelFilter}
onResetState={onResetState}
onOpen={onOpen}
/>
))}
</div>
) : (
<div className="max-w-4xl mx-auto pb-1">
<VideoCard
video={row[0]}
view="list"
onState={onState}
onToggleSave={onToggleSave}
onChannelFilter={onChannelFilter}
onResetState={onResetState}
onOpen={onOpen}
/>
</div>
)}
</div>
);
})}
</div>
);
}

View file

@ -79,7 +79,8 @@ export interface VideoDetail {
export interface FeedResponse {
items: Video[];
has_more: boolean;
offset: number;
// Opaque keyset cursor for the next page, or null when this is the last page.
next_cursor: string | null;
limit: number;
}
@ -308,11 +309,11 @@ function filterParams(f: FeedFilters): URLSearchParams {
return p;
}
function feedQuery(f: FeedFilters, offset: number, limit: number): string {
function feedQuery(f: FeedFilters, cursor: string | null, limit: number): string {
const p = filterParams(f);
p.set("sort", f.sort);
if (f.sort === "shuffle" && f.seed) p.set("seed", String(f.seed));
p.set("offset", String(offset));
if (cursor) p.set("cursor", cursor);
p.set("limit", String(limit));
return p.toString();
}
@ -511,10 +512,10 @@ export const api = {
version: (): Promise<VersionInfo> => req("/api/version"),
tags: (): Promise<Tag[]> => req("/api/tags"),
status: (): Promise<SyncStatus> => req("/api/sync/status"),
feed: (f: FeedFilters, offset: number, limit: number): Promise<FeedResponse> =>
req(`/api/feed?${feedQuery(f, offset, limit)}`),
feed: (f: FeedFilters, cursor: string | null, limit: number): Promise<FeedResponse> =>
req(`/api/feed?${feedQuery(f, cursor, limit)}`),
feedCount: (f: FeedFilters): Promise<{ count: number }> =>
req(`/api/feed/count?${feedQuery(f, 0, 0)}`),
req(`/api/feed/count?${filterParams(f).toString()}`),
facets: (f: FeedFilters): Promise<{ counts: Record<string, number> }> =>
req(`/api/facets?${filterParams(f).toString()}`),
// idempotent: setting a state / saving a progress checkpoint is safe to replay, so these