feat(m5d): demand-driven deep backfill + per-user ETA
Per-user opt-in to full-history (deep) backfill so a new user's unique channels no longer trigger a big shared-quota burst. - migration 0007: Subscription.deep_requested (default false); seed admins' existing subscriptions to preserve today's global-backfill behaviour - run_deep_backfill is now demand-driven: only channels at least one user has requested are deep-backfilled; recent backfill stays unconditional (cheap) - estimate_deep_backfill ETA helper (quota-bound) surfaced in /api/sync/my-status - POST /api/sync/deep-all to opt all my channels in; PATCH channels accepts deep_requested - UI: per-channel Full history toggle, Backfill everything action, deep progress + ETA in Channels header and Settings - Sync
This commit is contained in:
npeter83
parent
bc8db45323
commit
5f60b3e9fe
9 changed files with 256 additions and 18 deletions
|
|
@ -5,12 +5,14 @@ import {
|
|||
ArrowUp,
|
||||
Eye,
|
||||
EyeOff,
|
||||
History,
|
||||
Plus,
|
||||
RefreshCw,
|
||||
Search,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { api, type ManagedChannel, type Tag } from "../lib/api";
|
||||
import { formatEta } from "../lib/format";
|
||||
import { notify } from "../lib/notifications";
|
||||
import Tooltip from "./Tooltip";
|
||||
|
||||
|
|
@ -35,8 +37,10 @@ export default function Channels({
|
|||
const userTags = (tagsQuery.data ?? []).filter((t) => !t.system);
|
||||
|
||||
const patch = useMutation({
|
||||
mutationFn: (v: { id: string; body: { priority?: number; hidden?: boolean } }) =>
|
||||
api.updateChannel(v.id, v.body),
|
||||
mutationFn: (v: {
|
||||
id: string;
|
||||
body: { priority?: number; hidden?: boolean; deep_requested?: boolean };
|
||||
}) => api.updateChannel(v.id, v.body),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: ["channels"] }),
|
||||
});
|
||||
const attach = useMutation({
|
||||
|
|
@ -66,6 +70,18 @@ export default function Channels({
|
|||
mutationFn: (id: number) => api.deleteTag(id),
|
||||
onSuccess: () => invalidate(),
|
||||
});
|
||||
const deepAll = useMutation({
|
||||
mutationFn: () => api.deepAll(true),
|
||||
onSuccess: (r: { updated?: number }) => {
|
||||
qc.invalidateQueries({ queryKey: ["channels"] });
|
||||
qc.invalidateQueries({ queryKey: ["my-status"] });
|
||||
notify({
|
||||
level: "success",
|
||||
message: `Full history requested for ${r.updated ?? 0} channels`,
|
||||
});
|
||||
},
|
||||
onError: () => notify({ level: "error", message: "Couldn't request full history" }),
|
||||
});
|
||||
|
||||
const channels = (channelsQuery.data ?? []).filter(
|
||||
(c) => !q || (c.title ?? "").toLowerCase().includes(q.toLowerCase())
|
||||
|
|
@ -85,9 +101,16 @@ export default function Channels({
|
|||
/>
|
||||
<Stat
|
||||
label="Full history"
|
||||
value={`${s.channels_deep_done}/${s.channels_total}`}
|
||||
hint="Channels whose entire back-catalog is fetched. The rest backfill gradually."
|
||||
value={`${s.channels_deep_done}/${s.channels_deep_requested}`}
|
||||
hint="Channels whose entire back-catalog is fetched, out of the ones you've requested full history for."
|
||||
/>
|
||||
{s.deep_pending_count > 0 && (
|
||||
<Stat
|
||||
label="left"
|
||||
value={formatEta(s.deep_eta_seconds)}
|
||||
hint={`Rough estimate to finish full-history backfill of ${s.deep_pending_count} pending channel(s), based on the shared daily quota.`}
|
||||
/>
|
||||
)}
|
||||
<Stat label="My videos" value={s.my_videos.toLocaleString()} hint="Total videos available across your channels." />
|
||||
<Stat
|
||||
label="Quota left"
|
||||
|
|
@ -121,6 +144,19 @@ export default function Channels({
|
|||
Sync subscriptions
|
||||
</button>
|
||||
</Tooltip>
|
||||
<Tooltip
|
||||
side="bottom"
|
||||
hint="Request full back-catalog backfill for every channel you're subscribed to. Older videos and search become complete as the shared daily quota allows — this can take a while."
|
||||
>
|
||||
<button
|
||||
onClick={() => deepAll.mutate()}
|
||||
disabled={deepAll.isPending}
|
||||
className="glass-card glass-hover flex items-center gap-2 px-3 py-2 rounded-xl text-sm disabled:opacity-50 transition"
|
||||
>
|
||||
<History className={`w-4 h-4 ${deepAll.isPending ? "animate-pulse" : ""}`} />
|
||||
Backfill everything
|
||||
</button>
|
||||
</Tooltip>
|
||||
</div>
|
||||
|
||||
<p className="text-xs text-muted mb-4 leading-relaxed">
|
||||
|
|
@ -181,6 +217,9 @@ export default function Channels({
|
|||
onView={() => onViewChannel(c.id, c.title ?? "This channel")}
|
||||
onPriority={(d) => patch.mutate({ id: c.id, body: { priority: c.priority + d } })}
|
||||
onHide={() => patch.mutate({ id: c.id, body: { hidden: !c.hidden } })}
|
||||
onDeep={() =>
|
||||
patch.mutate({ id: c.id, body: { deep_requested: !c.deep_requested } })
|
||||
}
|
||||
onToggleTag={(tagId) =>
|
||||
c.tag_ids.includes(tagId)
|
||||
? detach.mutate({ id: c.id, tagId })
|
||||
|
|
@ -232,6 +271,7 @@ function ChannelRow({
|
|||
onView,
|
||||
onPriority,
|
||||
onHide,
|
||||
onDeep,
|
||||
onToggleTag,
|
||||
}: {
|
||||
c: ManagedChannel;
|
||||
|
|
@ -239,6 +279,7 @@ function ChannelRow({
|
|||
onView: () => void;
|
||||
onPriority: (delta: number) => void;
|
||||
onHide: () => void;
|
||||
onDeep: () => void;
|
||||
onToggleTag: (tagId: number) => void;
|
||||
}) {
|
||||
return (
|
||||
|
|
@ -279,11 +320,29 @@ function ChannelRow({
|
|||
label="recent"
|
||||
hint={c.recent_synced ? "Recent uploads synced." : "Recent uploads not fetched yet."}
|
||||
/>
|
||||
<SyncBadge
|
||||
ok={c.backfill_done}
|
||||
label="full"
|
||||
hint={c.backfill_done ? "Full history fetched." : "Only recent videos so far; full history pending."}
|
||||
/>
|
||||
{c.backfill_done ? (
|
||||
<SyncBadge ok label="full" hint="Full history fetched." />
|
||||
) : (
|
||||
<Tooltip
|
||||
hint={
|
||||
c.deep_requested
|
||||
? "Full history requested — this channel's whole back-catalog will backfill as the shared quota allows. Click to cancel the request."
|
||||
: "Only recent uploads so far. Click to request this channel's full back-catalog (older videos + complete search)."
|
||||
}
|
||||
>
|
||||
<button
|
||||
onClick={onDeep}
|
||||
className={`inline-flex items-center gap-1 text-[10px] px-1.5 py-0.5 rounded-full border transition ${
|
||||
c.deep_requested
|
||||
? "bg-accent text-accent-fg border-accent"
|
||||
: "bg-card border-border text-muted hover:border-accent"
|
||||
}`}
|
||||
>
|
||||
<History className="w-3 h-3" />
|
||||
{c.deep_requested ? "full history queued" : "get full history"}
|
||||
</button>
|
||||
</Tooltip>
|
||||
)}
|
||||
{userTags.map((t) => {
|
||||
const on = c.tag_ids.includes(t.id);
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
import { useCallback, useEffect, useState, useSyncExternalStore } from "react";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { Bell, Monitor, Pause, Play, RefreshCw, User, X } from "lucide-react";
|
||||
import { Bell, History, Monitor, Pause, Play, RefreshCw, User, X } from "lucide-react";
|
||||
import { SCHEMES, type Scheme, type ThemePrefs } from "../lib/theme";
|
||||
import { api, type Me } from "../lib/api";
|
||||
import { formatEta } from "../lib/format";
|
||||
import {
|
||||
configureNotifications,
|
||||
getNotifSettings,
|
||||
|
|
@ -335,6 +336,18 @@ function Sync({ me }: { me: Me }) {
|
|||
mutationFn: (paused: boolean) => (paused ? api.resumeSync() : api.pauseSync()),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: ["my-status"] }),
|
||||
});
|
||||
const deepAll = useMutation({
|
||||
mutationFn: () => api.deepAll(true),
|
||||
onSuccess: (r: { updated?: number }) => {
|
||||
qc.invalidateQueries({ queryKey: ["my-status"] });
|
||||
qc.invalidateQueries({ queryKey: ["channels"] });
|
||||
notify({
|
||||
level: "success",
|
||||
message: `Full history requested for ${r.updated ?? 0} channels`,
|
||||
});
|
||||
},
|
||||
onError: () => notify({ level: "error", message: "Couldn't request full history" }),
|
||||
});
|
||||
|
||||
return (
|
||||
<>
|
||||
|
|
@ -352,10 +365,18 @@ function Sync({ me }: { me: Me }) {
|
|||
</Row>
|
||||
<Row
|
||||
label="Full history"
|
||||
hint="Channels whose entire back-catalog has been fetched. The rest backfill gradually (opt in per channel)."
|
||||
hint="Channels whose entire back-catalog has been fetched, out of the ones you've requested full history for (opt in per channel on the Channels page)."
|
||||
>
|
||||
{`${s.channels_deep_done}/${s.channels_total}`}
|
||||
{`${s.channels_deep_done}/${s.channels_deep_requested}`}
|
||||
</Row>
|
||||
{s.deep_pending_count > 0 && (
|
||||
<Row
|
||||
label="Full history ETA"
|
||||
hint={`Rough estimate to finish full-history backfill of ${s.deep_pending_count} pending channel(s), based on the shared daily quota.`}
|
||||
>
|
||||
{formatEta(s.deep_eta_seconds)}
|
||||
</Row>
|
||||
)}
|
||||
<Row label="My videos" hint="Total videos available to you across your subscribed channels.">
|
||||
{s.my_videos.toLocaleString()}
|
||||
</Row>
|
||||
|
|
@ -382,6 +403,16 @@ function Sync({ me }: { me: Me }) {
|
|||
Sync subscriptions
|
||||
</button>
|
||||
</Tooltip>
|
||||
<Tooltip hint="Request full back-catalog backfill for every channel you're subscribed to. Older videos + complete search fill in as the shared daily quota allows.">
|
||||
<button
|
||||
onClick={() => deepAll.mutate()}
|
||||
disabled={deepAll.isPending}
|
||||
className="glass-card glass-hover flex items-center gap-2 px-3 py-2 rounded-xl text-sm disabled:opacity-50 transition mt-2"
|
||||
>
|
||||
<History className={`w-4 h-4 ${deepAll.isPending ? "animate-pulse" : ""}`} />
|
||||
Backfill everything
|
||||
</button>
|
||||
</Tooltip>
|
||||
</Section>
|
||||
|
||||
{me.role === "admin" && s && (
|
||||
|
|
|
|||
|
|
@ -144,6 +144,9 @@ export interface MyStatus {
|
|||
channels_deep_done: number;
|
||||
channels_recent_pending: number;
|
||||
channels_deep_pending: number;
|
||||
channels_deep_requested: number;
|
||||
deep_pending_count: number;
|
||||
deep_eta_seconds: number;
|
||||
my_videos: number;
|
||||
quota_used_today: number;
|
||||
quota_remaining_today: number;
|
||||
|
|
@ -160,6 +163,7 @@ export interface ManagedChannel {
|
|||
stored_videos: number;
|
||||
priority: number;
|
||||
hidden: boolean;
|
||||
deep_requested: boolean;
|
||||
tag_ids: number[];
|
||||
details_synced: boolean;
|
||||
recent_synced: boolean;
|
||||
|
|
@ -184,8 +188,12 @@ export const api = {
|
|||
// --- channel manager ---
|
||||
myStatus: (): Promise<MyStatus> => req("/api/sync/my-status"),
|
||||
channels: (): Promise<ManagedChannel[]> => req("/api/channels"),
|
||||
updateChannel: (id: string, patch: { priority?: number; hidden?: boolean }) =>
|
||||
req(`/api/channels/${id}`, { method: "PATCH", body: JSON.stringify(patch) }),
|
||||
updateChannel: (
|
||||
id: string,
|
||||
patch: { priority?: number; hidden?: boolean; deep_requested?: boolean }
|
||||
) => req(`/api/channels/${id}`, { method: "PATCH", body: JSON.stringify(patch) }),
|
||||
deepAll: (on = true) =>
|
||||
req(`/api/sync/deep-all?on=${on}`, { method: "POST" }),
|
||||
attachChannelTag: (id: string, tagId: number) =>
|
||||
req(`/api/channels/${id}/tags`, { method: "POST", body: JSON.stringify({ tag_id: tagId }) }),
|
||||
detachChannelTag: (id: string, tagId: number) =>
|
||||
|
|
|
|||
|
|
@ -34,6 +34,19 @@ export function formatDuration(sec: number | null): string {
|
|||
return h > 0 ? `${h}:${pad(m)}:${pad(s)}` : `${m}:${pad(s)}`;
|
||||
}
|
||||
|
||||
/** Coarse, human ETA for a remaining duration in seconds ("~3 hours", "~2 days"). */
|
||||
export function formatEta(seconds: number): string {
|
||||
if (seconds <= 0) return "done";
|
||||
const hours = seconds / 3600;
|
||||
if (hours < 1) return "< 1 hour";
|
||||
if (hours < 24) {
|
||||
const h = Math.round(hours);
|
||||
return `~${h} hour${h === 1 ? "" : "s"}`;
|
||||
}
|
||||
const d = Math.round(hours / 24);
|
||||
return `~${d} day${d === 1 ? "" : "s"}`;
|
||||
}
|
||||
|
||||
export function formatViews(n: number | null): string {
|
||||
if (n == null) return "";
|
||||
if (n >= 1e9) return `${(n / 1e9).toFixed(1).replace(/\.0$/, "")}B`;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue