diff --git a/backend/alembic/versions/0009_quota_events.py b/backend/alembic/versions/0009_quota_events.py new file mode 100644 index 0000000..740956f --- /dev/null +++ b/backend/alembic/versions/0009_quota_events.py @@ -0,0 +1,46 @@ +"""quota_events: per-spend audit log for per-user quota attribution + +Revision ID: 0009_quota_events +Revises: 0008_invites +Create Date: 2026-06-12 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "0009_quota_events" +down_revision: Union[str, None] = "0008_invites" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "quota_events", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column("action", sa.String(length=40), nullable=False), + sa.Column("units", sa.Integer(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + ) + op.create_index("ix_quota_events_user_id", "quota_events", ["user_id"]) + op.create_index("ix_quota_events_action", "quota_events", ["action"]) + op.create_index("ix_quota_events_created_at", "quota_events", ["created_at"]) + + +def downgrade() -> None: + op.drop_index("ix_quota_events_created_at", table_name="quota_events") + op.drop_index("ix_quota_events_action", table_name="quota_events") + op.drop_index("ix_quota_events_user_id", table_name="quota_events") + op.drop_table("quota_events") diff --git a/backend/app/main.py b/backend/app/main.py index 4fbea0e..9c4b801 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -26,7 +26,7 @@ from starlette.middleware.sessions import SessionMiddleware from app import auth from app.config import settings -from app.routes import admin, channels, feed, health, me, sync, tags +from app.routes import admin, channels, feed, health, me, quota, sync, tags from app.scheduler import shutdown_scheduler, start_scheduler @@ -67,6 +67,7 @@ app.include_router(feed.router) app.include_router(me.router) app.include_router(channels.router) app.include_router(admin.router) +app.include_router(quota.router) # The built SPA (populated by the Docker frontend build stage). STATIC_DIR = Path(__file__).parent / "static_spa" diff --git a/backend/app/models.py b/backend/app/models.py index f3341ed..243f769 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -265,6 +265,25 @@ class ApiQuotaUsage(Base): units_used: Mapped[int] = mapped_column(Integer, default=0, server_default="0") +class QuotaEvent(Base): + """Per-spend audit log for YouTube API quota attribution. `user_id` is the end-user + whose action caused the spend; NULL means background/system work (scheduler backfill, + enrichment, resync). The aggregate budget still lives in ApiQuotaUsage; this is detail.""" + + __tablename__ = "quota_events" + + id: Mapped[int] = mapped_column(primary_key=True) + # SET NULL on user delete: keep the audit trail (it just becomes system-attributed). + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), index=True + ) + action: Mapped[str] = mapped_column(String(40), index=True) + units: Mapped[int] = mapped_column(Integer) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), index=True + ) + + class AppState(Base): """Single-row global app state (admin-controlled).""" diff --git a/backend/app/quota.py b/backend/app/quota.py index b2acbd8..3fe57cf 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -4,22 +4,52 @@ The whole app shares one daily budget (the API resets at midnight US Pacific tim Steady-state work (RSS detection is free; enrichment of new videos is cheap) should always be allowed; expensive backfill must yield to the remaining budget. """ -from datetime import datetime +import contextlib +import contextvars +from datetime import datetime, timezone from zoneinfo import ZoneInfo from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session from app.config import settings -from app.models import ApiQuotaUsage +from app.models import ApiQuotaUsage, QuotaEvent _PACIFIC = ZoneInfo("America/Los_Angeles") +# Request/job-scoped attribution for quota spend: who triggered it and what kind of work. +# Set at entry points (route handlers, scheduler jobs) via attribute(); read by +# record_usage. Default = background/system, generic action. +_actor_id: contextvars.ContextVar[int | None] = contextvars.ContextVar( + "quota_actor_id", default=None +) +_action: contextvars.ContextVar[str] = contextvars.ContextVar( + "quota_action", default="api" +) + + +@contextlib.contextmanager +def attribute(actor_id: int | None, action: str): + """Attribute any quota spent in this block to `actor_id` (None = system) under `action`.""" + t1 = _actor_id.set(actor_id) + t2 = _action.set(action) + try: + yield + finally: + _actor_id.reset(t1) + _action.reset(t2) + def pacific_today(): return datetime.now(_PACIFIC).date() +def pacific_day_start_utc() -> datetime: + """UTC instant of the current Pacific day's midnight — aligns 'today' with the budget reset.""" + start = datetime.now(_PACIFIC).replace(hour=0, minute=0, second=0, microsecond=0) + return start.astimezone(timezone.utc) + + def units_used_today(db: Session) -> int: row = db.get(ApiQuotaUsage, pacific_today()) return row.units_used if row else 0 @@ -34,7 +64,7 @@ def can_spend(db: Session, units: int) -> bool: def record_usage(db: Session, units: int) -> None: - """Atomically add `units` to today's counter (upsert).""" + """Atomically add `units` to today's counter (upsert) and log an attribution event.""" if units <= 0: return day = pacific_today() @@ -47,4 +77,6 @@ def record_usage(db: Session, units: int) -> None: ) ) db.execute(stmt) + # Audit detail: who/what spent it (per-user attribution; NULL actor = system). + db.add(QuotaEvent(user_id=_actor_id.get(), action=_action.get(), units=units)) db.commit() diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index b99bac7..fc4e0d3 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -5,6 +5,7 @@ from fastapi import APIRouter, Depends, HTTPException from sqlalchemy import and_, func, select from sqlalchemy.orm import Session +from app import quota from app.auth import current_user, has_write_scope from app.db import get_db from app.models import Channel, ChannelTag, Subscription, Tag, User, Video @@ -121,7 +122,8 @@ def update_channel( # fetched its recent uploads yet, do that one channel now (cheap) instead of waiting for # the scheduler. Deep paging still follows on the scheduler's next run (recent-then-deep). if deep_turned_on and channel is not None and channel.recent_synced_at is None: - run_recent_backfill(db, [channel], max_channels=1) + with quota.attribute(user.id, "backfill_recent"): + run_recent_backfill(db, [channel], max_channels=1) return { "id": channel_id, @@ -152,7 +154,7 @@ def unsubscribe( detail="No YouTube subscription id on record — sync subscriptions first.", ) try: - with YouTubeClient(db, user) as yt: + with quota.attribute(user.id, "unsubscribe"), YouTubeClient(db, user) as yt: yt.delete_subscription(sub.yt_subscription_id) except YouTubeError as exc: raise HTTPException(status_code=502, detail=f"YouTube unsubscribe failed: {exc}") diff --git a/backend/app/routes/quota.py b/backend/app/routes/quota.py new file mode 100644 index 0000000..6dd6929 --- /dev/null +++ b/backend/app/routes/quota.py @@ -0,0 +1,106 @@ +"""Quota usage attribution: a user's own consumption + an admin breakdown across users. + +The aggregate daily budget lives in ApiQuotaUsage; here we read the per-spend audit log +(QuotaEvent) to attribute units to the user who triggered them (NULL actor = system). +""" +from datetime import datetime, timedelta, timezone + +from fastapi import APIRouter, Depends +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from app import quota +from app.auth import current_user +from app.db import get_db +from app.models import QuotaEvent, User +from app.routes.admin import admin_user + +router = APIRouter(prefix="/api/quota", tags=["quota"]) + + +def _sum(db: Session, *conds) -> int: + return int( + db.scalar( + select(func.coalesce(func.sum(QuotaEvent.units), 0)).where(*conds) + ) + or 0 + ) + + +@router.get("/my-usage") +def my_usage( + user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + now = datetime.now(timezone.utc) + mine = QuotaEvent.user_id == user.id + by_action = { + action: int(units) + for action, units in db.execute( + select(QuotaEvent.action, func.sum(QuotaEvent.units)) + .where(mine, QuotaEvent.created_at >= now - timedelta(days=30)) + .group_by(QuotaEvent.action) + ).all() + } + return { + "today": _sum(db, mine, QuotaEvent.created_at >= quota.pacific_day_start_utc()), + "last_7d": _sum(db, mine, QuotaEvent.created_at >= now - timedelta(days=7)), + "last_30d": _sum(db, mine, QuotaEvent.created_at >= now - timedelta(days=30)), + "all_time": _sum(db, mine), + "by_action": by_action, + } + + +@router.get("/admin") +def admin_usage( + days: int = 30, + _: User = Depends(admin_user), + db: Session = Depends(get_db), +) -> dict: + days = max(1, min(days, 365)) + since = datetime.now(timezone.utc) - timedelta(days=days) + + totals = db.execute( + select(QuotaEvent.user_id, func.sum(QuotaEvent.units)) + .where(QuotaEvent.created_at >= since) + .group_by(QuotaEvent.user_id) + ).all() + + per_action: dict[int | None, dict[str, int]] = {} + for uid, action, units in db.execute( + select(QuotaEvent.user_id, QuotaEvent.action, func.sum(QuotaEvent.units)) + .where(QuotaEvent.created_at >= since) + .group_by(QuotaEvent.user_id, QuotaEvent.action) + ).all(): + per_action.setdefault(uid, {})[action] = int(units) + + uids = [uid for uid, _ in totals if uid is not None] + emails = ( + {u.id: u.email for u in db.execute(select(User).where(User.id.in_(uids))).scalars()} + if uids + else {} + ) + + rows = [ + { + "user_id": uid, + "email": emails.get(uid, "(deleted user)") if uid is not None else "System", + "total": int(total), + "by_action": per_action.get(uid, {}), + } + for uid, total in totals + ] + rows.sort(key=lambda r: r["total"], reverse=True) + + # Instance-wide daily series in Pacific days (matches the budget reset). + day_expr = func.date(func.timezone("America/Los_Angeles", QuotaEvent.created_at)) + daily = [ + {"day": str(day), "total": int(units)} + for day, units in db.execute( + select(day_expr, func.sum(QuotaEvent.units)) + .where(QuotaEvent.created_at >= since) + .group_by(day_expr) + .order_by(day_expr) + ).all() + ] + + return {"range_days": days, "rows": rows, "daily": daily} diff --git a/backend/app/routes/sync.py b/backend/app/routes/sync.py index bc5cf3d..55dadfb 100644 --- a/backend/app/routes/sync.py +++ b/backend/app/routes/sync.py @@ -34,7 +34,8 @@ def sync_subscriptions( user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: before = quota.units_used_today(db) - result = import_subscriptions(db, user) + with quota.attribute(user.id, "sync_subscriptions"): + result = import_subscriptions(db, user) result["quota_used_estimate"] = quota.units_used_today(db) - before result["quota_remaining_today"] = quota.remaining_today(db) return result @@ -55,7 +56,8 @@ def sync_backfill( max_channels: int = 25, ) -> dict: before = quota.units_used_today(db) - result = run_recent_backfill(db, _user_channels(db, user), max_channels=max_channels) + with quota.attribute(user.id, "backfill_recent"): + result = run_recent_backfill(db, _user_channels(db, user), max_channels=max_channels) result["quota_used_estimate"] = quota.units_used_today(db) - before return result @@ -65,7 +67,8 @@ def sync_enrich( user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: before = quota.units_used_today(db) - enriched = run_enrich(db) + with quota.attribute(user.id, "enrich"): + enriched = run_enrich(db) return { "enriched": enriched, "quota_used_estimate": quota.units_used_today(db) - before, diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 8925890..bd93d07 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -4,6 +4,7 @@ import logging from apscheduler.schedulers.background import BackgroundScheduler +from app import quota from app.config import settings from app.db import SessionLocal from app.state import is_sync_paused @@ -42,14 +43,21 @@ def _rss_job() -> None: def _enrich_job() -> None: - _job("enrich", lambda db: run_enrich(db)) + def work(db): + with quota.attribute(None, "enrich"): + return run_enrich(db) + + _job("enrich", work) def _backfill_job() -> None: - # Recent-first for not-yet-synced channels, then deep backfill for the rest. + # Recent-first for not-yet-synced channels, then deep backfill for the rest. All + # background spend is attributed to the system (no actor), split by action. def work(db): - recent = run_recent_backfill(db, max_channels=25) - deep = run_deep_backfill(db, max_channels=10) + with quota.attribute(None, "backfill_recent"): + recent = run_recent_backfill(db, max_channels=25) + with quota.attribute(None, "backfill_deep"): + deep = run_deep_backfill(db, max_channels=10) return {"recent": recent, "deep": deep} _job("backfill", work) @@ -64,7 +72,11 @@ def _shorts_job() -> None: def _subscriptions_job() -> None: - _job("subscriptions", run_subscription_resync) + def work(db): + with quota.attribute(None, "subscription_resync"): + return run_subscription_resync(db) + + _job("subscriptions", work) def start_scheduler() -> None: diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 7e1e7a5..0f31790 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -22,6 +22,7 @@ import Header from "./components/Header"; import Sidebar from "./components/Sidebar"; import Feed from "./components/Feed"; import Channels from "./components/Channels"; +import Stats from "./components/Stats"; import SettingsPanel from "./components/SettingsPanel"; import Toaster from "./components/Toaster"; @@ -157,9 +158,7 @@ export default function App() { /> )}
- {page === "feed" ? ( - - ) : ( + {page === "channels" ? ( { @@ -167,6 +166,10 @@ export default function App() { setPage("feed"); }} /> + ) : page === "stats" && meQuery.data!.role === "admin" ? ( + + ) : ( + )}
diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx index bd5ca99..746015e 100644 --- a/frontend/src/components/Header.tsx +++ b/frontend/src/components/Header.tsx @@ -1,5 +1,5 @@ import { useRef, useState } from "react"; -import { Home, LogOut, Search, Settings, Shield, Tv } from "lucide-react"; +import { BarChart3, Home, LogOut, Search, Settings, Shield, Tv } from "lucide-react"; import type { FeedFilters, Me } from "../lib/api"; import type { Page } from "../lib/urlState"; import SyncStatus from "./SyncStatus"; @@ -48,7 +48,9 @@ export default function Header({ /> ) : ( -
Channel manager
+
+ {page === "stats" ? "Usage & stats" : "Channel manager"} +
)}
@@ -134,17 +136,24 @@ function AccountMenu({ )}
- {page === "channels" ? ( + {page !== "feed" && ( - ) : ( + )} + {page !== "channels" && ( )} + {me.role === "admin" && page !== "stats" && ( + + )} + ))} +
+
+ +

+ YouTube Data API units attributed by who triggered the spend. System is + shared background work (scheduled backfill, enrichment, resync) that isn't tied to one person. +

+ + {q.isLoading ? ( +
Loading…
+ ) : !data ? ( +
No data.
+ ) : ( + <> + {/* Daily totals (instance-wide, Pacific days) */} +
+
+ Daily total ({data.range_days}d · {grandTotal.toLocaleString()} units) +
+ {data.daily.length === 0 ? ( +
No usage in this range.
+ ) : ( +
+ {data.daily.map((d) => ( +
+ ))} +
+ )} +
+ + {/* Per-user breakdown */} +
+ {data.rows.length === 0 ? ( +
No usage in this range.
+ ) : ( + data.rows.map((r) => ( + + )) + )} +
+ + )} +
+ ); +} + +function UserRow({ row, max }: { row: AdminQuotaRow; max: number }) { + const [open, setOpen] = useState(false); + const actions = Object.entries(row.by_action).sort((a, b) => b[1] - a[1]); + const isSystem = row.user_id === null; + return ( +
+ + {/* Proportion bar */} +
+
+
+ {open && actions.length > 0 && ( +
+ {actions.map(([action, units]) => ( +
+ {quotaActionLabel(action)} + {units.toLocaleString()} +
+ ))} +
+ )} +
+ ); +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index bf693dd..5043300 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -165,6 +165,27 @@ export interface MyStatus { paused: boolean; } +export interface MyUsage { + today: number; + last_7d: number; + last_30d: number; + all_time: number; + by_action: Record; +} + +export interface AdminQuotaRow { + user_id: number | null; + email: string; + total: number; + by_action: Record; +} + +export interface AdminQuota { + range_days: number; + rows: AdminQuotaRow[]; + daily: { day: string; total: number }[]; +} + export interface ManagedChannel { id: string; title: string | null; @@ -215,6 +236,10 @@ export const api = { req(`/api/channels/${id}/subscription`, { method: "DELETE" }), syncSubscriptions: () => req("/api/sync/subscriptions", { method: "POST" }), + // --- quota usage --- + myUsage: (): Promise => req("/api/quota/my-usage"), + adminQuota: (days = 30): Promise => req(`/api/quota/admin?days=${days}`), + // --- onboarding / admin --- requestAccess: (email: string): Promise<{ status: string }> => req("/auth/request-access", { method: "POST", body: JSON.stringify({ email }) }), diff --git a/frontend/src/lib/format.ts b/frontend/src/lib/format.ts index 440e6db..b73a919 100644 --- a/frontend/src/lib/format.ts +++ b/frontend/src/lib/format.ts @@ -34,6 +34,20 @@ export function formatDuration(sec: number | null): string { return h > 0 ? `${h}:${pad(m)}:${pad(s)}` : `${m}:${pad(s)}`; } +const QUOTA_ACTION_LABELS: Record = { + sync_subscriptions: "Sync subscriptions", + backfill_recent: "Recent backfill", + backfill_deep: "Full-history backfill", + enrich: "Enrichment", + unsubscribe: "Unsubscribe", + subscription_resync: "Auto subscription resync", + api: "Other", +}; + +export function quotaActionLabel(action: string): string { + return QUOTA_ACTION_LABELS[action] ?? action; +} + /** Coarse, human ETA for a remaining duration in seconds ("~3 hours", "~2 days"). */ export function formatEta(seconds: number): string { if (seconds <= 0) return "done"; diff --git a/frontend/src/lib/urlState.ts b/frontend/src/lib/urlState.ts index c4fcb4f..8498865 100644 --- a/frontend/src/lib/urlState.ts +++ b/frontend/src/lib/urlState.ts @@ -74,12 +74,11 @@ export function hasFilterParams(params: URLSearchParams): boolean { return KEYS.some((k) => params.has(k)); } -export type Page = "feed" | "channels"; +export type Page = "feed" | "channels" | "stats"; export function readPage(): Page { - return new URLSearchParams(window.location.search).get("page") === "channels" - ? "channels" - : "feed"; + const p = new URLSearchParams(window.location.search).get("page"); + return p === "channels" || p === "stats" ? p : "feed"; } /** Reflect the current filters + page into the address bar without adding history entries. */