diff --git a/backend/alembic/versions/0016_video_maintenance.py b/backend/alembic/versions/0016_video_maintenance.py new file mode 100644 index 0000000..7eff8ca --- /dev/null +++ b/backend/alembic/versions/0016_video_maintenance.py @@ -0,0 +1,43 @@ +"""video maintenance / validation columns + +Revision ID: 0016_video_maintenance +Revises: 0015_scheduler_settings +Create Date: 2026-06-18 + +Adds the columns the maintenance/validation job needs to detect and retire unplayable +videos: videos.list?part=status fields (embeddable / privacy_status / upload_status) plus +the validation lifecycle (last_checked_at for rolling re-validation; unavailable_since + +unavailable_reason to flag a video as currently unplayable — hidden from the feed +immediately, hard-deleted after a grace period). Purely additive; all columns nullable. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0016_video_maintenance" +down_revision: Union[str, None] = "0015_scheduler_settings" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("videos", sa.Column("embeddable", sa.Boolean(), nullable=True)) + op.add_column("videos", sa.Column("privacy_status", sa.String(length=16), nullable=True)) + op.add_column("videos", sa.Column("upload_status", sa.String(length=16), nullable=True)) + op.add_column("videos", sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("videos", sa.Column("unavailable_since", sa.DateTime(timezone=True), nullable=True)) + op.add_column("videos", sa.Column("unavailable_reason", sa.String(length=24), nullable=True)) + op.create_index("ix_videos_last_checked_at", "videos", ["last_checked_at"]) + op.create_index("ix_videos_unavailable_since", "videos", ["unavailable_since"]) + + +def downgrade() -> None: + op.drop_index("ix_videos_unavailable_since", table_name="videos") + op.drop_index("ix_videos_last_checked_at", table_name="videos") + op.drop_column("videos", "unavailable_reason") + op.drop_column("videos", "unavailable_since") + op.drop_column("videos", "last_checked_at") + op.drop_column("videos", "upload_status") + op.drop_column("videos", "privacy_status") + op.drop_column("videos", "embeddable") diff --git a/backend/alembic/versions/0017_notifications.py b/backend/alembic/versions/0017_notifications.py new file mode 100644 index 0000000..057544e --- /dev/null +++ b/backend/alembic/versions/0017_notifications.py @@ -0,0 +1,59 @@ +"""per-user notification inbox (phase 1) + +Revision ID: 0017_notifications +Revises: 0016_video_maintenance +Create Date: 2026-06-18 + +Adds the durable, server-backed per-user notification table behind the inbox module. The +JSON payload column is named `data` (SQLAlchemy's declarative Base reserves `metadata`). +First producer is the maintenance job's batched "videos removed" notice. Coexists with the +existing client-side transient bell. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0017_notifications" +down_revision: Union[str, None] = "0016_video_maintenance" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "notifications", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("type", sa.String(length=32), nullable=False), + sa.Column("title", sa.String(length=255), nullable=False), + sa.Column("body", sa.Text(), nullable=True), + sa.Column("data", sa.JSON(), nullable=True), + sa.Column("read", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("dismissed", sa.Boolean(), nullable=False, server_default="false"), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now() + ), + ) + op.create_index("ix_notifications_user_id", "notifications", ["user_id"]) + op.create_index("ix_notifications_type", "notifications", ["type"]) + op.create_index("ix_notifications_read", "notifications", ["read"]) + op.create_index("ix_notifications_created_at", "notifications", ["created_at"]) + # Fast unread-badge count + inbox listing per user. + op.create_index( + "ix_notifications_user_read", "notifications", ["user_id", "read"] + ) + + +def downgrade() -> None: + op.drop_index("ix_notifications_user_read", table_name="notifications") + op.drop_index("ix_notifications_created_at", table_name="notifications") + op.drop_index("ix_notifications_read", table_name="notifications") + op.drop_index("ix_notifications_type", table_name="notifications") + op.drop_index("ix_notifications_user_id", table_name="notifications") + op.drop_table("notifications") diff --git a/backend/app/config.py b/backend/app/config.py index 52bbd8a..af60b2b 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -90,6 +90,17 @@ class Settings(BaseSettings): autotag_interval_minutes: int = 30 subscriptions_resync_minutes: int = 360 playlist_sync_minutes: int = 360 + # --- Maintenance / validation job --- + # Runs once a day by default (admin-tunable via the Scheduler dashboard). Grace periods + # are in days because that's the granularity that matters; the rolling re-validation + # re-checks the least-recently-checked N videos per run (1 unit / 50 videos), so on a + # ~233k catalog ~15000/day ≈ a 15-day full cycle for ~300 quota units. + maintenance_interval_minutes: int = 1440 + maintenance_revalidate_batch: int = 15000 + # Grace before a flagged-unavailable video is hard-deleted (cascades to states/playlist + # items). Deleted/private/paywalled get a recovery window; abandoned upcoming and + # confirmed-dead stuck-live have no age grace (deleted on the sweep that confirms them). + maintenance_grace_days: int = 7 # live_status values hidden from the feed by default. Completed-stream VODs # ("was_live") are real watchable content and stay visible. feed_default_hidden_live: str = "live,upcoming" diff --git a/backend/app/main.py b/backend/app/main.py index ca0b01d..d134beb 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -32,6 +32,7 @@ from app.routes import ( feed, health, me, + notifications, playlists, quota, scheduler as scheduler_routes, @@ -77,6 +78,7 @@ app.include_router(sync.router) app.include_router(tags.router) app.include_router(feed.router) app.include_router(me.router) +app.include_router(notifications.router) app.include_router(channels.router) app.include_router(playlists.router) app.include_router(admin.router) diff --git a/backend/app/models.py b/backend/app/models.py index f99e8ec..dfe3e13 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -203,6 +203,22 @@ class Video(Base): String(16), default="none", server_default="none", index=True ) enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + # videos.list?part=status fields (populated during enrichment); used by the + # maintenance/validation job to decide whether a video is playable anywhere. + embeddable: Mapped[bool | None] = mapped_column(Boolean) + privacy_status: Mapped[str | None] = mapped_column(String(16)) # public|unlisted|private + upload_status: Mapped[str | None] = mapped_column(String(16)) # processed|failed|rejected|deleted + # Maintenance/validation lifecycle. last_checked_at drives the rolling re-validation + # (least-recently-checked first); unavailable_since marks a video as currently + # unplayable (hidden from the feed immediately, hard-deleted after a grace period); + # unavailable_reason records why (deleted|private|paywalled|abandoned|stuck_live). + last_checked_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), index=True + ) + unavailable_since: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), index=True + ) + unavailable_reason: Mapped[str | None] = mapped_column(String(24)) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now() ) @@ -399,3 +415,36 @@ class PlaylistItem(Base): ) playlist: Mapped["Playlist"] = relationship(back_populates="items") + + +class Notification(Base): + """A durable, server-backed per-user notification (the inbox center, phase 1). + + Distinct from the client-side transient "bell" (toasts kept only in localStorage): + these survive reloads and devices, and are produced server-side (first producer = the + maintenance/validation job's batched "N saved/playlisted videos removed" notice). The + column is named `data` (not `metadata`, which SQLAlchemy's declarative Base reserves): + a free-form JSON payload (e.g. the list of removed videos) the UI can render. `read` + and `dismissed` are separate: read clears the unread badge but keeps the row in the + inbox; dismissed removes it from the default view. Unread rows never auto-expire; read + ones are trimmed past a soft per-user cap (see app.notifications.trim_read).""" + + __tablename__ = "notifications" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + type: Mapped[str] = mapped_column(String(32), index=True) # e.g. "maintenance" + title: Mapped[str] = mapped_column(String(255)) + body: Mapped[str | None] = mapped_column(Text) + data: Mapped[dict | None] = mapped_column(JSON) + read: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false", index=True + ) + dismissed: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false" + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), index=True + ) diff --git a/backend/app/notifications.py b/backend/app/notifications.py new file mode 100644 index 0000000..c4ce120 --- /dev/null +++ b/backend/app/notifications.py @@ -0,0 +1,58 @@ +"""Helpers for the durable, server-backed per-user notification inbox (phase 1). + +The inbox COEXISTS with the client-side transient "bell" (localStorage toasts): these rows +survive reloads/devices and are produced server-side. First producer is the maintenance +job. Unread rows never auto-expire; read rows are trimmed past a soft per-user cap so the +table can't grow without bound for a heavy user. +""" +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.models import Notification + +# Keep at most this many READ notifications per user; older read ones are pruned. Unread +# rows are never pruned (the user hasn't seen them yet). +READ_SOFT_CAP = 200 + + +def create_notification( + db: Session, + user_id: int, + type: str, + title: str, + body: str | None = None, + data: dict | None = None, + *, + commit: bool = True, +) -> Notification: + """Insert one notification for a user. Callers that batch many inserts in a single + transaction can pass commit=False and commit once themselves.""" + notif = Notification( + user_id=user_id, type=type, title=title, body=body, data=data + ) + db.add(notif) + if commit: + db.commit() + db.refresh(notif) + return notif + + +def trim_read(db: Session, user_id: int, cap: int = READ_SOFT_CAP) -> int: + """Delete a user's oldest READ notifications beyond `cap`. Returns the number removed. + Idempotent and cheap; safe to call after marking things read.""" + ids = ( + db.execute( + select(Notification.id) + .where(Notification.user_id == user_id, Notification.read.is_(True)) + .order_by(Notification.created_at.desc()) + .offset(cap) + ) + .scalars() + .all() + ) + if not ids: + return 0 + for nid in ids: + db.delete(db.get(Notification, nid)) + db.commit() + return len(ids) diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 04ef3be..8afa915 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -154,6 +154,11 @@ def _filtered_query( state, and_(state.video_id == Video.id, state.user_id == user.id) ) + # Hide videos the maintenance job has flagged as currently unplayable (deleted/private/ + # abandoned). They're either deleted after a grace period or unhidden if they recover; + # either way they shouldn't show in any feed view meanwhile. + query = query.where(Video.unavailable_since.is_(None)) + if channel_id: query = query.where(Video.channel_id == channel_id) if min_duration is not None: diff --git a/backend/app/routes/notifications.py b/backend/app/routes/notifications.py new file mode 100644 index 0000000..224a7f7 --- /dev/null +++ b/backend/app/routes/notifications.py @@ -0,0 +1,136 @@ +"""Per-user notification inbox (phase 1): the durable, server-backed center that coexists +with the client-side transient bell. Read-only listing plus read/dismiss/clear mutations. + +All endpoints use `current_user` (not `require_human`): notifications and their read/dismiss +state are personal UI state with no quota or YouTube involvement, so the shared demo account +manages its own inbox normally.""" +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import func, select, update +from sqlalchemy.orm import Session + +from app.auth import current_user +from app.db import get_db +from app.models import Notification, User +from app.notifications import trim_read + +router = APIRouter(prefix="/api/me/notifications", tags=["notifications"]) + + +def _serialize(n: Notification) -> dict: + return { + "id": n.id, + "type": n.type, + "title": n.title, + "body": n.body, + "data": n.data, + "read": n.read, + "dismissed": n.dismissed, + "created_at": n.created_at.isoformat() if n.created_at else None, + } + + +@router.get("") +def list_notifications( + include_dismissed: bool = False, + limit: int = Query(default=100, le=500), + offset: int = 0, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + where = [Notification.user_id == user.id] + if not include_dismissed: + where.append(Notification.dismissed.is_(False)) + total = db.scalar( + select(func.count()).select_from(Notification).where(*where) + ) or 0 + rows = ( + db.execute( + select(Notification) + .where(*where) + .order_by(Notification.created_at.desc()) + .offset(offset) + .limit(limit) + ) + .scalars() + .all() + ) + return {"items": [_serialize(n) for n in rows], "total": total} + + +@router.get("/unread_count") +def unread_count( + user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + """Cheap poll target for the nav badge (uses the (user_id, read) index).""" + count = db.scalar( + select(func.count()) + .select_from(Notification) + .where( + Notification.user_id == user.id, + Notification.read.is_(False), + Notification.dismissed.is_(False), + ) + ) + return {"count": count or 0} + + +def _owned(db: Session, notification_id: int, user: User) -> Notification: + n = db.get(Notification, notification_id) + if n is None or n.user_id != user.id: + raise HTTPException(status_code=404, detail="Notification not found") + return n + + +@router.post("/{notification_id}/read") +def mark_read( + notification_id: int, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + n = _owned(db, notification_id, user) + n.read = True + db.commit() + trim_read(db, user.id) + return {"ok": True} + + +@router.post("/read_all") +def mark_all_read( + user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + db.execute( + update(Notification) + .where(Notification.user_id == user.id, Notification.read.is_(False)) + .values(read=True) + ) + db.commit() + trim_read(db, user.id) + return {"ok": True} + + +@router.post("/{notification_id}/dismiss") +def dismiss( + notification_id: int, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + n = _owned(db, notification_id, user) + # Dismiss implies read (it no longer counts toward the unread badge). + n.dismissed = True + n.read = True + db.commit() + return {"ok": True} + + +@router.post("/clear") +def clear_all( + user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + """Dismiss every (still-visible) notification for this user.""" + db.execute( + update(Notification) + .where(Notification.user_id == user.id, Notification.dismissed.is_(False)) + .values(dismissed=True, read=True) + ) + db.commit() + return {"ok": True} diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 656db73..b6188ab 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -13,6 +13,7 @@ from app.db import SessionLocal from app.models import SchedulerSetting from app.state import is_sync_paused from app.sync.autotag import run_autotag_all +from app.sync.maintenance import run_maintenance from app.sync.playlists import sync_all_playlists from app.sync.runner import ( run_deep_backfill, @@ -44,6 +45,7 @@ JOB_INTERVALS: dict[str, int] = { "shorts": settings.shorts_probe_interval_minutes, "subscriptions": settings.subscriptions_resync_minutes, "playlist_sync": settings.playlist_sync_minutes, + "maintenance": settings.maintenance_interval_minutes, } # Sane bounds for an admin-set interval (minutes). @@ -201,6 +203,14 @@ def _playlist_sync_job() -> None: _job("playlist_sync", sync_all_playlists) +def _maintenance_job() -> None: + def work(db): + with quota.attribute(None, "maintenance"): + return run_maintenance(db) + + _job("maintenance", work) + + def start_scheduler() -> None: global _scheduler if not settings.scheduler_enabled or _scheduler is not None: @@ -219,6 +229,7 @@ def start_scheduler() -> None: "shorts": _shorts_job, "subscriptions": _subscriptions_job, "playlist_sync": _playlist_sync_job, + "maintenance": _maintenance_job, } scheduler = BackgroundScheduler(timezone="UTC") for job_id, fn in fns.items(): diff --git a/backend/app/sync/maintenance.py b/backend/app/sync/maintenance.py new file mode 100644 index 0000000..be9f4c9 --- /dev/null +++ b/backend/app/sync/maintenance.py @@ -0,0 +1,246 @@ +"""Maintenance / validation job: detect videos that can't be played anywhere and retire +them safely. + +Lifecycle (no permanent soft-delete state — just a grace timer before a hard delete): + + 1. Re-check flagged videos (those with unavailable_since set). If a video has recovered + (e.g. a private video made public again), clear the flag. Otherwise, if its grace + period has elapsed, HARD DELETE it (cascades to video_states / playlist_items). Before + deleting, gather which users had it saved or in a playlist so we can tell them. + 2. Rolling re-validation: re-fetch the least-recently-checked N currently-available + videos. Newly-unplayable ones get flagged (hidden from the feed immediately via + unavailable_since); the grace timer starts now and they're deleted in a later run if + they don't recover. + +Detection is ~free: a video missing from the videos.list response is deleted-or-private. +We never delete a live broadcast that still reports viewers (a legit 24/7 stream); we only +flag an `upcoming` premiere that blew past its scheduled start and never went live. + +Quota: phase 1 costs ~(flagged / 50) units, phase 2 ~(batch / 50). Both respect the same +backfill reserve as the other jobs so interactive syncs never starve. +""" +import logging +from datetime import datetime, timedelta, timezone + +from sqlalchemy import or_, select +from sqlalchemy.orm import Session + +from app import quota +from app.config import settings +from app.models import Playlist, PlaylistItem, Video, VideoState +from app.notifications import create_notification +from app.sync.runner import get_service_user +from app.sync.videos import apply_video_details, parse_dt +from app.youtube.client import YouTubeClient + +log = logging.getLogger("subfeed.maintenance") + +# Reasons recorded on Video.unavailable_reason. "removed" = absent from videos.list +# (deleted or made private); "abandoned" = an upcoming premiere that never went live. +GRACE_DAYS_BY_REASON = { + "removed": settings.maintenance_grace_days, + "abandoned": 0, # already past-due when detected; deleted on the confirming sweep +} +DEFAULT_GRACE_DAYS = settings.maintenance_grace_days + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _grace_days(reason: str | None) -> int: + return GRACE_DAYS_BY_REASON.get(reason or "", DEFAULT_GRACE_DAYS) + + +def _is_abandoned_upcoming(item: dict) -> bool: + """True for an `upcoming` broadcast whose scheduled start is >2 days past and which + never actually went live (no actualStartTime).""" + snippet = item.get("snippet", {}) + if snippet.get("liveBroadcastContent") != "upcoming": + return False + live = item.get("liveStreamingDetails") or {} + if live.get("actualStartTime"): + return False # it did go live at some point — not abandoned + scheduled = parse_dt(live.get("scheduledStartTime")) + if scheduled is None: + return False + return _now() - scheduled > timedelta(days=2) + + +def _evaluate(video: Video, item: dict | None) -> str | None: + """Given a fresh videos.list item (or None if absent), return an unavailable reason, + or None if the video is fine. Applies fresh metadata as a side effect when present.""" + if item is None: + return "removed" + apply_video_details(video, item) + if _is_abandoned_upcoming(item): + return "abandoned" + # A still-"live" broadcast is kept (a legit 24/7 stream); ended streams have already + # settled to was_live via apply_video_details. Nothing else here is a delete trigger — + # embeddable=false only blocks our in-app embed, not playback on youtube.com. + return None + + +def _collect_user_impact(db: Session, videos: list[Video]) -> dict[int, list[dict]]: + """For videos about to be deleted, map user_id -> list of {id, title, reason} for the + videos that user had saved or in one of their playlists. Must run BEFORE the delete, + while the video_states / playlist_items rows still exist.""" + by_id = {v.id: v for v in videos} + ids = list(by_id) + impact: dict[int, set[str]] = {} + if not ids: + return {} + # Saved (VideoState.status == "saved"). + for user_id, video_id in db.execute( + select(VideoState.user_id, VideoState.video_id).where( + VideoState.video_id.in_(ids), VideoState.status == "saved" + ) + ): + impact.setdefault(user_id, set()).add(video_id) + # In any of the user's playlists (covers watch-later, which is a playlist). + for user_id, video_id in db.execute( + select(Playlist.user_id, PlaylistItem.video_id) + .join(PlaylistItem, PlaylistItem.playlist_id == Playlist.id) + .where(PlaylistItem.video_id.in_(ids)) + ): + impact.setdefault(user_id, set()).add(video_id) + return { + user_id: [ + { + "id": vid, + "title": by_id[vid].title, + "reason": by_id[vid].unavailable_reason, + } + for vid in vids + ] + for user_id, vids in impact.items() + } + + +def _notify_removed(db: Session, impact: dict[int, list[dict]]) -> int: + """One batched notification per affected user (never per-video).""" + for user_id, videos in impact.items(): + n = len(videos) + create_notification( + db, + user_id=user_id, + type="maintenance", + title="Videos removed", + body=( + f"{n} saved or playlisted " + f"{'video was' if n == 1 else 'videos were'} removed because " + "they are no longer available on YouTube." + ), + data={"count": n, "videos": videos}, + commit=False, + ) + db.commit() + return len(impact) + + +def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict: + """Phase 1: recover or hard-delete videos already flagged unavailable.""" + flagged = ( + db.execute(select(Video).where(Video.unavailable_since.is_not(None))) + .scalars() + .all() + ) + if not flagged: + return {"checked": 0, "recovered": 0, "deleted": 0, "notified": 0} + items = {it["id"]: it for it in yt.get_videos([v.id for v in flagged])} + now = _now() + recovered = 0 + to_delete: list[Video] = [] + for video in flagged: + reason = _evaluate(video, items.get(video.id)) + video.last_checked_at = now + if reason is None: + # Recovered: unhide. + video.unavailable_since = None + video.unavailable_reason = None + recovered += 1 + continue + # Still unavailable; keep the original reason/timer unless it changed. + if reason != video.unavailable_reason: + video.unavailable_reason = reason + deadline = video.unavailable_since + timedelta(days=_grace_days(reason)) + if now >= deadline: + to_delete.append(video) + db.commit() # persist recoveries / reason updates / last_checked_at + + notified = 0 + deleted = 0 + if to_delete: + impact = _collect_user_impact(db, to_delete) + for video in to_delete: + db.delete(video) + db.commit() # cascades to video_states / playlist_items + deleted = len(to_delete) + notified = _notify_removed(db, impact) + return { + "checked": len(flagged), + "recovered": recovered, + "deleted": deleted, + "notified": notified, + } + + +def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict: + """Phase 2: re-check the least-recently-checked currently-available videos and flag any + that have become unplayable (hidden immediately; deleted later if they don't recover).""" + batch = settings.maintenance_revalidate_batch + flagged_new = 0 + checked = 0 + now = _now() + # Process in videos.list-sized pages so we commit incrementally and stop on low quota. + while checked < batch: + if quota.remaining_today(db) <= settings.backfill_quota_reserve: + break + page = ( + db.execute( + select(Video) + .where(Video.unavailable_since.is_(None)) + .order_by( + Video.last_checked_at.is_(None).desc(), # never-checked first + Video.last_checked_at.asc(), + ) + .limit(min(50, batch - checked)) + ) + .scalars() + .all() + ) + if not page: + break + items = {it["id"]: it for it in yt.get_videos([v.id for v in page])} + for video in page: + reason = _evaluate(video, items.get(video.id)) + video.last_checked_at = now + if reason is not None: + video.unavailable_since = now + video.unavailable_reason = reason + flagged_new += 1 + db.commit() + checked += len(page) + return {"checked": checked, "flagged": flagged_new} + + +def run_maintenance(db: Session) -> dict: + """Entry point invoked by the scheduler. Quota-attributed to the system (no actor).""" + user = get_service_user(db) + if user is None: + return {"reason": "no service user"} + with YouTubeClient(db, user) as yt: + phase1 = _recheck_flagged(db, yt) + phase2 = ( + _revalidate_rolling(db, yt) + if quota.remaining_today(db) > settings.backfill_quota_reserve + else {"checked": 0, "flagged": 0, "skipped": "low quota"} + ) + return { + "rechecked": phase1["checked"], + "recovered": phase1["recovered"], + "deleted": phase1["deleted"], + "notified": phase1["notified"], + "revalidated": phase2["checked"], + "flagged": phase2["flagged"], + } diff --git a/backend/app/sync/videos.py b/backend/app/sync/videos.py index c4f002d..eaeeff2 100644 --- a/backend/app/sync/videos.py +++ b/backend/app/sync/videos.py @@ -217,6 +217,7 @@ def apply_video_details(video: Video, item: dict) -> None: stats = item.get("statistics", {}) topics = item.get("topicDetails", {}) live = item.get("liveStreamingDetails") + status = item.get("status", {}) video.title = snippet.get("title") or video.title video.description = snippet.get("description") @@ -232,6 +233,11 @@ def apply_video_details(video: Video, item: dict) -> None: "defaultAudioLanguage" ) video.live_status = _live_status(snippet, live) + # status part: used by the maintenance/validation job to judge playability. + if status: + video.embeddable = status.get("embeddable") + video.privacy_status = status.get("privacyStatus") + video.upload_status = status.get("uploadStatus") # is_short is decided separately by the youtube.com/shorts probe (run_shorts_classification). diff --git a/backend/app/youtube/client.py b/backend/app/youtube/client.py index f0d0da4..c65c333 100644 --- a/backend/app/youtube/client.py +++ b/backend/app/youtube/client.py @@ -327,7 +327,7 @@ class YouTubeClient: data = self._get( "videos", { - "part": "snippet,contentDetails,statistics,topicDetails,liveStreamingDetails", + "part": "snippet,contentDetails,statistics,topicDetails,liveStreamingDetails,status", "id": ",".join(batch), "maxResults": 50, }, diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 4bf2686..ccfe17a 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -10,7 +10,7 @@ import { saveLocalTheme, type ThemePrefs, } from "./lib/theme"; -import { hasFilterParams, paramsToFilters, readPage, stripUrlParams, type Page } from "./lib/urlState"; +import { hasFilterParams, isPage, paramsToFilters, readPage, stripUrlParams, type Page } from "./lib/urlState"; import { loadLayout, normalizeLayout, @@ -29,6 +29,7 @@ import Playlists from "./components/Playlists"; import Stats from "./components/Stats"; import Scheduler from "./components/Scheduler"; import SettingsPanel from "./components/SettingsPanel"; +import NotificationsPanel from "./components/NotificationsPanel"; import OnboardingWizard from "./components/OnboardingWizard"; import { shouldAutoOpenOnboarding } from "./lib/onboarding"; import Toaster from "./components/Toaster"; @@ -60,15 +61,7 @@ function loadInitialPage(): Page { const params = new URLSearchParams(window.location.search); if (params.has("page")) return readPage(); const stored = localStorage.getItem(PAGE_KEY); - if ( - stored === "channels" || - stored === "stats" || - stored === "playlists" || - stored === "settings" || - stored === "scheduler" - ) - return stored; - return "feed"; + return isPage(stored) ? stored : "feed"; } function loadStoredFilters(): FeedFilters { @@ -311,6 +304,8 @@ export default function App() { ) : page === "playlists" ? ( + ) : page === "notifications" ? ( + ) : page === "settings" ? ( ( + const renderItem = ({ page: p, icon: Icon, label, badge }: NavItem) => ( ); diff --git a/frontend/src/components/NotificationsPanel.tsx b/frontend/src/components/NotificationsPanel.tsx new file mode 100644 index 0000000..273e1f4 --- /dev/null +++ b/frontend/src/components/NotificationsPanel.tsx @@ -0,0 +1,176 @@ +import { useTranslation } from "react-i18next"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { Bell, Check, CheckCheck, Trash2, X } from "lucide-react"; +import { api, type AppNotification } from "../lib/api"; +import { useLiveQuery } from "../lib/useLiveQuery"; + +// The durable, server-backed notification center (inbox phase 1). It COEXISTS with the +// client-side transient bell in the nav rail: the bell is a quick peek at recent toasts kept +// in localStorage, this is the full, filterable, clearable history synced from the server. +const POLL_MS = 8000; + +function relativeTime(iso: string | null, t: (k: string, o?: any) => string): string { + if (!iso) return ""; + const then = new Date(iso).getTime(); + const secs = Math.max(0, Math.round((Date.now() - then) / 1000)); + // Reuse the bell's relative-time strings (notifications.time.*) — same format, no dupes. + if (secs < 60) return t("notifications.time.justNow"); + const mins = Math.round(secs / 60); + if (mins < 60) return t("notifications.time.minutes", { count: mins }); + const hours = Math.round(mins / 60); + if (hours < 24) return t("notifications.time.hours", { count: hours }); + const days = Math.round(hours / 24); + return t("notifications.time.days", { count: days }); +} + +export default function NotificationsPanel() { + const { t } = useTranslation(); + const qc = useQueryClient(); + const q = useLiveQuery<{ items: AppNotification[]; total: number }>( + ["notifications"], + () => api.notifications(), + { intervalMs: POLL_MS } + ); + const items = q.data?.items ?? []; + + // Any mutation refreshes both the list and the nav badge's unread count. + const refresh = () => { + qc.invalidateQueries({ queryKey: ["notifications"] }); + qc.invalidateQueries({ queryKey: ["notif-unread"] }); + }; + const readMut = useMutation({ mutationFn: api.markNotificationRead, onSuccess: refresh }); + const readAllMut = useMutation({ mutationFn: api.markAllNotificationsRead, onSuccess: refresh }); + const dismissMut = useMutation({ mutationFn: api.dismissNotification, onSuccess: refresh }); + const clearMut = useMutation({ mutationFn: api.clearNotifications, onSuccess: refresh }); + + const hasUnread = items.some((n) => !n.read); + + return ( +
+
+ +
+
{t("inbox.title")}
+
{t("inbox.subtitle")}
+
+ {items.length > 0 && ( +
+ + +
+ )} +
+ + {q.isLoading && !q.data ? ( +
{t("common.loading")}
+ ) : items.length === 0 ? ( +
+ + {t("inbox.empty")} +
+ ) : ( +
+ {items.map((n) => ( + readMut.mutate(n.id)} + onDismiss={() => dismissMut.mutate(n.id)} + /> + ))} +
+ )} +
+ ); +} + +function NotificationRow({ + n, + t, + onRead, + onDismiss, +}: { + n: AppNotification; + t: (k: string, o?: any) => string; + onRead: () => void; + onDismiss: () => void; +}) { + // The maintenance producer ships the affected videos in `data.videos`; list a few titles. + const videos: { id: string; title: string | null }[] = Array.isArray(n.data?.videos) + ? n.data!.videos + : []; + // Known notification types are rendered from i18n (so they're trilingual) using the typed + // payload; the server-stored title/body are an English fallback for any unknown type. + const isMaintenance = n.type === "maintenance" && typeof n.data?.count === "number"; + const title = isMaintenance ? t("inbox.maintenance.title") : n.title; + const body = isMaintenance + ? t("inbox.maintenance.body", { count: n.data!.count }) + : n.body; + return ( +
+ {!n.read && } +
+
+ {title} + + {relativeTime(n.created_at, t)} + +
+ {body &&
{body}
} + {videos.length > 0 && ( +
    + {videos.slice(0, 8).map((v) => ( +
  • + {v.title || v.id} +
    • + ))} + {videos.length > 8 && ( +
  • + {t("inbox.andMore", { count: videos.length - 8 })} +
    • + )} +
+ )} +
+
+ {!n.read && ( + + )} + +
+
+ ); +} diff --git a/frontend/src/i18n/locales/de/inbox.json b/frontend/src/i18n/locales/de/inbox.json new file mode 100644 index 0000000..39a2ddb --- /dev/null +++ b/frontend/src/i18n/locales/de/inbox.json @@ -0,0 +1,16 @@ +{ + "navLabel": "Benachrichtigungen", + "title": "Benachrichtigungen", + "subtitle": "Neuigkeiten aus deiner Bibliothek und vom System.", + "empty": "Alles erledigt.", + "markAllRead": "Alle als gelesen markieren", + "clearAll": "Alle löschen", + "markRead": "Als gelesen markieren", + "dismiss": "Verwerfen", + "andMore": "und {{count}} weitere", + "maintenance": { + "title": "Videos entfernt", + "body_one": "{{count}} gespeichertes oder in einer Playlist befindliches Video wurde entfernt, weil es auf YouTube nicht mehr verfügbar ist.", + "body_other": "{{count}} gespeicherte oder in Playlists befindliche Videos wurden entfernt, weil sie auf YouTube nicht mehr verfügbar sind." + } +} diff --git a/frontend/src/i18n/locales/en/inbox.json b/frontend/src/i18n/locales/en/inbox.json new file mode 100644 index 0000000..7f972a4 --- /dev/null +++ b/frontend/src/i18n/locales/en/inbox.json @@ -0,0 +1,16 @@ +{ + "navLabel": "Notifications", + "title": "Notifications", + "subtitle": "Updates from your library and the system.", + "empty": "You're all caught up.", + "markAllRead": "Mark all read", + "clearAll": "Clear all", + "markRead": "Mark read", + "dismiss": "Dismiss", + "andMore": "and {{count}} more", + "maintenance": { + "title": "Videos removed", + "body_one": "{{count}} saved or playlisted video was removed because it's no longer available on YouTube.", + "body_other": "{{count}} saved or playlisted videos were removed because they're no longer available on YouTube." + } +} diff --git a/frontend/src/i18n/locales/hu/inbox.json b/frontend/src/i18n/locales/hu/inbox.json new file mode 100644 index 0000000..9167954 --- /dev/null +++ b/frontend/src/i18n/locales/hu/inbox.json @@ -0,0 +1,16 @@ +{ + "navLabel": "Értesítések", + "title": "Értesítések", + "subtitle": "Frissítések a könyvtáradból és a rendszertől.", + "empty": "Nincs új értesítés.", + "markAllRead": "Összes olvasott", + "clearAll": "Összes törlése", + "markRead": "Olvasottnak jelöl", + "dismiss": "Elvetés", + "andMore": "és még {{count}}", + "maintenance": { + "title": "Videók eltávolítva", + "body_one": "{{count}} mentett vagy lejátszási listás videó eltávolítva, mert már nem elérhető a YouTube-on.", + "body_other": "{{count}} mentett vagy lejátszási listás videó eltávolítva, mert már nem elérhetők a YouTube-on." + } +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 12d4d82..006a834 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -356,6 +356,19 @@ export interface Account { active: boolean; } +// A durable, server-backed notification (the inbox center). Distinct from the client-side +// transient toast/bell, which lives only in localStorage. +export interface AppNotification { + id: number; + type: string; + title: string; + body: string | null; + data: Record | null; + read: boolean; + dismissed: boolean; + created_at: string | null; +} + export const api = { me: (): Promise => req("/api/me"), accounts: (): Promise => req("/api/me/accounts"), @@ -478,6 +491,19 @@ export const api = { addInvite: (email: string) => req("/api/admin/invites", { method: "POST", body: JSON.stringify({ email }) }), + // --- notification inbox (durable, server-backed) --- + notifications: (includeDismissed = false): Promise<{ items: AppNotification[]; total: number }> => + req(`/api/me/notifications?include_dismissed=${includeDismissed}`), + notificationUnreadCount: (): Promise<{ count: number }> => + req("/api/me/notifications/unread_count"), + markNotificationRead: (id: number) => + req(`/api/me/notifications/${id}/read`, { method: "POST" }), + markAllNotificationsRead: () => + req("/api/me/notifications/read_all", { method: "POST" }), + dismissNotification: (id: number) => + req(`/api/me/notifications/${id}/dismiss`, { method: "POST" }), + clearNotifications: () => req("/api/me/notifications/clear", { method: "POST" }), + // --- user tags --- createTag: (t: { name: string; color?: string; category?: string }) => req("/api/tags", { method: "POST", body: JSON.stringify(t) }), diff --git a/frontend/src/lib/urlState.ts b/frontend/src/lib/urlState.ts index ab2a38b..8483ee7 100644 --- a/frontend/src/lib/urlState.ts +++ b/frontend/src/lib/urlState.ts @@ -78,17 +78,29 @@ export function hasFilterParams(params: URLSearchParams): boolean { return KEYS.some((k) => params.has(k)); } -export type Page = "feed" | "channels" | "stats" | "playlists" | "settings" | "scheduler"; +// The single source of truth for valid page ids; `Page` and the runtime validator both +// derive from it, so adding a page is a one-line change (no parallel allowlists to keep in +// sync). "feed" is the default/fallback. +export const PAGES = [ + "feed", + "channels", + "stats", + "playlists", + "settings", + "scheduler", + "notifications", +] as const; + +export type Page = (typeof PAGES)[number]; + +/** Narrow an arbitrary string to a known Page (else null). */ +export function isPage(p: string | null | undefined): p is Page { + return !!p && (PAGES as readonly string[]).includes(p); +} export function readPage(): Page { const p = new URLSearchParams(window.location.search).get("page"); - return p === "channels" || - p === "stats" || - p === "playlists" || - p === "settings" || - p === "scheduler" - ? p - : "feed"; + return isPage(p) ? p : "feed"; } /** Build a shareable absolute URL that reproduces the current filters (+ page). Used by the