diff --git a/backend/alembic/versions/0033_channel_explore.py b/backend/alembic/versions/0033_channel_explore.py new file mode 100644 index 0000000..d12187b --- /dev/null +++ b/backend/alembic/versions/0033_channel_explore.py @@ -0,0 +1,83 @@ +"""channel explore: about-page fields, explore provenance, explored_channels + +Revision ID: 0033_channel_explore +Revises: 0032_search_document +Create Date: 2026-06-30 + +Supports the channel-explore feature (a dedicated channel page + ephemeral browsing of +un-subscribed channels): + - channels.total_view_count / published_at / banner_url / external_links — richer "About" + metadata fetched from channels.list (part=statistics, snippet, brandingSettings). + - channels.from_explore — this channel exists only because someone explored it (not an + organic subscription / not a search find); drives the ephemeral cleanup job. + - videos.via_explore — this video entered the catalog via a channel exploration. Hidden + from everyone's Library by default; visible only to users who explored (or subscribe to) + the channel (a per-user gate in the feed query). + - explored_channels — per-user record of "I explored this channel": gates channel-page + access to its ephemeral videos and drives the grace-period purge of un-kept channels. +All new flags default false / NULL, so every existing row stays organic. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0033_channel_explore" +down_revision: Union[str, None] = "0032_search_document" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("channels", sa.Column("total_view_count", sa.BigInteger(), nullable=True)) + op.add_column( + "channels", sa.Column("published_at", sa.DateTime(timezone=True), nullable=True) + ) + op.add_column("channels", sa.Column("banner_url", sa.String(length=1024), nullable=True)) + op.add_column("channels", sa.Column("external_links", sa.JSON(), nullable=True)) + op.add_column( + "channels", + sa.Column("from_explore", sa.Boolean(), nullable=False, server_default="false"), + ) + op.create_index("ix_channels_from_explore", "channels", ["from_explore"]) + + op.add_column( + "videos", + sa.Column("via_explore", sa.Boolean(), nullable=False, server_default="false"), + ) + op.create_index("ix_videos_via_explore", "videos", ["via_explore"]) + + op.create_table( + "explored_channels", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("channel_id", sa.String(length=32), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["channel_id"], ["channels.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"), + ) + op.create_index("ix_explored_channels_user_id", "explored_channels", ["user_id"]) + op.create_index("ix_explored_channels_channel_id", "explored_channels", ["channel_id"]) + + +def downgrade() -> None: + op.drop_index("ix_explored_channels_channel_id", table_name="explored_channels") + op.drop_index("ix_explored_channels_user_id", table_name="explored_channels") + op.drop_table("explored_channels") + + op.drop_index("ix_videos_via_explore", table_name="videos") + op.drop_column("videos", "via_explore") + + op.drop_index("ix_channels_from_explore", table_name="channels") + op.drop_column("channels", "from_explore") + op.drop_column("channels", "external_links") + op.drop_column("channels", "banner_url") + op.drop_column("channels", "published_at") + op.drop_column("channels", "total_view_count") diff --git a/backend/app/config.py b/backend/app/config.py index 78c1bd8..cdd2a2f 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -118,6 +118,12 @@ class Settings(BaseSettings): # items). Deleted/private/paywalled get a recovery window; abandoned upcoming and # confirmed-dead stuck-live have no age grace (deleted on the sweep that confirms them). maintenance_grace_days: int = 7 + # --- Channel-explore cleanup job --- + # Runs daily by default (admin-tunable via the Scheduler dashboard). An explored-but-never- + # kept channel (no subscription, no per-user state on its videos) is hard-deleted this many + # days after it was last explored — the grace clock is ExploredChannel.created_at. + explore_cleanup_minutes: int = 1440 + explore_grace_days: int = 7 # live_status values hidden from the feed by default. Completed-stream VODs # ("was_live") are real watchable content and stay visible. feed_default_hidden_live: str = "live,upcoming" diff --git a/backend/app/models.py b/backend/app/models.py index a35a10f..8c4ab52 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -165,6 +165,12 @@ class Channel(Base, TimestampMixin): uploads_playlist_id: Mapped[str | None] = mapped_column(String(64)) subscriber_count: Mapped[int | None] = mapped_column(BigInteger) video_count: Mapped[int | None] = mapped_column(BigInteger) + # Channel "About" metadata (fetched with the same channels.list call, no extra quota): + # lifetime view count, the channel's join date, banner image, and external links. + total_view_count: Mapped[int | None] = mapped_column(BigInteger) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + banner_url: Mapped[str | None] = mapped_column(String(1024)) + external_links: Mapped[list | None] = mapped_column(JSON) topic_categories: Mapped[list | None] = mapped_column(JSON) default_language: Mapped[str | None] = mapped_column(String(16)) country: Mapped[str | None] = mapped_column(String(8)) @@ -183,6 +189,14 @@ class Channel(Base, TimestampMixin): from_search: Mapped[bool] = mapped_column( Boolean, default=False, server_default="false", index=True ) + # True when this channel exists only because a user EXPLORED it (opened its page without + # subscribing) — an ephemeral, per-explorer addition. The cleanup job hard-deletes such a + # channel (and its via_explore videos) once nobody keeps it (no subscription, no live + # ExploredChannel row, no per-user state) after a grace period. Cleared when someone + # subscribes (the "keep" signal promotes it to organic). + from_explore: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false", index=True + ) videos: Mapped[list["Video"]] = relationship(back_populates="channel") subscriptions: Mapped[list["Subscription"]] = relationship(back_populates="channel") @@ -260,6 +274,14 @@ class Video(Base, TimestampMixin): via_search: Mapped[bool] = mapped_column( Boolean, default=False, server_default="false", index=True ) + # True when this video entered the catalog because a user EXPLORED its (un-subscribed) + # channel. Like via_search it's hidden from the shared Library by default, but it's + # additionally PER-USER private: a via_explore video is only visible to users who + # explored or subscribe to its channel (a gate in the feed query), so one user's + # curiosity never floods everyone else's catalog. Left False by the organic ingest paths. + via_explore: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false", index=True + ) # none | live | upcoming | premiere | was_live live_status: Mapped[str] = mapped_column( String(16), default="none", server_default="none", index=True @@ -388,6 +410,31 @@ class SearchFind(Base, TimestampMixin): ) +class ExploredChannel(Base, TimestampMixin): + """Per-user record of a channel the user opened the page of WITHOUT subscribing. + + Exploring an un-subscribed channel ingests its recent uploads (flagged via_explore) so the + user can browse them, but keeps them out of everyone else's catalog. This row (a) gates the + explorer's access to those ephemeral videos in the feed query and (b) drives the grace-period + purge: a from_explore channel with no live ExploredChannel row (and no subscription / no + per-user state on its videos) is hard-deleted by the cleanup job. Subscribing is the "keep" + signal — it promotes the channel to organic and these rows become irrelevant. `created_at` + (from TimestampMixin) is the grace clock; a re-visit refreshes it.""" + + __tablename__ = "explored_channels" + __table_args__ = ( + UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + channel_id: Mapped[str] = mapped_column( + ForeignKey("channels.id", ondelete="CASCADE"), index=True + ) + + class ApiQuotaUsage(Base): """Tracks YouTube Data API units spent per Pacific-time day (the quota resets at midnight Pacific). The whole app shares one daily budget.""" diff --git a/backend/app/quota.py b/backend/app/quota.py index 1e72e56..0e44839 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -38,6 +38,7 @@ class QuotaAction: VIDEOS_LOOKUP = "videos_lookup" VIDEOS_SEARCH = "videos_search" CHANNELS_DISCOVER = "channels_discover" + CHANNELS_EXPLORE = "channels_explore" CHANNELS_SUBSCRIBE = "channels_subscribe" CHANNELS_UNSUBSCRIBE = "channels_unsubscribe" MAINTENANCE_REVALIDATE = "maintenance_revalidate" diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index f1d00eb..6ffdd7f 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -5,15 +5,16 @@ import logging from datetime import datetime, timezone from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy import and_, func, select +from sqlalchemy import and_, func, select, update from sqlalchemy.orm import Session -from app import quota -from app.auth import current_user, has_write_scope +from app import quota, sysconfig +from app.auth import current_user, has_write_scope, require_human from app.db import get_db from app.models import ( Channel, ChannelTag, + ExploredChannel, Playlist, PlaylistItem, Subscription, @@ -22,6 +23,7 @@ from app.models import ( Video, ) from app.routes.admin import admin_user +from app.sync.explore import explore_ingest_page from app.sync.runner import run_recent_backfill from app.sync.subscriptions import apply_channel_details from app.youtube.client import YouTubeClient, YouTubeError @@ -205,6 +207,133 @@ def discover_channels( ] +def _channel_detail_dict( + channel: Channel, *, subscribed: bool, explored: bool, stored: int +) -> dict: + return { + "id": channel.id, + "title": channel.title, + "handle": channel.handle, + "description": channel.description, + "thumbnail_url": channel.thumbnail_url, + "banner_url": channel.banner_url, + "subscriber_count": channel.subscriber_count, + "video_count": channel.video_count, + "total_view_count": channel.total_view_count, + "published_at": channel.published_at.isoformat() if channel.published_at else None, + "external_links": channel.external_links or [], + "country": channel.country, + "subscribed": subscribed, + "explored": explored, + "stored_videos": stored, + "from_explore": channel.from_explore, + "details_synced": channel.details_synced_at is not None, + } + + +@router.get("/{channel_id}") +def channel_detail( + channel_id: str, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + """One channel's "About" detail for the channel page header/About tab, plus this user's + relationship to it (subscribed / explored) and how many of its videos are in the catalog. + Lazily fills the About metadata (view count, join date, banner, links) the first time it's + needed — a stub, or a channel last enriched before those fields existed (published_at is a + reliable sentinel: every real channel has a creation date). The enrich uses the API key + (1 unit, cached after), so it's skipped for the quota-less demo account.""" + channel = db.get(Channel, channel_id) + if channel is None: + raise HTTPException(status_code=404, detail="Unknown channel") + if channel.published_at is None and not user.is_demo: + try: + with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt: + apply_channel_details(db, yt.get_channels([channel_id])) + db.commit() + channel = db.get(Channel, channel_id) + except YouTubeError as exc: + log.warning("channel detail enrich failed (%s): %s", channel_id, exc) + db.rollback() + subscribed = db.execute( + select(Subscription.id).where( + Subscription.user_id == user.id, Subscription.channel_id == channel_id + ) + ).first() is not None + explored = db.execute( + select(ExploredChannel.id).where( + ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id + ) + ).first() is not None + stored = db.scalar(select(func.count(Video.id)).where(Video.channel_id == channel_id)) or 0 + return _channel_detail_dict( + channel, subscribed=subscribed, explored=explored, stored=int(stored) + ) + + +@router.post("/{channel_id}/explore") +def explore_channel( + channel_id: str, + page_token: str | None = None, + user: User = Depends(require_human), + db: Session = Depends(get_db), +) -> dict: + """Open an un-subscribed channel for browsing. Records the exploration (a per-user, + grace-clocked row that gates this user's access to the channel's ephemeral videos and + drives the later cleanup), flags the channel ephemeral unless someone organically follows + it, then ingests one page of its uploads (newest-first) marked via_explore + enriched. + Returns the next page token so the page can "load more". Guarded against the shared + backfill quota reserve; blocked for the demo account (it spends shared quota).""" + channel = db.get(Channel, channel_id) + if channel is None: + raise HTTPException(status_code=404, detail="Unknown channel") + if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): + raise HTTPException( + status_code=429, + detail="The shared daily quota reserve is reached — try exploring later.", + ) + + # Record/refresh the exploration (DB only) before ingesting, so the explorer can see the + # videos as soon as they land. Flag the channel ephemeral unless it's organically followed. + existing = db.execute( + select(ExploredChannel).where( + ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id + ) + ).scalar_one_or_none() + if existing is None: + db.add(ExploredChannel(user_id=user.id, channel_id=channel_id)) + else: + existing.created_at = datetime.now(timezone.utc) # refresh the grace clock + followed = db.execute( + select(Subscription.id).where(Subscription.channel_id == channel_id).limit(1) + ).first() + if followed is None and not channel.from_explore: + channel.from_explore = True + db.commit() + + try: + with quota.attribute(user.id, quota.QuotaAction.CHANNELS_EXPLORE), YouTubeClient(db, user) as yt: + # Enrich for the About panel + the uploads playlist id (needed to ingest videos). + if ( + channel.details_synced_at is None + or channel.uploads_playlist_id is None + or channel.published_at is None + ): + apply_channel_details(db, yt.get_channels([channel_id])) + db.commit() + channel = db.get(Channel, channel_id) + result = explore_ingest_page(db, yt, channel, page_token) + except YouTubeError as exc: + raise HTTPException( + status_code=422, detail=f"YouTube couldn't load this channel: {exc}" + ) + return { + "channel_id": channel_id, + "ingested": result["ingested"], + "next_page_token": result["next_page_token"], + } + + def _user_subscription(db: Session, user: User, channel_id: str) -> Subscription: sub = db.execute( select(Subscription).where( @@ -332,6 +461,14 @@ def subscribe( subscribed_at=datetime.now(timezone.utc), ) ) + # Subscribing is the "keep" signal: a previously-explored channel becomes a normal organic + # one. Drop its ephemeral flags so the cleanup job leaves it alone and its videos stop being + # per-user private (they join the shared Library like any subscribed channel's uploads). + if channel.from_explore: + channel.from_explore = False + db.execute( + update(Video).where(Video.channel_id == channel_id, Video.via_explore.is_(True)).values(via_explore=False) + ) db.commit() return {"subscribed": channel_id} diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 528a42e..5969edb 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -16,6 +16,7 @@ from app.db import get_db from app.models import ( Channel, ChannelTag, + ExploredChannel, Playlist, PlaylistItem, SearchFind, @@ -180,6 +181,23 @@ def _filtered_query( # either way they shouldn't show in any feed view meanwhile. query = query.where(Video.unavailable_since.is_(None)) + # Explored videos are PER-EXPLORER private: a via_explore video (ingested when someone + # opened an un-subscribed channel's page) is visible only to users who explored or + # subscribe to its channel, so one user's curiosity never leaks into everyone else's + # catalog. Applied in BOTH scopes; the channel page reaches its ephemeral videos by + # asking for scope=all + channel_id (the channel is in this accessible set). + accessible_explore = ( + select(Subscription.channel_id).where(Subscription.user_id == user.id) + ).union( + select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id) + ) + query = query.where( + or_( + Video.via_explore.is_(False), + Video.channel_id.in_(accessible_explore), + ) + ) + # Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag # (search-discovered by anyone). In "my" scope it's the user's OWN provenance: # "organic" = your subscriptions, "search" = videos you found via your own search, diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index fd8de99..b39bd95 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -16,6 +16,7 @@ from app.models import SchedulerSetting from app.notifications import create_notification from app.state import is_sync_paused from app.sync.autotag import run_autotag_all +from app.sync.explore import purge_unkept_explored from app.sync.maintenance import run_maintenance from app.sync.playlists import sync_all_playlists from app.sync.runner import ( @@ -57,6 +58,7 @@ JOB_INTERVALS: dict[str, int] = { "playlist_sync": settings.playlist_sync_minutes, "maintenance": settings.maintenance_interval_minutes, "demo_reset": settings.demo_reset_minutes, + "explore_cleanup": settings.explore_cleanup_minutes, } # Sane bounds for an admin-set interval (minutes). @@ -268,6 +270,12 @@ def _demo_reset_job() -> None: _job("demo_reset", work) +def _explore_cleanup_job() -> None: + # Hard-delete explored-but-never-kept channels and their untouched ephemeral videos after + # the grace period. 0 quota (pure DB), so no quota attribution. + _job("explore_cleanup", purge_unkept_explored) + + # job_id -> wrapper. The single source of truth for which jobs exist and how to run one, # shared by start_scheduler (recurring registration) and trigger_job (manual "run now"). JOB_FUNCS: dict[str, Callable[[], None]] = { @@ -280,6 +288,7 @@ JOB_FUNCS: dict[str, Callable[[], None]] = { "playlist_sync": _playlist_sync_job, "maintenance": _maintenance_job, "demo_reset": _demo_reset_job, + "explore_cleanup": _explore_cleanup_job, } diff --git a/backend/app/sync/explore.py b/backend/app/sync/explore.py new file mode 100644 index 0000000..7257862 --- /dev/null +++ b/backend/app/sync/explore.py @@ -0,0 +1,145 @@ +"""Channel exploration: ingest an un-subscribed channel's uploads on demand (flagged +via_explore, kept per-explorer) and purge channels nobody ends up keeping. + +A user opens a channel's page without subscribing → we pull a page of its recent uploads, +mark the new rows via_explore, and enrich them so they're immediately browsable. The videos +stay private to users who explored (or subscribe to) the channel — the feed query gates them +— so one user's curiosity never floods everyone's catalog. Subscribing is the "keep" signal; +otherwise the cleanup job hard-deletes the channel and its untouched ephemeral videos after a +grace period (mirrors the maintenance job's grace-then-delete safety).""" +import logging +from datetime import timedelta + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session + +from app import sysconfig +from app.models import ( + Channel, + ExploredChannel, + PlaylistItem, + SearchFind, + Subscription, + Video, + VideoState, +) +from app.sync.videos import _insert_stubs, _stub_from_playlist_item, apply_video_details +from app.utils import now_utc as _now +from app.youtube.client import YouTubeClient + +log = logging.getLogger("siftlode.explore") + + +def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int: + """Enrich just the not-yet-enriched videos among `video_ids` (videos.list, 1 unit/50).""" + if not video_ids: + return 0 + videos = ( + db.execute( + select(Video).where( + Video.id.in_(video_ids), Video.enriched_at.is_(None) + ) + ) + .scalars() + .all() + ) + if not videos: + return 0 + items = {it["id"]: it for it in yt.get_videos([v.id for v in videos])} + now = _now() + for video in videos: + item = items.get(video.id) + if item is None: + video.enriched_at = now # unavailable; stop retrying + continue + apply_video_details(video, item) + video.enriched_at = now + db.commit() + return len(videos) + + +def explore_ingest_page( + db: Session, yt: YouTubeClient, channel: Channel, page_token: str | None = None +) -> dict: + """Ingest one page (up to 50, newest-first) of the channel's uploads, marking the new + videos via_explore, then enrich them. Returns the count ingested and the next page token + (None when the back-catalogue is exhausted) so the caller can offer "load more". Stateless + pagination: the caller round-trips the token, so the channel's own backfill cursor (used by + the organic deep backfill) is never touched.""" + if not channel.uploads_playlist_id: + return {"ingested": 0, "next_page_token": None} + data = yt.get_playlist_items(channel.uploads_playlist_id, page_token) + stubs: list[dict] = [] + for item in data.get("items", []): + stub = _stub_from_playlist_item(item, channel.id) + if stub is not None: + stub["via_explore"] = True + stubs.append(stub) + inserted = _insert_stubs(db, stubs) + self_enriched = _enrich_new(db, yt, [s["id"] for s in stubs]) + log.info( + "explore ingest channel=%s ingested=%s enriched=%s", channel.id, inserted, self_enriched + ) + return {"ingested": inserted, "next_page_token": data.get("nextPageToken")} + + +def purge_unkept_explored(db: Session) -> dict: + """Cleanup job: hard-delete explored channels nobody kept and their untouched ephemeral + videos, after a grace period (mirrors the maintenance grace-then-delete safety). + + Keep signals that spare a channel/video: + - any user SUBSCRIBES to the channel (subscribe promotes it to organic — channel-level), or + - a live ExploredChannel row (someone explored it within the grace window — channel-level), or + - a video has per-user state: watched/in-progress (VideoState), in a playlist (PlaylistItem), + or surfaced by the user's own search (SearchFind) — these survive even if the channel goes. + A channel row is dropped only once it holds no videos at all. 0 quota.""" + grace_days = sysconfig.get_int(db, "explore_grace_days") + cutoff = _now() - timedelta(days=grace_days) + + expired = db.execute( + delete(ExploredChannel).where(ExploredChannel.created_at < cutoff) + ).rowcount + db.commit() + + sub_exists = ( + select(Subscription.id).where(Subscription.channel_id == Channel.id).exists() + ) + exp_exists = ( + select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists() + ) + orphan_ids = ( + db.execute( + select(Channel.id).where( + Channel.from_explore.is_(True), ~sub_exists, ~exp_exists + ) + ) + .scalars() + .all() + ) + if not orphan_ids: + return {"expired": expired, "videos_deleted": 0, "channels_deleted": 0} + + state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists() + pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists() + find_exists = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists() + videos_deleted = db.execute( + delete(Video).where( + Video.channel_id.in_(orphan_ids), + Video.via_explore.is_(True), + ~state_exists, + ~pl_exists, + ~find_exists, + ) + ).rowcount + db.commit() + + has_videos = select(Video.id).where(Video.channel_id == Channel.id).exists() + channels_deleted = db.execute( + delete(Channel).where(Channel.id.in_(orphan_ids), ~has_videos) + ).rowcount + db.commit() + return { + "expired": expired, + "videos_deleted": videos_deleted, + "channels_deleted": channels_deleted, + } diff --git a/backend/app/sync/subscriptions.py b/backend/app/sync/subscriptions.py index adafce3..c0f643a 100644 --- a/backend/app/sync/subscriptions.py +++ b/backend/app/sync/subscriptions.py @@ -18,6 +18,29 @@ def _to_int(value) -> int | None: return None +def _parse_dt(value: str | None) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def _external_links(branding: dict) -> list | None: + """Channel "About" links. The API exposes a few in brandingSettings.channel.profileColor/ + /... varies; the durable spot is brandingSettings.channel.links is gone post-2023, so we + read what's present (settings/links) defensively and keep [{title, url}] pairs.""" + channel = branding.get("channel", {}) + links = channel.get("links") or [] + out = [ + {"title": l.get("title"), "url": l.get("url")} + for l in links + if isinstance(l, dict) and l.get("url") + ] + return out or None + + def apply_channel_details(db: Session, items: list[dict]) -> None: now = datetime.now(timezone.utc) for item in items: @@ -28,6 +51,7 @@ def apply_channel_details(db: Session, items: list[dict]) -> None: content = item.get("contentDetails", {}) stats = item.get("statistics", {}) topics = item.get("topicDetails", {}) + branding = item.get("brandingSettings", {}) channel.title = snippet.get("title") or channel.title channel.handle = snippet.get("customUrl") @@ -40,6 +64,10 @@ def apply_channel_details(db: Session, items: list[dict]) -> None: channel.uploads_playlist_id = content.get("relatedPlaylists", {}).get("uploads") channel.subscriber_count = _to_int(stats.get("subscriberCount")) channel.video_count = _to_int(stats.get("videoCount")) + channel.total_view_count = _to_int(stats.get("viewCount")) + channel.published_at = _parse_dt(snippet.get("publishedAt")) + channel.banner_url = branding.get("image", {}).get("bannerExternalUrl") + channel.external_links = _external_links(branding) channel.topic_categories = topics.get("topicCategories") channel.details_synced_at = now diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index c6d8004..133b0bb 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -51,6 +51,9 @@ SPECS: tuple[ConfigSpec, ...] = ( # --- Batch sizes --- ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50), ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000), + # --- Channel explore --- + # Days an explored-but-unsubscribed channel is kept before the cleanup job purges it. + ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650), # --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) --- ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True), # Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key). diff --git a/backend/app/youtube/client.py b/backend/app/youtube/client.py index b4e5ef6..0fc0ebf 100644 --- a/backend/app/youtube/client.py +++ b/backend/app/youtube/client.py @@ -187,7 +187,7 @@ class YouTubeClient: data = self._get( "channels", { - "part": "snippet,contentDetails,statistics,topicDetails", + "part": "snippet,contentDetails,statistics,topicDetails,brandingSettings", "id": ",".join(batch), "maxResults": 50, }, diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b777c3a..73d36d3 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -28,6 +28,8 @@ import Header from "./components/Header"; import NavSidebar from "./components/NavSidebar"; import Sidebar from "./components/Sidebar"; import Feed from "./components/Feed"; +import ChannelPage from "./components/ChannelPage"; +import BackToTop from "./components/BackToTop"; import Channels, { type ChannelStatusFilter, type ChannelsView } from "./components/Channels"; import Playlists from "./components/Playlists"; import Stats from "./components/Stats"; @@ -140,6 +142,27 @@ export default function App() { if (window.history.state?._yt) window.history.back(); else setYtSearch(null); }, []); + // The open channel page (null = no channel page). Like the YouTube-search sub-view it owns a + // browser-history entry (history.state._chan = channel id, _chanName = a display name to show + // before the detail loads), so Back closes the channel page (after any player opened over it) + // and reload returns to the normal app. Channels/videos are reached by id, so it's not in the URL. + const [channelView, setChannelView] = useState<{ id: string; name?: string } | null>(() => { + // Restore the open channel page across a reload (it rides in history.state, which survives F5), + // unlike the YouTube search which is intentionally dropped (it would re-spend quota). + const c = window.history.state?._chan as string | undefined; + return c ? { id: c, name: (window.history.state?._chanName as string) ?? undefined } : null; + }); + const openChannel = useCallback((id: string, name?: string) => { + setChannelView({ id, name }); + const st = window.history.state || {}; + if (st._chan) + window.history.replaceState({ ...st, _chan: id, _chanName: name ?? null }, ""); + else window.history.pushState({ ...st, _chan: id, _chanName: name ?? null }, ""); + }, []); + const closeChannel = useCallback(() => { + if (window.history.state?._chan) window.history.back(); + else setChannelView(null); + }, []); const [wizardOpen, setWizardOpen] = useState(false); // Channel status chip, persisted so a reload (F5) keeps it; clamp a stale value at render. const [channelFilterRaw, setChannelFilter] = usePersistedState(LS.channelFilter, "all"); @@ -186,14 +209,17 @@ export default function App() { } function setPage(next: Page) { - if (next === page) return; + // While a channel page is open it overlays the content column, so a nav click must close it + // even when the underlying page is unchanged (e.g. "Feed" while a channel opened from the feed). + if (next === page && !channelView) return; const go = () => { + setChannelView(null); // leave any open channel page when navigating via the rail setPageState(next); localStorage.setItem(PAGE_KEY, next); // Push an in-app history entry so the browser/mouse Back button steps through pages // instead of leaving the app (e.g. back to the OAuth redirect). The URL stays clean — // the page rides in history.state, not the query string (filters never go in the URL). - // A fresh { sfPage } (no carried-over _sub/_ov markers) so each page starts at its root. + // A fresh { sfPage } (no carried-over _sub/_ov/_chan markers) so each page starts at its root. window.history.pushState({ sfPage: next }, ""); }; // Guard leaving the Settings page with unsaved preference changes. @@ -247,6 +273,8 @@ export default function App() { useEffect(() => { // Drop any stale _yt from a prior session (a reload starts on the normal feed; ytSearch // begins null), so the first Back doesn't resurrect a search we're no longer showing. + // Drop a stale _yt (a reload starts on the normal feed; a search would re-spend quota), but + // KEEP _chan/_chanName so the open channel page survives F5 (channelView restores from it above). const { _yt: _staleYt, ...rest } = window.history.state || {}; window.history.replaceState({ ...rest, sfPage: page }, ""); function onPop(e: PopStateEvent) { @@ -273,6 +301,9 @@ export default function App() { // The YouTube-search sub-view rides in history.state._yt, so Back/Forward restores or // clears it to match the entry we landed on (and a player popped first leaves it intact). setYtSearch((e.state?._yt as string) ?? null); + // The channel page rides in history.state._chan the same way. + const chan = e.state?._chan as string | undefined; + setChannelView(chan ? { id: chan, name: (e.state?._chanName as string) ?? undefined } : null); } window.addEventListener("popstate", onPop); return () => window.removeEventListener("popstate", onPop); @@ -518,6 +549,18 @@ export default function App() { language={i18n.language as LangCode} />
+ {channelView ? ( + + ) : ( + <>
setWizardOpen(true)} - onViewChannel={(id, name) => { - setFilters({ ...filters, channelId: id, channelName: name, show: "all" }); - setPage("feed"); - }} + onViewChannel={(id, name) => openChannel(id, name)} onFilterByTag={(tagId, name) => { setFilters({ ...filters, tags: [tagId], channelId: undefined, channelName: undefined }); setPage("feed"); @@ -599,10 +639,13 @@ export default function App() { ytSearch={ytSearch} onYtSearch={enterYtSearch} onExitYtSearch={exitYtSearch} + onOpenChannel={openChannel} /> )}
+ + )} {/* Toasts rise from the bottom-left, by the notification bell in the nav rail. Anchored inside the content column so they clear the sidebar automatically (collapsed or expanded) without tracking its width. */} @@ -625,6 +668,8 @@ export default function App() { setNotesOpen(false)} highlight={notesHighlight} /> )} + {/* Re-binds to the active page's scroll container on navigation (page or channel change). */} + ); } diff --git a/frontend/src/components/BackToTop.tsx b/frontend/src/components/BackToTop.tsx new file mode 100644 index 0000000..26bc792 --- /dev/null +++ b/frontend/src/components/BackToTop.tsx @@ -0,0 +1,46 @@ +import { useEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; +import { ArrowUp } from "lucide-react"; +import { useTranslation } from "react-i18next"; + +// A floating "back to top" button that fades in once the current page's scroll container is +// scrolled past a threshold, and smooth-scrolls it back to the top. Every page scrolls inside its +// own
(the normal pages share App's; the channel page has its own), +// so it targets the live
and re-binds whenever `dep` changes (i.e. on navigation). Portaled +// to so its fixed position is viewport-relative regardless of transformed ancestors. +export default function BackToTop({ dep, threshold = 600 }: { dep?: unknown; threshold?: number }) { + const { t } = useTranslation(); + const [show, setShow] = useState(false); + const scrollerRef = useRef(null); + + useEffect(() => { + const el = document.querySelector("main"); + scrollerRef.current = el; + if (!el) { + setShow(false); + return; + } + const onScroll = () => setShow(el.scrollTop > threshold); + onScroll(); // a freshly-bound page starts at the top → hidden + el.addEventListener("scroll", onScroll, { passive: true }); + return () => el.removeEventListener("scroll", onScroll); + }, [dep, threshold]); + + const toTop = () => scrollerRef.current?.scrollTo({ top: 0, behavior: "smooth" }); + + // No aria-label/title attributes: ad-block "annoyance" cosmetic filters (e.g. Brave Shields) + // commonly hide floating corner buttons via attribute selectors like [aria-label="Back to top"]. + // The accessible name comes from a visually-hidden child instead, which those selectors can't match. + return createPortal( + , + document.body + ); +} diff --git a/frontend/src/components/ChannelPage.tsx b/frontend/src/components/ChannelPage.tsx new file mode 100644 index 0000000..fc856f7 --- /dev/null +++ b/frontend/src/components/ChannelPage.tsx @@ -0,0 +1,309 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ArrowLeft, Check, ExternalLink, Loader2, Plus, RefreshCw } from "lucide-react"; +import Avatar from "./Avatar"; +import Feed from "./Feed"; +import { useConfirm } from "./ConfirmProvider"; +import { notify } from "../lib/notifications"; +import { api, type FeedFilters, type Me } from "../lib/api"; +import { channelYouTubeUrl, formatViews } from "../lib/format"; + +// A dedicated channel page: an "About"-style header (banner, avatar, stats, subscribe) over the +// channel's videos (the catalog filtered to this channel). For an un-subscribed channel it +// auto-ingests recent uploads in the background ("explore") so they're browsable immediately, +// and offers "load more" to page deeper — without permanently polluting everyone's library +// (the backend keeps explored videos private to this user until they subscribe). +export default function ChannelPage({ + channelId, + initialName, + me, + view, + onBack, + onOpenChannel, +}: { + channelId: string; + initialName?: string; + me: Me; + view: "grid" | "list"; + onBack: () => void; + onOpenChannel: (id: string, name?: string) => void; +}) { + const { t, i18n } = useTranslation(); + const qc = useQueryClient(); + const confirm = useConfirm(); + const [tab, setTab] = useState<"videos" | "about">("videos"); + + const detail = useQuery({ + queryKey: ["channel", channelId], + queryFn: () => api.channelDetail(channelId), + }); + const ch = detail.data; + + // Background auto-ingest ("explore") of an un-subscribed channel's uploads. + const [nextToken, setNextToken] = useState(undefined); + const [exploring, setExploring] = useState(false); + const autoTried = useRef(null); + + const runExplore = useCallback( + (token?: string | null) => { + setExploring(true); + return api + .exploreChannel(channelId, token ?? undefined) + .then((r) => { + setNextToken(r.next_page_token); + qc.invalidateQueries({ queryKey: ["feed"] }); + // Refetch the detail so `explored` flips true → the "Exploring" badge shows on the + // first visit (the GET that drove this render predated the explore that just ran). + qc.invalidateQueries({ queryKey: ["channel", channelId] }); + }) + .finally(() => setExploring(false)); + }, + [channelId, qc] + ); + + // First visit of an un-subscribed channel → ingest its recent uploads. Skipped for the demo + // account (explore spends shared quota) and when videos are already present (a revisit, or a + // subscribed channel whose uploads the scheduler backfills). + useEffect(() => { + if (!ch || me.is_demo) return; + if (autoTried.current === channelId) return; + if (ch.subscribed) return; + if (ch.explored && ch.stored_videos > 0) return; + autoTried.current = channelId; + runExplore().catch(() => {}); + }, [ch, channelId, me.is_demo, runExplore]); + + const subscribe = useMutation({ + mutationFn: () => api.subscribeChannel(channelId), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["channel", channelId] }); + qc.invalidateQueries({ queryKey: ["feed"] }); + qc.invalidateQueries({ queryKey: ["channels"] }); + notify({ level: "info", message: t("channel.subscribed", { name: ch?.title ?? "" }) }); + }, + }); + const unsubscribe = useMutation({ + mutationFn: () => api.unsubscribeChannel(channelId), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["channel", channelId] }); + qc.invalidateQueries({ queryKey: ["feed"] }); + qc.invalidateQueries({ queryKey: ["channels"] }); + }, + }); + const onUnsubscribe = async () => { + const ok = await confirm({ + title: t("channel.unsubTitle"), + message: t("channel.unsubBody", { name: ch?.title ?? channelId }), + confirmLabel: t("channel.unsubscribe"), + danger: true, + }); + if (ok) unsubscribe.mutate(); + }; + + // Channel-scoped feed: the whole catalog filtered to this channel (scope=all + source=all so + // the explored, per-user-private videos show — the backend gates them to this explorer). + const [filters, setFilters] = useState(() => ({ + tags: [], + tagMode: "or", + q: "", + sort: "newest", + scope: "all", + librarySource: "all", + includeNormal: true, + includeShorts: false, + includeLive: true, + show: "all", + channelId, + channelName: initialName, + })); + + const name = ch?.title ?? initialName ?? channelId; + const ytUrl = channelYouTubeUrl(channelId, ch?.handle); + const joined = ch?.published_at + ? new Date(ch.published_at).toLocaleDateString(i18n.language, { year: "numeric", month: "short" }) + : null; + // Handle + stats on one compact meta line under the name (saves the separate stats row). + const metaParts = [ + ch?.handle ? `@${ch.handle.replace(/^@/, "")}` : null, + ch?.subscriber_count != null ? t("channel.subscribers", { formatted: formatViews(ch.subscriber_count) }) : null, + ch?.video_count != null ? t("channel.videoCount", { count: ch.video_count }) : null, + ch?.total_view_count != null ? t("channel.totalViews", { formatted: formatViews(ch.total_view_count) }) : null, + joined ? t("channel.joined", { date: joined }) : null, + ].filter(Boolean) as string[]; + const tabClass = (active: boolean) => + active + ? "pb-2 text-sm font-medium border-b-2 border-fg text-fg" + : "pb-2 text-sm text-muted hover:text-fg border-b-2 border-transparent"; + + return ( +
+ {/* Banner + back — OPTION C: inset, rounded card (not full-bleed) */} +
+ {ch?.banner_url ? ( + // Match YouTube's banner: the stored bannerExternalUrl is the full 16:9 template at a + // low default size, so request a crisp wide version (=w1707) and object-cover the + // desktop "safe area" — the centre 2560×423 (~6:1) band. +
+ +
+ ) : ( +
+ )} + +
+ +
+ {/* Identity + actions. relative+z-10 so the avatar that overlaps up into the banner + paints ABOVE it (the banner's positioned container would otherwise cover it). */} +
+ +
+
+

{name}

+ {ch?.explored && !ch?.subscribed && ( + + {t("channel.exploringBadge")} + + )} +
+
+ {metaParts.map((part, i) => ( + + {i > 0 && ·} + {part} + + ))} +
+
+
+ + + + {!me.is_demo && + (ch?.subscribed ? ( + + ) : ( + + ))} +
+
+ + {exploring && ( +
+ + {t("channel.loadingVideos")} +
+ )} + + {/* Tabs */} +
+ + +
+
+ + {/* Tab content */} + {tab === "videos" ? ( + <> + {}} + ytSearch={null} + onYtSearch={() => {}} + onExitYtSearch={() => {}} + onOpenChannel={onOpenChannel} + channelScoped + /> + {!ch?.subscribed && nextToken && ( +
+ +
+ )} + + ) : ( +
+ {ch?.description ? ( +

{ch.description}

+ ) : ( +

{t("channel.noDescription")}

+ )} + {ch?.external_links && ch.external_links.length > 0 && ( +
+ {ch.external_links.map((l, i) => ( + + + {l.title || l.url} + + ))} +
+ )} +
+ )} +
+ ); +} diff --git a/frontend/src/components/Feed.tsx b/frontend/src/components/Feed.tsx index 7205633..6c66450 100644 --- a/frontend/src/components/Feed.tsx +++ b/frontend/src/components/Feed.tsx @@ -77,6 +77,8 @@ export default function Feed({ ytSearch, onYtSearch, onExitYtSearch, + onOpenChannel, + channelScoped, }: { filters: FeedFilters; setFilters: (f: FeedFilters) => void; @@ -91,8 +93,17 @@ export default function Feed({ ytSearch: string | null; onYtSearch: (q: string) => void; onExitYtSearch: () => void; + // Open a channel's dedicated page (from a card's channel name / avatar). Threaded down to + // the cards via VirtualFeed. + onOpenChannel?: (channelId: string, channelName?: string) => void; + // True when this feed is scoped to a single channel (the channel page). Drops sort options + // that are constant within one channel (subscribers, priority) — they'd be meaningless. + channelScoped?: boolean; }) { const { t } = useTranslation(); + const sortKeys = channelScoped + ? SORT_KEYS.filter((k) => k !== "subscribers" && k !== "priority") + : SORT_KEYS; const [overrides, setOverrides] = useState>({}); const [savedOverrides, setSavedOverrides] = useState>({}); // The open player: which video and where to start (null = resume from saved position). @@ -257,11 +268,11 @@ export default function Feed({ [qc] ); - const onChannelFilter = useCallback( + const handleOpenChannel = useCallback( (channelId: string, channelName: string) => { - setFilters({ ...filters, channelId, channelName }); + onOpenChannel?.(channelId, channelName); }, - [filters, setFilters] + [onOpenChannel] ); const onToggleSave = useCallback( @@ -347,7 +358,7 @@ export default function Feed({ view={view} onState={onState} onToggleSave={onToggleSave} - onChannelFilter={onChannelFilter} + onOpenChannel={handleOpenChannel} onResetState={onResetState} onOpen={openVideo} hasNextPage={false} @@ -378,6 +389,7 @@ export default function Feed({ startAt={activeVideo.startAt} onClose={() => setActiveVideo(null)} onState={onState} + onOpenChannel={onOpenChannel} /> )} @@ -430,7 +442,7 @@ export default function Feed({ const toolbar = (
-
+
{SHOW_IDS.map((id) => (
-
- +