From bc4c3624239c0399cd4a06ac4adf6cdd874d48b1 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 30 Jun 2026 02:52:44 +0200 Subject: [PATCH] =?UTF-8?q?feat(channels):=20channel-explore=20backend=20?= =?UTF-8?q?=E2=80=94=20about=20metadata,=20ephemeral=20provenance,=20clean?= =?UTF-8?q?up?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the backend for a dedicated channel page + ephemeral browsing of un-subscribed channels: - migration 0033: channels.{total_view_count,published_at,banner_url,external_links, from_explore}, videos.via_explore, explored_channels (per-user, grace-clocked). - enrich channels with part=brandingSettings (banner/links) + statistics.viewCount + snippet.publishedAt — no extra quota. - GET /api/channels/{id}: About detail + this user's relationship; lazy-enriches the new About fields (published_at sentinel). - POST /api/channels/{id}/explore (require_human, quota-reserve guarded): records the exploration, flags the channel ephemeral unless followed, ingests one page of uploads (via_explore) + enriches; returns next_page_token for load-more. - feed: per-explorer gate — a via_explore video is visible only to users who explored or subscribe to its channel, so exploration never leaks into everyone's catalog. - subscribe = keep: clears from_explore + via_explore on the channel's videos. - scheduler explore_cleanup job + explore_grace_days config: hard-delete explored-but- unkept channels and their untouched ephemeral videos after a grace period. --- .../alembic/versions/0033_channel_explore.py | 83 ++++++++++ backend/app/config.py | 6 + backend/app/models.py | 47 ++++++ backend/app/quota.py | 1 + backend/app/routes/channels.py | 143 ++++++++++++++++- backend/app/routes/feed.py | 18 +++ backend/app/scheduler.py | 9 ++ backend/app/sync/explore.py | 145 ++++++++++++++++++ backend/app/sync/subscriptions.py | 28 ++++ backend/app/sysconfig.py | 3 + backend/app/youtube/client.py | 2 +- 11 files changed, 481 insertions(+), 4 deletions(-) create mode 100644 backend/alembic/versions/0033_channel_explore.py create mode 100644 backend/app/sync/explore.py diff --git a/backend/alembic/versions/0033_channel_explore.py b/backend/alembic/versions/0033_channel_explore.py new file mode 100644 index 0000000..d12187b --- /dev/null +++ b/backend/alembic/versions/0033_channel_explore.py @@ -0,0 +1,83 @@ +"""channel explore: about-page fields, explore provenance, explored_channels + +Revision ID: 0033_channel_explore +Revises: 0032_search_document +Create Date: 2026-06-30 + +Supports the channel-explore feature (a dedicated channel page + ephemeral browsing of +un-subscribed channels): + - channels.total_view_count / published_at / banner_url / external_links — richer "About" + metadata fetched from channels.list (part=statistics, snippet, brandingSettings). + - channels.from_explore — this channel exists only because someone explored it (not an + organic subscription / not a search find); drives the ephemeral cleanup job. + - videos.via_explore — this video entered the catalog via a channel exploration. Hidden + from everyone's Library by default; visible only to users who explored (or subscribe to) + the channel (a per-user gate in the feed query). + - explored_channels — per-user record of "I explored this channel": gates channel-page + access to its ephemeral videos and drives the grace-period purge of un-kept channels. +All new flags default false / NULL, so every existing row stays organic. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0033_channel_explore" +down_revision: Union[str, None] = "0032_search_document" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("channels", sa.Column("total_view_count", sa.BigInteger(), nullable=True)) + op.add_column( + "channels", sa.Column("published_at", sa.DateTime(timezone=True), nullable=True) + ) + op.add_column("channels", sa.Column("banner_url", sa.String(length=1024), nullable=True)) + op.add_column("channels", sa.Column("external_links", sa.JSON(), nullable=True)) + op.add_column( + "channels", + sa.Column("from_explore", sa.Boolean(), nullable=False, server_default="false"), + ) + op.create_index("ix_channels_from_explore", "channels", ["from_explore"]) + + op.add_column( + "videos", + sa.Column("via_explore", sa.Boolean(), nullable=False, server_default="false"), + ) + op.create_index("ix_videos_via_explore", "videos", ["via_explore"]) + + op.create_table( + "explored_channels", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("channel_id", sa.String(length=32), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["channel_id"], ["channels.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"), + ) + op.create_index("ix_explored_channels_user_id", "explored_channels", ["user_id"]) + op.create_index("ix_explored_channels_channel_id", "explored_channels", ["channel_id"]) + + +def downgrade() -> None: + op.drop_index("ix_explored_channels_channel_id", table_name="explored_channels") + op.drop_index("ix_explored_channels_user_id", table_name="explored_channels") + op.drop_table("explored_channels") + + op.drop_index("ix_videos_via_explore", table_name="videos") + op.drop_column("videos", "via_explore") + + op.drop_index("ix_channels_from_explore", table_name="channels") + op.drop_column("channels", "from_explore") + op.drop_column("channels", "external_links") + op.drop_column("channels", "banner_url") + op.drop_column("channels", "published_at") + op.drop_column("channels", "total_view_count") diff --git a/backend/app/config.py b/backend/app/config.py index 78c1bd8..cdd2a2f 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -118,6 +118,12 @@ class Settings(BaseSettings): # items). Deleted/private/paywalled get a recovery window; abandoned upcoming and # confirmed-dead stuck-live have no age grace (deleted on the sweep that confirms them). maintenance_grace_days: int = 7 + # --- Channel-explore cleanup job --- + # Runs daily by default (admin-tunable via the Scheduler dashboard). An explored-but-never- + # kept channel (no subscription, no per-user state on its videos) is hard-deleted this many + # days after it was last explored — the grace clock is ExploredChannel.created_at. + explore_cleanup_minutes: int = 1440 + explore_grace_days: int = 7 # live_status values hidden from the feed by default. Completed-stream VODs # ("was_live") are real watchable content and stay visible. feed_default_hidden_live: str = "live,upcoming" diff --git a/backend/app/models.py b/backend/app/models.py index a35a10f..8c4ab52 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -165,6 +165,12 @@ class Channel(Base, TimestampMixin): uploads_playlist_id: Mapped[str | None] = mapped_column(String(64)) subscriber_count: Mapped[int | None] = mapped_column(BigInteger) video_count: Mapped[int | None] = mapped_column(BigInteger) + # Channel "About" metadata (fetched with the same channels.list call, no extra quota): + # lifetime view count, the channel's join date, banner image, and external links. + total_view_count: Mapped[int | None] = mapped_column(BigInteger) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + banner_url: Mapped[str | None] = mapped_column(String(1024)) + external_links: Mapped[list | None] = mapped_column(JSON) topic_categories: Mapped[list | None] = mapped_column(JSON) default_language: Mapped[str | None] = mapped_column(String(16)) country: Mapped[str | None] = mapped_column(String(8)) @@ -183,6 +189,14 @@ class Channel(Base, TimestampMixin): from_search: Mapped[bool] = mapped_column( Boolean, default=False, server_default="false", index=True ) + # True when this channel exists only because a user EXPLORED it (opened its page without + # subscribing) — an ephemeral, per-explorer addition. The cleanup job hard-deletes such a + # channel (and its via_explore videos) once nobody keeps it (no subscription, no live + # ExploredChannel row, no per-user state) after a grace period. Cleared when someone + # subscribes (the "keep" signal promotes it to organic). + from_explore: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false", index=True + ) videos: Mapped[list["Video"]] = relationship(back_populates="channel") subscriptions: Mapped[list["Subscription"]] = relationship(back_populates="channel") @@ -260,6 +274,14 @@ class Video(Base, TimestampMixin): via_search: Mapped[bool] = mapped_column( Boolean, default=False, server_default="false", index=True ) + # True when this video entered the catalog because a user EXPLORED its (un-subscribed) + # channel. Like via_search it's hidden from the shared Library by default, but it's + # additionally PER-USER private: a via_explore video is only visible to users who + # explored or subscribe to its channel (a gate in the feed query), so one user's + # curiosity never floods everyone else's catalog. Left False by the organic ingest paths. + via_explore: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false", index=True + ) # none | live | upcoming | premiere | was_live live_status: Mapped[str] = mapped_column( String(16), default="none", server_default="none", index=True @@ -388,6 +410,31 @@ class SearchFind(Base, TimestampMixin): ) +class ExploredChannel(Base, TimestampMixin): + """Per-user record of a channel the user opened the page of WITHOUT subscribing. + + Exploring an un-subscribed channel ingests its recent uploads (flagged via_explore) so the + user can browse them, but keeps them out of everyone else's catalog. This row (a) gates the + explorer's access to those ephemeral videos in the feed query and (b) drives the grace-period + purge: a from_explore channel with no live ExploredChannel row (and no subscription / no + per-user state on its videos) is hard-deleted by the cleanup job. Subscribing is the "keep" + signal — it promotes the channel to organic and these rows become irrelevant. `created_at` + (from TimestampMixin) is the grace clock; a re-visit refreshes it.""" + + __tablename__ = "explored_channels" + __table_args__ = ( + UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + channel_id: Mapped[str] = mapped_column( + ForeignKey("channels.id", ondelete="CASCADE"), index=True + ) + + class ApiQuotaUsage(Base): """Tracks YouTube Data API units spent per Pacific-time day (the quota resets at midnight Pacific). The whole app shares one daily budget.""" diff --git a/backend/app/quota.py b/backend/app/quota.py index 1e72e56..0e44839 100644 --- a/backend/app/quota.py +++ b/backend/app/quota.py @@ -38,6 +38,7 @@ class QuotaAction: VIDEOS_LOOKUP = "videos_lookup" VIDEOS_SEARCH = "videos_search" CHANNELS_DISCOVER = "channels_discover" + CHANNELS_EXPLORE = "channels_explore" CHANNELS_SUBSCRIBE = "channels_subscribe" CHANNELS_UNSUBSCRIBE = "channels_unsubscribe" MAINTENANCE_REVALIDATE = "maintenance_revalidate" diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index f1d00eb..6ffdd7f 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -5,15 +5,16 @@ import logging from datetime import datetime, timezone from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy import and_, func, select +from sqlalchemy import and_, func, select, update from sqlalchemy.orm import Session -from app import quota -from app.auth import current_user, has_write_scope +from app import quota, sysconfig +from app.auth import current_user, has_write_scope, require_human from app.db import get_db from app.models import ( Channel, ChannelTag, + ExploredChannel, Playlist, PlaylistItem, Subscription, @@ -22,6 +23,7 @@ from app.models import ( Video, ) from app.routes.admin import admin_user +from app.sync.explore import explore_ingest_page from app.sync.runner import run_recent_backfill from app.sync.subscriptions import apply_channel_details from app.youtube.client import YouTubeClient, YouTubeError @@ -205,6 +207,133 @@ def discover_channels( ] +def _channel_detail_dict( + channel: Channel, *, subscribed: bool, explored: bool, stored: int +) -> dict: + return { + "id": channel.id, + "title": channel.title, + "handle": channel.handle, + "description": channel.description, + "thumbnail_url": channel.thumbnail_url, + "banner_url": channel.banner_url, + "subscriber_count": channel.subscriber_count, + "video_count": channel.video_count, + "total_view_count": channel.total_view_count, + "published_at": channel.published_at.isoformat() if channel.published_at else None, + "external_links": channel.external_links or [], + "country": channel.country, + "subscribed": subscribed, + "explored": explored, + "stored_videos": stored, + "from_explore": channel.from_explore, + "details_synced": channel.details_synced_at is not None, + } + + +@router.get("/{channel_id}") +def channel_detail( + channel_id: str, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + """One channel's "About" detail for the channel page header/About tab, plus this user's + relationship to it (subscribed / explored) and how many of its videos are in the catalog. + Lazily fills the About metadata (view count, join date, banner, links) the first time it's + needed — a stub, or a channel last enriched before those fields existed (published_at is a + reliable sentinel: every real channel has a creation date). The enrich uses the API key + (1 unit, cached after), so it's skipped for the quota-less demo account.""" + channel = db.get(Channel, channel_id) + if channel is None: + raise HTTPException(status_code=404, detail="Unknown channel") + if channel.published_at is None and not user.is_demo: + try: + with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt: + apply_channel_details(db, yt.get_channels([channel_id])) + db.commit() + channel = db.get(Channel, channel_id) + except YouTubeError as exc: + log.warning("channel detail enrich failed (%s): %s", channel_id, exc) + db.rollback() + subscribed = db.execute( + select(Subscription.id).where( + Subscription.user_id == user.id, Subscription.channel_id == channel_id + ) + ).first() is not None + explored = db.execute( + select(ExploredChannel.id).where( + ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id + ) + ).first() is not None + stored = db.scalar(select(func.count(Video.id)).where(Video.channel_id == channel_id)) or 0 + return _channel_detail_dict( + channel, subscribed=subscribed, explored=explored, stored=int(stored) + ) + + +@router.post("/{channel_id}/explore") +def explore_channel( + channel_id: str, + page_token: str | None = None, + user: User = Depends(require_human), + db: Session = Depends(get_db), +) -> dict: + """Open an un-subscribed channel for browsing. Records the exploration (a per-user, + grace-clocked row that gates this user's access to the channel's ephemeral videos and + drives the later cleanup), flags the channel ephemeral unless someone organically follows + it, then ingests one page of its uploads (newest-first) marked via_explore + enriched. + Returns the next page token so the page can "load more". Guarded against the shared + backfill quota reserve; blocked for the demo account (it spends shared quota).""" + channel = db.get(Channel, channel_id) + if channel is None: + raise HTTPException(status_code=404, detail="Unknown channel") + if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): + raise HTTPException( + status_code=429, + detail="The shared daily quota reserve is reached — try exploring later.", + ) + + # Record/refresh the exploration (DB only) before ingesting, so the explorer can see the + # videos as soon as they land. Flag the channel ephemeral unless it's organically followed. + existing = db.execute( + select(ExploredChannel).where( + ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id + ) + ).scalar_one_or_none() + if existing is None: + db.add(ExploredChannel(user_id=user.id, channel_id=channel_id)) + else: + existing.created_at = datetime.now(timezone.utc) # refresh the grace clock + followed = db.execute( + select(Subscription.id).where(Subscription.channel_id == channel_id).limit(1) + ).first() + if followed is None and not channel.from_explore: + channel.from_explore = True + db.commit() + + try: + with quota.attribute(user.id, quota.QuotaAction.CHANNELS_EXPLORE), YouTubeClient(db, user) as yt: + # Enrich for the About panel + the uploads playlist id (needed to ingest videos). + if ( + channel.details_synced_at is None + or channel.uploads_playlist_id is None + or channel.published_at is None + ): + apply_channel_details(db, yt.get_channels([channel_id])) + db.commit() + channel = db.get(Channel, channel_id) + result = explore_ingest_page(db, yt, channel, page_token) + except YouTubeError as exc: + raise HTTPException( + status_code=422, detail=f"YouTube couldn't load this channel: {exc}" + ) + return { + "channel_id": channel_id, + "ingested": result["ingested"], + "next_page_token": result["next_page_token"], + } + + def _user_subscription(db: Session, user: User, channel_id: str) -> Subscription: sub = db.execute( select(Subscription).where( @@ -332,6 +461,14 @@ def subscribe( subscribed_at=datetime.now(timezone.utc), ) ) + # Subscribing is the "keep" signal: a previously-explored channel becomes a normal organic + # one. Drop its ephemeral flags so the cleanup job leaves it alone and its videos stop being + # per-user private (they join the shared Library like any subscribed channel's uploads). + if channel.from_explore: + channel.from_explore = False + db.execute( + update(Video).where(Video.channel_id == channel_id, Video.via_explore.is_(True)).values(via_explore=False) + ) db.commit() return {"subscribed": channel_id} diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 528a42e..5969edb 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -16,6 +16,7 @@ from app.db import get_db from app.models import ( Channel, ChannelTag, + ExploredChannel, Playlist, PlaylistItem, SearchFind, @@ -180,6 +181,23 @@ def _filtered_query( # either way they shouldn't show in any feed view meanwhile. query = query.where(Video.unavailable_since.is_(None)) + # Explored videos are PER-EXPLORER private: a via_explore video (ingested when someone + # opened an un-subscribed channel's page) is visible only to users who explored or + # subscribe to its channel, so one user's curiosity never leaks into everyone else's + # catalog. Applied in BOTH scopes; the channel page reaches its ephemeral videos by + # asking for scope=all + channel_id (the channel is in this accessible set). + accessible_explore = ( + select(Subscription.channel_id).where(Subscription.user_id == user.id) + ).union( + select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id) + ) + query = query.where( + or_( + Video.via_explore.is_(False), + Video.channel_id.in_(accessible_explore), + ) + ) + # Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag # (search-discovered by anyone). In "my" scope it's the user's OWN provenance: # "organic" = your subscriptions, "search" = videos you found via your own search, diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index fd8de99..b39bd95 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -16,6 +16,7 @@ from app.models import SchedulerSetting from app.notifications import create_notification from app.state import is_sync_paused from app.sync.autotag import run_autotag_all +from app.sync.explore import purge_unkept_explored from app.sync.maintenance import run_maintenance from app.sync.playlists import sync_all_playlists from app.sync.runner import ( @@ -57,6 +58,7 @@ JOB_INTERVALS: dict[str, int] = { "playlist_sync": settings.playlist_sync_minutes, "maintenance": settings.maintenance_interval_minutes, "demo_reset": settings.demo_reset_minutes, + "explore_cleanup": settings.explore_cleanup_minutes, } # Sane bounds for an admin-set interval (minutes). @@ -268,6 +270,12 @@ def _demo_reset_job() -> None: _job("demo_reset", work) +def _explore_cleanup_job() -> None: + # Hard-delete explored-but-never-kept channels and their untouched ephemeral videos after + # the grace period. 0 quota (pure DB), so no quota attribution. + _job("explore_cleanup", purge_unkept_explored) + + # job_id -> wrapper. The single source of truth for which jobs exist and how to run one, # shared by start_scheduler (recurring registration) and trigger_job (manual "run now"). JOB_FUNCS: dict[str, Callable[[], None]] = { @@ -280,6 +288,7 @@ JOB_FUNCS: dict[str, Callable[[], None]] = { "playlist_sync": _playlist_sync_job, "maintenance": _maintenance_job, "demo_reset": _demo_reset_job, + "explore_cleanup": _explore_cleanup_job, } diff --git a/backend/app/sync/explore.py b/backend/app/sync/explore.py new file mode 100644 index 0000000..7257862 --- /dev/null +++ b/backend/app/sync/explore.py @@ -0,0 +1,145 @@ +"""Channel exploration: ingest an un-subscribed channel's uploads on demand (flagged +via_explore, kept per-explorer) and purge channels nobody ends up keeping. + +A user opens a channel's page without subscribing → we pull a page of its recent uploads, +mark the new rows via_explore, and enrich them so they're immediately browsable. The videos +stay private to users who explored (or subscribe to) the channel — the feed query gates them +— so one user's curiosity never floods everyone's catalog. Subscribing is the "keep" signal; +otherwise the cleanup job hard-deletes the channel and its untouched ephemeral videos after a +grace period (mirrors the maintenance job's grace-then-delete safety).""" +import logging +from datetime import timedelta + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session + +from app import sysconfig +from app.models import ( + Channel, + ExploredChannel, + PlaylistItem, + SearchFind, + Subscription, + Video, + VideoState, +) +from app.sync.videos import _insert_stubs, _stub_from_playlist_item, apply_video_details +from app.utils import now_utc as _now +from app.youtube.client import YouTubeClient + +log = logging.getLogger("siftlode.explore") + + +def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int: + """Enrich just the not-yet-enriched videos among `video_ids` (videos.list, 1 unit/50).""" + if not video_ids: + return 0 + videos = ( + db.execute( + select(Video).where( + Video.id.in_(video_ids), Video.enriched_at.is_(None) + ) + ) + .scalars() + .all() + ) + if not videos: + return 0 + items = {it["id"]: it for it in yt.get_videos([v.id for v in videos])} + now = _now() + for video in videos: + item = items.get(video.id) + if item is None: + video.enriched_at = now # unavailable; stop retrying + continue + apply_video_details(video, item) + video.enriched_at = now + db.commit() + return len(videos) + + +def explore_ingest_page( + db: Session, yt: YouTubeClient, channel: Channel, page_token: str | None = None +) -> dict: + """Ingest one page (up to 50, newest-first) of the channel's uploads, marking the new + videos via_explore, then enrich them. Returns the count ingested and the next page token + (None when the back-catalogue is exhausted) so the caller can offer "load more". Stateless + pagination: the caller round-trips the token, so the channel's own backfill cursor (used by + the organic deep backfill) is never touched.""" + if not channel.uploads_playlist_id: + return {"ingested": 0, "next_page_token": None} + data = yt.get_playlist_items(channel.uploads_playlist_id, page_token) + stubs: list[dict] = [] + for item in data.get("items", []): + stub = _stub_from_playlist_item(item, channel.id) + if stub is not None: + stub["via_explore"] = True + stubs.append(stub) + inserted = _insert_stubs(db, stubs) + self_enriched = _enrich_new(db, yt, [s["id"] for s in stubs]) + log.info( + "explore ingest channel=%s ingested=%s enriched=%s", channel.id, inserted, self_enriched + ) + return {"ingested": inserted, "next_page_token": data.get("nextPageToken")} + + +def purge_unkept_explored(db: Session) -> dict: + """Cleanup job: hard-delete explored channels nobody kept and their untouched ephemeral + videos, after a grace period (mirrors the maintenance grace-then-delete safety). + + Keep signals that spare a channel/video: + - any user SUBSCRIBES to the channel (subscribe promotes it to organic — channel-level), or + - a live ExploredChannel row (someone explored it within the grace window — channel-level), or + - a video has per-user state: watched/in-progress (VideoState), in a playlist (PlaylistItem), + or surfaced by the user's own search (SearchFind) — these survive even if the channel goes. + A channel row is dropped only once it holds no videos at all. 0 quota.""" + grace_days = sysconfig.get_int(db, "explore_grace_days") + cutoff = _now() - timedelta(days=grace_days) + + expired = db.execute( + delete(ExploredChannel).where(ExploredChannel.created_at < cutoff) + ).rowcount + db.commit() + + sub_exists = ( + select(Subscription.id).where(Subscription.channel_id == Channel.id).exists() + ) + exp_exists = ( + select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists() + ) + orphan_ids = ( + db.execute( + select(Channel.id).where( + Channel.from_explore.is_(True), ~sub_exists, ~exp_exists + ) + ) + .scalars() + .all() + ) + if not orphan_ids: + return {"expired": expired, "videos_deleted": 0, "channels_deleted": 0} + + state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists() + pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists() + find_exists = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists() + videos_deleted = db.execute( + delete(Video).where( + Video.channel_id.in_(orphan_ids), + Video.via_explore.is_(True), + ~state_exists, + ~pl_exists, + ~find_exists, + ) + ).rowcount + db.commit() + + has_videos = select(Video.id).where(Video.channel_id == Channel.id).exists() + channels_deleted = db.execute( + delete(Channel).where(Channel.id.in_(orphan_ids), ~has_videos) + ).rowcount + db.commit() + return { + "expired": expired, + "videos_deleted": videos_deleted, + "channels_deleted": channels_deleted, + } diff --git a/backend/app/sync/subscriptions.py b/backend/app/sync/subscriptions.py index adafce3..c0f643a 100644 --- a/backend/app/sync/subscriptions.py +++ b/backend/app/sync/subscriptions.py @@ -18,6 +18,29 @@ def _to_int(value) -> int | None: return None +def _parse_dt(value: str | None) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def _external_links(branding: dict) -> list | None: + """Channel "About" links. The API exposes a few in brandingSettings.channel.profileColor/ + /... varies; the durable spot is brandingSettings.channel.links is gone post-2023, so we + read what's present (settings/links) defensively and keep [{title, url}] pairs.""" + channel = branding.get("channel", {}) + links = channel.get("links") or [] + out = [ + {"title": l.get("title"), "url": l.get("url")} + for l in links + if isinstance(l, dict) and l.get("url") + ] + return out or None + + def apply_channel_details(db: Session, items: list[dict]) -> None: now = datetime.now(timezone.utc) for item in items: @@ -28,6 +51,7 @@ def apply_channel_details(db: Session, items: list[dict]) -> None: content = item.get("contentDetails", {}) stats = item.get("statistics", {}) topics = item.get("topicDetails", {}) + branding = item.get("brandingSettings", {}) channel.title = snippet.get("title") or channel.title channel.handle = snippet.get("customUrl") @@ -40,6 +64,10 @@ def apply_channel_details(db: Session, items: list[dict]) -> None: channel.uploads_playlist_id = content.get("relatedPlaylists", {}).get("uploads") channel.subscriber_count = _to_int(stats.get("subscriberCount")) channel.video_count = _to_int(stats.get("videoCount")) + channel.total_view_count = _to_int(stats.get("viewCount")) + channel.published_at = _parse_dt(snippet.get("publishedAt")) + channel.banner_url = branding.get("image", {}).get("bannerExternalUrl") + channel.external_links = _external_links(branding) channel.topic_categories = topics.get("topicCategories") channel.details_synced_at = now diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index c6d8004..133b0bb 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -51,6 +51,9 @@ SPECS: tuple[ConfigSpec, ...] = ( # --- Batch sizes --- ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50), ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000), + # --- Channel explore --- + # Days an explored-but-unsubscribed channel is kept before the cleanup job purges it. + ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650), # --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) --- ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True), # Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key). diff --git a/backend/app/youtube/client.py b/backend/app/youtube/client.py index b4e5ef6..0fc0ebf 100644 --- a/backend/app/youtube/client.py +++ b/backend/app/youtube/client.py @@ -187,7 +187,7 @@ class YouTubeClient: data = self._get( "channels", { - "part": "snippet,contentDetails,statistics,topicDetails", + "part": "snippet,contentDetails,statistics,topicDetails,brandingSettings", "id": ",".join(batch), "maxResults": 50, },