peter/siftlode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(channels): channel-explore backend — about metadata, ephemeral provenance, cleanup

Browse source 
Adds the backend for a dedicated channel page + ephemeral browsing of un-subscribed
channels:
- migration 0033: channels.{total_view_count,published_at,banner_url,external_links,
  from_explore}, videos.via_explore, explored_channels (per-user, grace-clocked).
- enrich channels with part=brandingSettings (banner/links) + statistics.viewCount +
  snippet.publishedAt — no extra quota.
- GET /api/channels/{id}: About detail + this user's relationship; lazy-enriches the new
  About fields (published_at sentinel).
- POST /api/channels/{id}/explore (require_human, quota-reserve guarded): records the
  exploration, flags the channel ephemeral unless followed, ingests one page of uploads
  (via_explore) + enriches; returns next_page_token for load-more.
- feed: per-explorer gate — a via_explore video is visible only to users who explored or
  subscribe to its channel, so exploration never leaks into everyone's catalog.
- subscribe = keep: clears from_explore + via_explore on the channel's videos.
- scheduler explore_cleanup job + explore_grace_days config: hard-delete explored-but-
  unkept channels and their untouched ephemeral videos after a grace period.
This commit is contained in:
npeter83 2026-06-30 02:52:44 +02:00
parent bb7b9972fd
commit bc4c362423
11 changed files with 481 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

6
backend/app/config.py
View file

@ -118,6 +118,12 @@ class Settings(BaseSettings):
# items). Deleted/private/paywalled get a recovery window; abandoned upcoming and
# confirmed-dead stuck-live have no age grace (deleted on the sweep that confirms them).
maintenance_grace_days: int = 7
# --- Channel-explore cleanup job ---
# Runs daily by default (admin-tunable via the Scheduler dashboard). An explored-but-never-
# kept channel (no subscription, no per-user state on its videos) is hard-deleted this many
# days after it was last explored — the grace clock is ExploredChannel.created_at.
explore_cleanup_minutes: int = 1440
explore_grace_days: int = 7
# live_status values hidden from the feed by default. Completed-stream VODs
# ("was_live") are real watchable content and stay visible.
feed_default_hidden_live: str = "live,upcoming"

47
backend/app/models.py
View file

@ -165,6 +165,12 @@ class Channel(Base, TimestampMixin):
uploads_playlist_id: Mapped[str | None] = mapped_column(String(64))
subscriber_count: Mapped[int | None] = mapped_column(BigInteger)
video_count: Mapped[int | None] = mapped_column(BigInteger)
# Channel "About" metadata (fetched with the same channels.list call, no extra quota):
# lifetime view count, the channel's join date, banner image, and external links.
total_view_count: Mapped[int | None] = mapped_column(BigInteger)
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
banner_url: Mapped[str | None] = mapped_column(String(1024))
external_links: Mapped[list | None] = mapped_column(JSON)
topic_categories: Mapped[list | None] = mapped_column(JSON)
default_language: Mapped[str | None] = mapped_column(String(16))
country: Mapped[str | None] = mapped_column(String(8))
@ -183,6 +189,14 @@ class Channel(Base, TimestampMixin):
from_search: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false", index=True
)
# True when this channel exists only because a user EXPLORED it (opened its page without
# subscribing) — an ephemeral, per-explorer addition. The cleanup job hard-deletes such a
# channel (and its via_explore videos) once nobody keeps it (no subscription, no live
# ExploredChannel row, no per-user state) after a grace period. Cleared when someone
# subscribes (the "keep" signal promotes it to organic).
from_explore: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false", index=True
)
videos: Mapped[list["Video"]] = relationship(back_populates="channel")
subscriptions: Mapped[list["Subscription"]] = relationship(back_populates="channel")
@ -260,6 +274,14 @@ class Video(Base, TimestampMixin):
via_search: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false", index=True
)
# True when this video entered the catalog because a user EXPLORED its (un-subscribed)
# channel. Like via_search it's hidden from the shared Library by default, but it's
# additionally PER-USER private: a via_explore video is only visible to users who
# explored or subscribe to its channel (a gate in the feed query), so one user's
# curiosity never floods everyone else's catalog. Left False by the organic ingest paths.
via_explore: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false", index=True
)
# none | live | upcoming | premiere | was_live
live_status: Mapped[str] = mapped_column(
String(16), default="none", server_default="none", index=True
@ -388,6 +410,31 @@ class SearchFind(Base, TimestampMixin):
)
class ExploredChannel(Base, TimestampMixin):
"""Per-user record of a channel the user opened the page of WITHOUT subscribing.
Exploring an un-subscribed channel ingests its recent uploads (flagged via_explore) so the
user can browse them, but keeps them out of everyone else's catalog. This row (a) gates the
explorer's access to those ephemeral videos in the feed query and (b) drives the grace-period
purge: a from_explore channel with no live ExploredChannel row (and no subscription / no
per-user state on its videos) is hard-deleted by the cleanup job. Subscribing is the "keep"
signal — it promotes the channel to organic and these rows become irrelevant. `created_at`
(from TimestampMixin) is the grace clock; a re-visit refreshes it."""
__tablename__ = "explored_channels"
__table_args__ = (
UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
channel_id: Mapped[str] = mapped_column(
ForeignKey("channels.id", ondelete="CASCADE"), index=True
)
class ApiQuotaUsage(Base):
"""Tracks YouTube Data API units spent per Pacific-time day (the quota resets at
midnight Pacific). The whole app shares one daily budget."""

1
backend/app/quota.py
View file

@ -38,6 +38,7 @@ class QuotaAction:
VIDEOS_LOOKUP = "videos_lookup"
VIDEOS_SEARCH = "videos_search"
CHANNELS_DISCOVER = "channels_discover"
CHANNELS_EXPLORE = "channels_explore"
CHANNELS_SUBSCRIBE = "channels_subscribe"
CHANNELS_UNSUBSCRIBE = "channels_unsubscribe"
MAINTENANCE_REVALIDATE = "maintenance_revalidate"

143
backend/app/routes/channels.py
View file

@ -5,15 +5,16 @@ import logging
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import and_, func, select
from sqlalchemy import and_, func, select, update
from sqlalchemy.orm import Session
from app import quota
from app.auth import current_user, has_write_scope
from app import quota, sysconfig
from app.auth import current_user, has_write_scope, require_human
from app.db import get_db
from app.models import (
Channel,
ChannelTag,
ExploredChannel,
Playlist,
PlaylistItem,
Subscription,
@ -22,6 +23,7 @@ from app.models import (
Video,
)
from app.routes.admin import admin_user
from app.sync.explore import explore_ingest_page
from app.sync.runner import run_recent_backfill
from app.sync.subscriptions import apply_channel_details
from app.youtube.client import YouTubeClient, YouTubeError
@ -205,6 +207,133 @@ def discover_channels(
]
def _channel_detail_dict(
channel: Channel, *, subscribed: bool, explored: bool, stored: int
) -> dict:
return {
"id": channel.id,
"title": channel.title,
"handle": channel.handle,
"description": channel.description,
"thumbnail_url": channel.thumbnail_url,
"banner_url": channel.banner_url,
"subscriber_count": channel.subscriber_count,
"video_count": channel.video_count,
"total_view_count": channel.total_view_count,
"published_at": channel.published_at.isoformat() if channel.published_at else None,
"external_links": channel.external_links or [],
"country": channel.country,
"subscribed": subscribed,
"explored": explored,
"stored_videos": stored,
"from_explore": channel.from_explore,
"details_synced": channel.details_synced_at is not None,
}
@router.get("/{channel_id}")
def channel_detail(
channel_id: str,
user: User = Depends(current_user),
db: Session = Depends(get_db),
) -> dict:
"""One channel's "About" detail for the channel page header/About tab, plus this user's
relationship to it (subscribed / explored) and how many of its videos are in the catalog.
Lazily fills the About metadata (view count, join date, banner, links) the first time it's
needed — a stub, or a channel last enriched before those fields existed (published_at is a
reliable sentinel: every real channel has a creation date). The enrich uses the API key
(1 unit, cached after), so it's skipped for the quota-less demo account."""
channel = db.get(Channel, channel_id)
if channel is None:
raise HTTPException(status_code=404, detail="Unknown channel")
if channel.published_at is None and not user.is_demo:
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt:
apply_channel_details(db, yt.get_channels([channel_id]))
db.commit()
channel = db.get(Channel, channel_id)
except YouTubeError as exc:
log.warning("channel detail enrich failed (%s): %s", channel_id, exc)
db.rollback()
subscribed = db.execute(
select(Subscription.id).where(
Subscription.user_id == user.id, Subscription.channel_id == channel_id
)
).first() is not None
explored = db.execute(
select(ExploredChannel.id).where(
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id
)
).first() is not None
stored = db.scalar(select(func.count(Video.id)).where(Video.channel_id == channel_id)) or 0
return _channel_detail_dict(
channel, subscribed=subscribed, explored=explored, stored=int(stored)
)
@router.post("/{channel_id}/explore")
def explore_channel(
channel_id: str,
page_token: str | None = None,
user: User = Depends(require_human),
db: Session = Depends(get_db),
) -> dict:
"""Open an un-subscribed channel for browsing. Records the exploration (a per-user,
grace-clocked row that gates this user's access to the channel's ephemeral videos and
drives the later cleanup), flags the channel ephemeral unless someone organically follows
it, then ingests one page of its uploads (newest-first) marked via_explore + enriched.
Returns the next page token so the page can "load more". Guarded against the shared
backfill quota reserve; blocked for the demo account (it spends shared quota)."""
channel = db.get(Channel, channel_id)
if channel is None:
raise HTTPException(status_code=404, detail="Unknown channel")
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
raise HTTPException(
status_code=429,
detail="The shared daily quota reserve is reached — try exploring later.",
)
# Record/refresh the exploration (DB only) before ingesting, so the explorer can see the
# videos as soon as they land. Flag the channel ephemeral unless it's organically followed.
existing = db.execute(
select(ExploredChannel).where(
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == channel_id
)
).scalar_one_or_none()
if existing is None:
db.add(ExploredChannel(user_id=user.id, channel_id=channel_id))
else:
existing.created_at = datetime.now(timezone.utc) # refresh the grace clock
followed = db.execute(
select(Subscription.id).where(Subscription.channel_id == channel_id).limit(1)
).first()
if followed is None and not channel.from_explore:
channel.from_explore = True
db.commit()
try:
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_EXPLORE), YouTubeClient(db, user) as yt:
# Enrich for the About panel + the uploads playlist id (needed to ingest videos).
if (
channel.details_synced_at is None
or channel.uploads_playlist_id is None
or channel.published_at is None
):
apply_channel_details(db, yt.get_channels([channel_id]))
db.commit()
channel = db.get(Channel, channel_id)
result = explore_ingest_page(db, yt, channel, page_token)
except YouTubeError as exc:
raise HTTPException(
status_code=422, detail=f"YouTube couldn't load this channel: {exc}"
)
return {
"channel_id": channel_id,
"ingested": result["ingested"],
"next_page_token": result["next_page_token"],
}
def _user_subscription(db: Session, user: User, channel_id: str) -> Subscription:
sub = db.execute(
select(Subscription).where(
@ -332,6 +461,14 @@ def subscribe(
subscribed_at=datetime.now(timezone.utc),
)
)
# Subscribing is the "keep" signal: a previously-explored channel becomes a normal organic
# one. Drop its ephemeral flags so the cleanup job leaves it alone and its videos stop being
# per-user private (they join the shared Library like any subscribed channel's uploads).
if channel.from_explore:
channel.from_explore = False
db.execute(
update(Video).where(Video.channel_id == channel_id, Video.via_explore.is_(True)).values(via_explore=False)
)
db.commit()
return {"subscribed": channel_id}

18
backend/app/routes/feed.py
View file

@ -16,6 +16,7 @@ from app.db import get_db
from app.models import (
Channel,
ChannelTag,
ExploredChannel,
Playlist,
PlaylistItem,
SearchFind,
@ -180,6 +181,23 @@ def _filtered_query(
# either way they shouldn't show in any feed view meanwhile.
query = query.where(Video.unavailable_since.is_(None))
# Explored videos are PER-EXPLORER private: a via_explore video (ingested when someone
# opened an un-subscribed channel's page) is visible only to users who explored or
# subscribe to its channel, so one user's curiosity never leaks into everyone else's
# catalog. Applied in BOTH scopes; the channel page reaches its ephemeral videos by
# asking for scope=all + channel_id (the channel is in this accessible set).
accessible_explore = (
select(Subscription.channel_id).where(Subscription.user_id == user.id)
).union(
select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id)
)
query = query.where(
or_(
Video.via_explore.is_(False),
Video.channel_id.in_(accessible_explore),
)
)
# Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag
# (search-discovered by anyone). In "my" scope it's the user's OWN provenance:
# "organic" = your subscriptions, "search" = videos you found via your own search,

9
backend/app/scheduler.py
View file

@ -16,6 +16,7 @@ from app.models import SchedulerSetting
from app.notifications import create_notification
from app.state import is_sync_paused
from app.sync.autotag import run_autotag_all
from app.sync.explore import purge_unkept_explored
from app.sync.maintenance import run_maintenance
from app.sync.playlists import sync_all_playlists
from app.sync.runner import (
@ -57,6 +58,7 @@ JOB_INTERVALS: dict[str, int] = {
"playlist_sync": settings.playlist_sync_minutes,
"maintenance": settings.maintenance_interval_minutes,
"demo_reset": settings.demo_reset_minutes,
"explore_cleanup": settings.explore_cleanup_minutes,
}
# Sane bounds for an admin-set interval (minutes).
@ -268,6 +270,12 @@ def _demo_reset_job() -> None:
_job("demo_reset", work)
def _explore_cleanup_job() -> None:
# Hard-delete explored-but-never-kept channels and their untouched ephemeral videos after
# the grace period. 0 quota (pure DB), so no quota attribution.
_job("explore_cleanup", purge_unkept_explored)
# job_id -> wrapper. The single source of truth for which jobs exist and how to run one,
# shared by start_scheduler (recurring registration) and trigger_job (manual "run now").
JOB_FUNCS: dict[str, Callable[[], None]] = {
@ -280,6 +288,7 @@ JOB_FUNCS: dict[str, Callable[[], None]] = {
"playlist_sync": _playlist_sync_job,
"maintenance": _maintenance_job,
"demo_reset": _demo_reset_job,
"explore_cleanup": _explore_cleanup_job,
}

145
backend/app/sync/explore.py Normal file
View file

@ -0,0 +1,145 @@
"""Channel exploration: ingest an un-subscribed channel's uploads on demand (flagged
via_explore, kept per-explorer) and purge channels nobody ends up keeping.
A user opens a channel's page without subscribing → we pull a page of its recent uploads,
mark the new rows via_explore, and enrich them so they're immediately browsable. The videos
stay private to users who explored (or subscribe to) the channel — the feed query gates them
— so one user's curiosity never floods everyone's catalog. Subscribing is the "keep" signal;
otherwise the cleanup job hard-deletes the channel and its untouched ephemeral videos after a
grace period (mirrors the maintenance job's grace-then-delete safety)."""
import logging
from datetime import timedelta
from sqlalchemy import delete, select
from sqlalchemy.orm import Session
from app import sysconfig
from app.models import (
Channel,
ExploredChannel,
PlaylistItem,
SearchFind,
Subscription,
Video,
VideoState,
)
from app.sync.videos import _insert_stubs, _stub_from_playlist_item, apply_video_details
from app.utils import now_utc as _now
from app.youtube.client import YouTubeClient
log = logging.getLogger("siftlode.explore")
def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int:
"""Enrich just the not-yet-enriched videos among `video_ids` (videos.list, 1 unit/50)."""
if not video_ids:
return 0
videos = (
db.execute(
select(Video).where(
Video.id.in_(video_ids), Video.enriched_at.is_(None)
)
)
.scalars()
.all()
)
if not videos:
return 0
items = {it["id"]: it for it in yt.get_videos([v.id for v in videos])}
now = _now()
for video in videos:
item = items.get(video.id)
if item is None:
video.enriched_at = now # unavailable; stop retrying
continue
apply_video_details(video, item)
video.enriched_at = now
db.commit()
return len(videos)
def explore_ingest_page(
db: Session, yt: YouTubeClient, channel: Channel, page_token: str | None = None
) -> dict:
"""Ingest one page (up to 50, newest-first) of the channel's uploads, marking the new
videos via_explore, then enrich them. Returns the count ingested and the next page token
(None when the back-catalogue is exhausted) so the caller can offer "load more". Stateless
pagination: the caller round-trips the token, so the channel's own backfill cursor (used by
the organic deep backfill) is never touched."""
if not channel.uploads_playlist_id:
return {"ingested": 0, "next_page_token": None}
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
stubs: list[dict] = []
for item in data.get("items", []):
stub = _stub_from_playlist_item(item, channel.id)
if stub is not None:
stub["via_explore"] = True
stubs.append(stub)
inserted = _insert_stubs(db, stubs)
self_enriched = _enrich_new(db, yt, [s["id"] for s in stubs])
log.info(
"explore ingest channel=%s ingested=%s enriched=%s", channel.id, inserted, self_enriched
)
return {"ingested": inserted, "next_page_token": data.get("nextPageToken")}
def purge_unkept_explored(db: Session) -> dict:
"""Cleanup job: hard-delete explored channels nobody kept and their untouched ephemeral
videos, after a grace period (mirrors the maintenance grace-then-delete safety).
Keep signals that spare a channel/video:
- any user SUBSCRIBES to the channel (subscribe promotes it to organic — channel-level), or
- a live ExploredChannel row (someone explored it within the grace window — channel-level), or
- a video has per-user state: watched/in-progress (VideoState), in a playlist (PlaylistItem),
or surfaced by the user's own search (SearchFind) — these survive even if the channel goes.
A channel row is dropped only once it holds no videos at all. 0 quota."""
grace_days = sysconfig.get_int(db, "explore_grace_days")
cutoff = _now() - timedelta(days=grace_days)
expired = db.execute(
delete(ExploredChannel).where(ExploredChannel.created_at < cutoff)
).rowcount
db.commit()
sub_exists = (
select(Subscription.id).where(Subscription.channel_id == Channel.id).exists()
)
exp_exists = (
select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists()
)
orphan_ids = (
db.execute(
select(Channel.id).where(
Channel.from_explore.is_(True), ~sub_exists, ~exp_exists
)
)
.scalars()
.all()
)
if not orphan_ids:
return {"expired": expired, "videos_deleted": 0, "channels_deleted": 0}
state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
find_exists = select(SearchFind.id).where(SearchFind.video_id == Video.id).exists()
videos_deleted = db.execute(
delete(Video).where(
Video.channel_id.in_(orphan_ids),
Video.via_explore.is_(True),
~state_exists,
~pl_exists,
~find_exists,
)
).rowcount
db.commit()
has_videos = select(Video.id).where(Video.channel_id == Channel.id).exists()
channels_deleted = db.execute(
delete(Channel).where(Channel.id.in_(orphan_ids), ~has_videos)
).rowcount
db.commit()
return {
"expired": expired,
"videos_deleted": videos_deleted,
"channels_deleted": channels_deleted,
}

28
backend/app/sync/subscriptions.py
View file

@ -18,6 +18,29 @@ def _to_int(value) -> int | None:
return None
def _parse_dt(value: str | None) -> datetime | None:
if not value:
return None
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
def _external_links(branding: dict) -> list | None:
"""Channel "About" links. The API exposes a few in brandingSettings.channel.profileColor/
/... varies; the durable spot is brandingSettings.channel.links is gone post-2023, so we
read what's present (settings/links) defensively and keep [{title, url}] pairs."""
channel = branding.get("channel", {})
links = channel.get("links") or []
out = [
{"title": l.get("title"), "url": l.get("url")}
for l in links
if isinstance(l, dict) and l.get("url")
]
return out or None
def apply_channel_details(db: Session, items: list[dict]) -> None:
now = datetime.now(timezone.utc)
for item in items:
@ -28,6 +51,7 @@ def apply_channel_details(db: Session, items: list[dict]) -> None:
content = item.get("contentDetails", {})
stats = item.get("statistics", {})
topics = item.get("topicDetails", {})
branding = item.get("brandingSettings", {})
channel.title = snippet.get("title") or channel.title
channel.handle = snippet.get("customUrl")
@ -40,6 +64,10 @@ def apply_channel_details(db: Session, items: list[dict]) -> None:
channel.uploads_playlist_id = content.get("relatedPlaylists", {}).get("uploads")
channel.subscriber_count = _to_int(stats.get("subscriberCount"))
channel.video_count = _to_int(stats.get("videoCount"))
channel.total_view_count = _to_int(stats.get("viewCount"))
channel.published_at = _parse_dt(snippet.get("publishedAt"))
channel.banner_url = branding.get("image", {}).get("bannerExternalUrl")
channel.external_links = _external_links(branding)
channel.topic_categories = topics.get("topicCategories")
channel.details_synced_at = now

3
backend/app/sysconfig.py
View file

@ -51,6 +51,9 @@ SPECS: tuple[ConfigSpec, ...] = (
# --- Batch sizes ---
ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000),
# --- Channel explore ---
# Days an explored-but-unsubscribed channel is kept before the cleanup job purges it.
ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650),
# --- YouTube Data API key (secret; optional — public reads prefer it over OAuth) ---
ConfigSpec("youtube_api_key", "str", "youtube", "youtube_api_key", secret=True),
# Optional egress proxy for YouTube API calls (fixed-IP host for an IP-restricted key).

2
backend/app/youtube/client.py
View file

@ -187,7 +187,7 @@ class YouTubeClient:
data = self._get(
"channels",
{
"part": "snippet,contentDetails,statistics,topicDetails",
"part": "snippet,contentDetails,statistics,topicDetails,brandingSettings",
"id": ",".join(batch),
"maxResults": 50,
},