diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..37e8237 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +# Stage 1: build the frontend SPA +FROM node:20-alpine AS frontend +WORKDIR /fe +COPY frontend/package.json ./ +RUN npm install +COPY frontend/ ./ +RUN npm run build + +# Stage 2: backend runtime, serving the built SPA +FROM python:3.12-slim +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +COPY backend/requirements.txt . +RUN pip install -r requirements.txt + +COPY backend/ . +COPY --from=frontend /fe/dist ./app/static_spa + +RUN adduser --disabled-password --gecos "" appuser && chown -R appuser /app +USER appuser + +EXPOSE 8000 +CMD ["sh", "entrypoint.sh"] diff --git a/README.md b/README.md index 7361bfa..95af6e8 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,12 @@ once and stored locally, so filtering/searching/sorting are instant and don't bu > - **M3** (auto-tagging): system tags for channel language (offline detection) and topic > (from YouTube topics + dominant category), regenerated automatically; user tags are > never overwritten. +> - **M4** (reader UI): React + Vite SPA with four color schemes (dark/light) and adjustable +> text size; grid/list feed scoped to your subscriptions, with faceted tag filters, +> content-type toggles (Normal/Shorts/Live), date range, search, sort, watch/save/hide +> state and per-channel filtering; clicking a video opens youtube.com. Accurate Shorts +> detection via the /shorts probe, a sync-status indicator with admin pause/resume, a +> filtered video count, and structured timestamped logging. ## Requirements diff --git a/backend/alembic/versions/0004_feed_state_prefs.py b/backend/alembic/versions/0004_feed_state_prefs.py new file mode 100644 index 0000000..a8ca407 --- /dev/null +++ b/backend/alembic/versions/0004_feed_state_prefs.py @@ -0,0 +1,52 @@ +"""user preferences + per-user video watch state + +Revision ID: 0004_feed_state_prefs +Revises: 0003_tags +Create Date: 2026-06-11 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "0004_feed_state_prefs" +down_revision: Union[str, None] = "0003_tags" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("users", sa.Column("preferences", sa.JSON(), nullable=True)) + + op.create_table( + "video_states", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "video_id", + sa.String(length=16), + sa.ForeignKey("videos.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("status", sa.String(length=12), nullable=False, server_default="new"), + sa.Column("watched_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.UniqueConstraint("user_id", "video_id", name="uq_user_video"), + ) + op.create_index("ix_video_states_user_id", "video_states", ["user_id"]) + op.create_index("ix_video_states_video_id", "video_states", ["video_id"]) + + +def downgrade() -> None: + op.drop_table("video_states") + op.drop_column("users", "preferences") diff --git a/backend/alembic/versions/0005_shorts_probed.py b/backend/alembic/versions/0005_shorts_probed.py new file mode 100644 index 0000000..a336cd7 --- /dev/null +++ b/backend/alembic/versions/0005_shorts_probed.py @@ -0,0 +1,28 @@ +"""add videos.shorts_probed + +Revision ID: 0005_shorts_probed +Revises: 0004_feed_state_prefs +Create Date: 2026-06-11 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "0005_shorts_probed" +down_revision: Union[str, None] = "0004_feed_state_prefs" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "videos", + sa.Column("shorts_probed", sa.Boolean(), nullable=False, server_default="false"), + ) + op.create_index("ix_videos_shorts_probed", "videos", ["shorts_probed"]) + + +def downgrade() -> None: + op.drop_index("ix_videos_shorts_probed", table_name="videos") + op.drop_column("videos", "shorts_probed") diff --git a/backend/alembic/versions/0006_app_state.py b/backend/alembic/versions/0006_app_state.py new file mode 100644 index 0000000..2720e2a --- /dev/null +++ b/backend/alembic/versions/0006_app_state.py @@ -0,0 +1,28 @@ +"""app_state (admin-controlled global flags) + +Revision ID: 0006_app_state +Revises: 0005_shorts_probed +Create Date: 2026-06-11 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "0006_app_state" +down_revision: Union[str, None] = "0005_shorts_probed" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "app_state", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("sync_paused", sa.Boolean(), nullable=False, server_default="false"), + ) + op.execute("INSERT INTO app_state (id, sync_paused) VALUES (1, false)") + + +def downgrade() -> None: + op.drop_table("app_state") diff --git a/backend/app/auth.py b/backend/app/auth.py index bc56522..1466c72 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime, timezone from authlib.integrations.starlette_client import OAuth, OAuthError @@ -14,6 +15,8 @@ from app.security import encrypt # (unsubscribe, playlist export). openid/email/profile give us the account identity. SCOPES = "openid email profile https://www.googleapis.com/auth/youtube" +log = logging.getLogger("subfeed.auth") + router = APIRouter(prefix="/auth", tags=["auth"]) oauth = OAuth() @@ -28,13 +31,14 @@ oauth.register( @router.get("/login") async def login(request: Request): - # access_type=offline + prompt=consent must be on the authorization URL so Google - # returns a refresh_token (required for background sync). + # access_type=offline ensures a refresh_token on first authorization. We avoid + # prompt=consent so returning users get a quick sign-in; the stored refresh token + # is kept when Google doesn't re-issue one (see the callback). return await oauth.google.authorize_redirect( request, settings.oauth_redirect_url, access_type="offline", - prompt="consent", + prompt="select_account", include_granted_scopes="true", ) @@ -52,6 +56,7 @@ async def callback(request: Request, db: Session = Depends(get_db)): email = (userinfo.get("email") or "").lower() if not email or email not in settings.allowed_email_set: + log.warning("Login denied (not on invite list): %s", email or "") raise HTTPException( status_code=403, detail="This Google account is not on the invite list." ) @@ -80,6 +85,7 @@ async def callback(request: Request, db: Session = Depends(get_db)): db.commit() request.session["user_id"] = user.id + log.info("Login: %s (id=%s, role=%s)", email, user.id, user.role) return RedirectResponse(url="/") diff --git a/backend/app/config.py b/backend/app/config.py index c149a52..f496a10 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -34,8 +34,11 @@ class Settings(BaseSettings): backfill_recent_max_videos: int = 100 backfill_recent_max_days: int = 365 - # Videos at or below this duration (seconds) are treated as Shorts. - shorts_max_seconds: int = 60 + # Shorts are confirmed by probing youtube.com/shorts/. Only videos at or below + # this duration (and not livestreams) are probed; longer videos are never Shorts. + shorts_probe_max_seconds: int = 180 + shorts_probe_batch: int = 150 + shorts_probe_interval_minutes: int = 2 # videos.list accepts up to 50 ids per call. enrich_batch_size: int = 50 @@ -52,6 +55,7 @@ class Settings(BaseSettings): # Number of recent video titles sampled per channel for language detection. autotag_title_sample: int = 40 autotag_interval_minutes: int = 30 + subscriptions_resync_minutes: int = 360 # live_status values hidden from the feed by default. Completed-stream VODs # ("was_live") are real watchable content and stay visible. feed_default_hidden_live: str = "live,upcoming" diff --git a/backend/app/main.py b/backend/app/main.py index 0b97348..9690916 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,7 +1,24 @@ +import logging +import sys from contextlib import asynccontextmanager from pathlib import Path -from fastapi import FastAPI +from fastapi import FastAPI, HTTPException + +# When started via uvicorn --log-config the "subfeed" logger is already configured +# (see log_config.json). This block is a timestamped fallback for other entrypoints +# (tests, scripts) so our logs are never silently dropped. +_subfeed_logger = logging.getLogger("subfeed") +if not _subfeed_logger.handlers: + _handler = logging.StreamHandler(sys.stdout) + _handler.setFormatter( + logging.Formatter("%(asctime)s %(levelname)-5s [%(name)s] %(message)s") + ) + _subfeed_logger.addHandler(_handler) + _subfeed_logger.setLevel(logging.INFO) + _subfeed_logger.propagate = False + +log = logging.getLogger("subfeed.app") from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles @@ -9,16 +26,18 @@ from starlette.middleware.sessions import SessionMiddleware from app import auth from app.config import settings -from app.routes import health, sync, tags +from app.routes import feed, health, me, sync, tags from app.scheduler import shutdown_scheduler, start_scheduler @asynccontextmanager async def lifespan(app: FastAPI): + log.info("Subfeed starting up") start_scheduler() try: yield finally: + log.info("Subfeed shutting down") shutdown_scheduler() @@ -44,11 +63,26 @@ app.include_router(health.router) app.include_router(auth.router) app.include_router(sync.router) app.include_router(tags.router) +app.include_router(feed.router) +app.include_router(me.router) -STATIC_DIR = Path(__file__).parent / "static" -app.mount("/assets", StaticFiles(directory=STATIC_DIR / "assets"), name="assets") +# The built SPA (populated by the Docker frontend build stage). +STATIC_DIR = Path(__file__).parent / "static_spa" +app.mount( + "/assets", + StaticFiles(directory=STATIC_DIR / "assets", check_dir=False), + name="assets", +) @app.get("/") async def index() -> FileResponse: return FileResponse(STATIC_DIR / "index.html") + + +@app.get("/{full_path:path}") +async def spa_fallback(full_path: str) -> FileResponse: + # Client-side routes fall back to index.html; real API/asset paths are matched above. + if full_path.startswith(("api/", "auth/", "healthz", "assets/")): + raise HTTPException(status_code=404) + return FileResponse(STATIC_DIR / "index.html") diff --git a/backend/app/models.py b/backend/app/models.py index c5f9012..a3e6f68 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -28,6 +28,8 @@ class User(Base): display_name: Mapped[str | None] = mapped_column(String(255)) avatar_url: Mapped[str | None] = mapped_column(String(1024)) role: Mapped[str] = mapped_column(String(16), default="user", server_default="user") + # Free-form UI preferences (theme, color scheme, font scale, default filters…). + preferences: Mapped[dict | None] = mapped_column(JSON) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now() ) @@ -144,6 +146,10 @@ class Video(Base): is_short: Mapped[bool] = mapped_column( Boolean, default=False, server_default="false", index=True ) + # Whether the youtube.com/shorts/ probe has run for this video. + shorts_probed: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false", index=True + ) # none | live | upcoming | premiere | was_live live_status: Mapped[str] = mapped_column( String(16), default="none", server_default="none", index=True @@ -201,6 +207,28 @@ class ChannelTag(Base): confidence: Mapped[float | None] = mapped_column(Float) +class VideoState(Base): + """Per-user watch state for a video. Rows exist only for non-default states; the + absence of a row means 'new/unseen'.""" + + __tablename__ = "video_states" + __table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_video"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), index=True + ) + video_id: Mapped[str] = mapped_column( + ForeignKey("videos.id", ondelete="CASCADE"), index=True + ) + # new | watched | saved | hidden + status: Mapped[str] = mapped_column(String(12), default="new", server_default="new") + watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + class ApiQuotaUsage(Base): """Tracks YouTube Data API units spent per Pacific-time day (the quota resets at midnight Pacific). The whole app shares one daily budget.""" @@ -209,3 +237,14 @@ class ApiQuotaUsage(Base): day: Mapped[date] = mapped_column(Date, primary_key=True) units_used: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + + +class AppState(Base): + """Single-row global app state (admin-controlled).""" + + __tablename__ = "app_state" + + id: Mapped[int] = mapped_column(primary_key=True, default=1) + sync_paused: Mapped[bool] = mapped_column( + Boolean, default=False, server_default="false" + ) diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py new file mode 100644 index 0000000..10af4de --- /dev/null +++ b/backend/app/routes/feed.py @@ -0,0 +1,282 @@ +from datetime import date, datetime, timedelta, timezone + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import Select, and_, false, func, or_, select +from sqlalchemy.orm import Session, aliased + +from app.auth import current_user +from app.db import get_db +from app.models import Channel, ChannelTag, Subscription, Tag, User, Video, VideoState + +router = APIRouter(prefix="/api", tags=["feed"]) + +VALID_STATES = {"new", "watched", "saved", "hidden"} +HIDDEN_LIVE = ("live", "upcoming") + + +def _channel_url(channel_id: str, handle: str | None) -> str: + if handle and handle.startswith("@"): + return f"https://www.youtube.com/{handle}" + return f"https://www.youtube.com/channel/{channel_id}" + + +def _serialize(row) -> dict: + return { + "id": row.id, + "title": row.title, + "channel_id": row.channel_id, + "channel_title": row.channel_title, + "channel_thumbnail": row.channel_thumbnail, + "channel_url": _channel_url(row.channel_id, row.channel_handle), + "published_at": row.published_at.isoformat() if row.published_at else None, + "thumbnail_url": row.thumbnail_url, + "duration_seconds": row.duration_seconds, + "view_count": row.view_count, + "is_short": row.is_short, + "live_status": row.live_status, + "status": row.status or "new", + "watch_url": f"https://www.youtube.com/watch?v={row.id}", + } + + +def _filtered_query( + db: Session, + user: User, + *, + tags: list[int], + tag_mode: str, + channel_id: str | None, + q: str | None, + min_duration: int | None, + max_duration: int | None, + max_age_days: int | None, + published_after: date | None, + published_before: date | None, + show_normal: bool, + include_shorts: bool, + include_live: bool, + show: str, +) -> tuple[Select, object]: + """Build the feed query (joins + all WHERE filters), shared by /feed and /feed/count. + Returns the column-bearing select plus the watch-status expression for sorting.""" + state = aliased(VideoState) + status_expr = func.coalesce(state.status, "new") + + query = ( + select( + Video.id, + Video.title, + Video.channel_id, + Channel.title.label("channel_title"), + Channel.thumbnail_url.label("channel_thumbnail"), + Channel.handle.label("channel_handle"), + Video.published_at, + Video.thumbnail_url, + Video.duration_seconds, + Video.view_count, + Video.is_short, + Video.live_status, + status_expr.label("status"), + ) + .join(Channel, Channel.id == Video.channel_id) + # Only channels this user is subscribed to (and hasn't hidden). + .join( + Subscription, + and_( + Subscription.channel_id == Video.channel_id, + Subscription.user_id == user.id, + Subscription.hidden.is_(False), + ), + ) + .outerjoin(state, and_(state.video_id == Video.id, state.user_id == user.id)) + ) + + if channel_id: + query = query.where(Video.channel_id == channel_id) + if min_duration is not None: + query = query.where(Video.duration_seconds >= min_duration) + if max_duration is not None: + query = query.where(Video.duration_seconds <= max_duration) + if max_age_days is not None: + cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400 + query = query.where( + Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc) + ) + if published_after is not None: + start = datetime.combine( + published_after, datetime.min.time(), tzinfo=timezone.utc + ) + query = query.where(Video.published_at >= start) + if published_before is not None: + end = datetime.combine( + published_before, datetime.min.time(), tzinfo=timezone.utc + ) + timedelta(days=1) + query = query.where(Video.published_at < end) + if q: + like = f"%{q}%" + query = query.where(or_(Video.title.ilike(like), Channel.title.ilike(like))) + + if tags: + # AND across tag categories (e.g. language AND topic narrows), OR within a + # category; the any/all toggle controls multiple topic tags. + cat_rows = db.execute(select(Tag.id, Tag.category).where(Tag.id.in_(tags))).all() + by_category: dict[str, list[int]] = {} + for tag_id, category in cat_rows: + by_category.setdefault(category, []).append(tag_id) + visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id) + for category, ids in by_category.items(): + if category == "topic" and tag_mode == "and" and len(ids) > 1: + sub = ( + select(ChannelTag.channel_id) + .where(ChannelTag.tag_id.in_(ids), visible) + .group_by(ChannelTag.channel_id) + .having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids))) + ) + else: + sub = select(ChannelTag.channel_id).where( + ChannelTag.tag_id.in_(ids), visible + ) + query = query.where(Video.channel_id.in_(sub)) + + # Content type: Normal / Shorts / Live·Upcoming as a union of enabled types. + # Explicit Watched/Saved/Hidden views show every type so nothing goes missing. + explicit_view = show in ("watched", "saved", "hidden") + if not explicit_view: + type_clauses = [] + if show_normal: + type_clauses.append( + and_(Video.is_short.is_(False), Video.live_status.notin_(HIDDEN_LIVE)) + ) + if include_shorts: + type_clauses.append(Video.is_short.is_(True)) + if include_live: + type_clauses.append(Video.live_status.in_(HIDDEN_LIVE)) + query = query.where(or_(*type_clauses) if type_clauses else false()) + + if show == "unwatched": + query = query.where(status_expr.notin_(("watched", "hidden"))) + elif show == "watched": + query = query.where(status_expr == "watched") + elif show == "saved": + query = query.where(status_expr == "saved") + elif show == "hidden": + query = query.where(status_expr == "hidden") + else: # all + query = query.where(status_expr != "hidden") + + return query, status_expr + + +# Shared query parameters for /feed and /feed/count. +def _feed_params( + tags: list[int] = Query(default=[]), + tag_mode: str = "or", + channel_id: str | None = None, + q: str | None = None, + min_duration: int | None = None, + max_duration: int | None = None, + max_age_days: int | None = None, + published_after: date | None = None, + published_before: date | None = None, + show_normal: bool = True, + include_shorts: bool = False, + include_live: bool = False, + show: str = "unwatched", +) -> dict: + return { + "tags": tags, + "tag_mode": tag_mode, + "channel_id": channel_id, + "q": q, + "min_duration": min_duration, + "max_duration": max_duration, + "max_age_days": max_age_days, + "published_after": published_after, + "published_before": published_before, + "show_normal": show_normal, + "include_shorts": include_shorts, + "include_live": include_live, + "show": show, + } + + +SORTS = { + "newest": Video.published_at.desc().nulls_last(), + "oldest": Video.published_at.asc().nulls_last(), + "views": Video.view_count.desc().nulls_last(), + "duration_desc": Video.duration_seconds.desc().nulls_last(), + "duration_asc": Video.duration_seconds.asc().nulls_last(), + "title": func.lower(Video.title).asc().nulls_last(), + "subscribers": Channel.subscriber_count.desc().nulls_last(), +} + + +@router.get("/feed") +def get_feed( + params: dict = Depends(_feed_params), + sort: str = "newest", + seed: int = 0, + limit: int = Query(default=60, le=200), + offset: int = 0, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + query, _status = _filtered_query(db, user, **params) + order = SORTS.get(sort) + if order is None and sort == "shuffle": + order = func.md5(func.concat(Video.id, str(seed))) + query = query.order_by(order if order is not None else SORTS["newest"]) + + rows = db.execute(query.offset(offset).limit(limit + 1)).all() + has_more = len(rows) > limit + return { + "items": [_serialize(r) for r in rows[:limit]], + "has_more": has_more, + "offset": offset, + "limit": limit, + } + + +@router.get("/feed/count") +def get_feed_count( + params: dict = Depends(_feed_params), + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + query, _status = _filtered_query(db, user, **params) + total = db.scalar(select(func.count()).select_from(query.subquery())) + return {"count": total or 0} + + +@router.post("/videos/{video_id}/state") +def set_video_state( + video_id: str, + payload: dict, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + status = payload.get("status") + if status not in VALID_STATES: + raise HTTPException(status_code=400, detail=f"status must be one of {VALID_STATES}") + if db.get(Video, video_id) is None: + raise HTTPException(status_code=404, detail="Unknown video") + + row = db.execute( + select(VideoState).where( + VideoState.user_id == user.id, VideoState.video_id == video_id + ) + ).scalar_one_or_none() + + if status == "new": + if row is not None: + db.delete(row) + db.commit() + return {"video_id": video_id, "status": "new"} + + if row is None: + row = VideoState(user_id=user.id, video_id=video_id) + db.add(row) + row.status = status + row.watched_at = datetime.now(timezone.utc) if status == "watched" else row.watched_at + db.commit() + return {"video_id": video_id, "status": status} diff --git a/backend/app/routes/me.py b/backend/app/routes/me.py new file mode 100644 index 0000000..9919aed --- /dev/null +++ b/backend/app/routes/me.py @@ -0,0 +1,35 @@ +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from app.auth import current_user +from app.db import get_db +from app.models import User + +router = APIRouter(prefix="/api/me", tags=["me"]) + + +@router.get("") +def get_me(user: User = Depends(current_user)) -> dict: + return { + "id": user.id, + "email": user.email, + "display_name": user.display_name, + "avatar_url": user.avatar_url, + "role": user.role, + "preferences": user.preferences or {}, + } + + +@router.put("/preferences") +def update_preferences( + preferences: dict, + user: User = Depends(current_user), + db: Session = Depends(get_db), +) -> dict: + # Merge so partial updates don't wipe other keys. + merged = dict(user.preferences or {}) + merged.update(preferences) + user.preferences = merged + db.add(user) + db.commit() + return {"preferences": merged} diff --git a/backend/app/routes/sync.py b/backend/app/routes/sync.py index e76493a..4efd941 100644 --- a/backend/app/routes/sync.py +++ b/backend/app/routes/sync.py @@ -1,8 +1,8 @@ -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException from sqlalchemy import func, select from sqlalchemy.orm import Session -from app import quota +from app import quota, state from app.auth import current_user from app.db import get_db from app.models import Channel, Subscription, User, Video @@ -79,7 +79,35 @@ def sync_status( return { "subscriptions": sub_count, "channels_total": db.scalar(select(func.count()).select_from(Channel)), + "channels_backfilling": db.scalar( + select(func.count()).select_from(Channel).where(Channel.backfill_done.is_(False)) + ), "videos_total": db.scalar(select(func.count()).select_from(Video)), + "pending_enrich": db.scalar( + select(func.count()).select_from(Video).where(Video.enriched_at.is_(None)) + ), "quota_used_today": quota.units_used_today(db), "quota_remaining_today": quota.remaining_today(db), + "paused": state.is_sync_paused(db), + "is_admin": user.role == "admin", } + + +@router.post("/pause") +def pause_sync( + user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + if user.role != "admin": + raise HTTPException(status_code=403, detail="Admin only") + state.set_sync_paused(db, True) + return {"paused": True} + + +@router.post("/resume") +def resume_sync( + user: User = Depends(current_user), db: Session = Depends(get_db) +) -> dict: + if user.role != "admin": + raise HTTPException(status_code=403, detail="Admin only") + state.set_sync_paused(db, False) + return {"paused": False} diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index d95e589..8925890 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -6,12 +6,15 @@ from apscheduler.schedulers.background import BackgroundScheduler from app.config import settings from app.db import SessionLocal +from app.state import is_sync_paused from app.sync.autotag import run_autotag_all from app.sync.runner import ( run_deep_backfill, run_enrich, run_recent_backfill, run_rss_poll, + run_shorts, + run_subscription_resync, ) logger = logging.getLogger("subfeed.scheduler") @@ -22,6 +25,9 @@ _scheduler: BackgroundScheduler | None = None def _job(name: str, fn) -> None: db = SessionLocal() try: + if is_sync_paused(db): + logger.info("job %s skipped (sync paused)", name) + return result = fn(db) logger.info("job %s -> %s", name, result) except Exception: @@ -53,6 +59,14 @@ def _autotag_job() -> None: _job("autotag", lambda db: run_autotag_all(db, only_missing=True)) +def _shorts_job() -> None: + _job("shorts", run_shorts) + + +def _subscriptions_job() -> None: + _job("subscriptions", run_subscription_resync) + + def start_scheduler() -> None: global _scheduler if not settings.scheduler_enabled or _scheduler is not None: @@ -76,6 +90,18 @@ def start_scheduler() -> None: minutes=settings.autotag_interval_minutes, id="autotag", ) + scheduler.add_job( + _shorts_job, + "interval", + minutes=settings.shorts_probe_interval_minutes, + id="shorts", + ) + scheduler.add_job( + _subscriptions_job, + "interval", + minutes=settings.subscriptions_resync_minutes, + id="subscriptions", + ) scheduler.start() _scheduler = scheduler logger.info("scheduler started") diff --git a/backend/app/state.py b/backend/app/state.py new file mode 100644 index 0000000..ad85e19 --- /dev/null +++ b/backend/app/state.py @@ -0,0 +1,29 @@ +"""Global admin-controlled app state (e.g. pausing background sync).""" +import logging + +from sqlalchemy.orm import Session + +from app.models import AppState + +log = logging.getLogger("subfeed.state") + + +def _row(db: Session) -> AppState: + row = db.get(AppState, 1) + if row is None: + row = AppState(id=1, sync_paused=False) + db.add(row) + db.commit() + return row + + +def is_sync_paused(db: Session) -> bool: + return _row(db).sync_paused + + +def set_sync_paused(db: Session, paused: bool) -> None: + row = _row(db) + row.sync_paused = paused + db.add(row) + db.commit() + log.info("Background sync %s", "paused" if paused else "resumed") diff --git a/backend/app/sync/autotag.py b/backend/app/sync/autotag.py index 121efe3..e10db9e 100644 --- a/backend/app/sync/autotag.py +++ b/backend/app/sync/autotag.py @@ -5,9 +5,14 @@ detection over a sample of recent video titles. Topics are mapped from YouTube's topicDetails categories and the channel's dominant video category. System tags are regenerated freely; user tags are never touched here. """ +import logging +import re + from sqlalchemy import and_, exists, func, select from sqlalchemy.orm import Session +log = logging.getLogger("subfeed.autotag") + from app.config import settings from app.models import Channel, ChannelTag, Tag, Video @@ -143,9 +148,18 @@ def map_topic_slug(slug: str) -> str | None: return _TOPIC_SLUGS.get(slug) +def _clean_title(title: str | None) -> str: + """Strip emojis, @mentions, #tags, URLs, numbers and punctuation so the language + detector sees actual words, not caps/emoji-heavy noise.""" + text = title or "" + text = re.sub(r"http\S+", " ", text) + text = re.sub(r"[@#]\w+", " ", text) + text = re.sub(r"\d+", " ", text) + text = re.sub(r"[^\w\s]", " ", text, flags=re.UNICODE).replace("_", " ") + return " ".join(w for w in text.split() if len(w) > 1) + + def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None, float]: - if channel.default_language: - return channel.default_language.split("-")[0].lower(), 0.99 titles = ( db.execute( select(Video.title) @@ -156,11 +170,16 @@ def detect_channel_language(db: Session, channel: Channel) -> tuple[str | None, .scalars() .all() ) - text = " . ".join(t for t in titles if t).strip() - if len(text) < 15: - return None, 0.0 - lang, confidence = _classify(text) - return lang, float(confidence) + # Detect over one cleaned, concatenated blob — more context and far less skew from + # short, emoji/caps-heavy titles than per-title voting. + blob = " ".join(_clean_title(t) for t in titles).strip() + if len(blob) >= 15: + lang, confidence = _classify(blob) + return lang, float(confidence) + # Sparse text: fall back to the channel's declared language if any. + if channel.default_language: + return channel.default_language.split("-")[0].lower(), 0.6 + return None, 0.0 def compute_channel_topics(db: Session, channel: Channel) -> set[str]: @@ -257,6 +276,7 @@ def run_autotag_all(db: Session, only_missing: bool = False) -> dict: tagged += 1 except Exception: db.rollback() + log.exception("Auto-tagging failed for channel %s", channel.id) removed = _cleanup_orphan_system_tags(db) return {"channels_tagged": tagged, "orphan_tags_removed": removed} diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index 9f14932..a8f87c9 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -4,17 +4,23 @@ Channel/video data is shared across users, so background work acts through a "se user" (any user with a stored refresh token) unless a YOUTUBE_API_KEY is configured for public reads. """ +import logging + from sqlalchemy import select from sqlalchemy.orm import Session from app import quota from app.config import settings + +log = logging.getLogger("subfeed.sync") from app.models import Channel, OAuthToken, User +from app.sync.subscriptions import import_subscriptions from app.sync.videos import ( backfill_channel_deep, backfill_channel_recent, enrich_pending, poll_rss_channel, + run_shorts_classification, ) from app.youtube.client import YouTubeClient @@ -41,6 +47,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> int: new += poll_rss_channel(db, channel) except Exception: db.rollback() + log.exception("RSS poll failed for channel %s", channel.id) return new @@ -60,6 +67,31 @@ def run_enrich(db: Session, max_batches: int = 200, floor: int = 200) -> int: return total +def run_shorts(db: Session) -> dict: + return run_shorts_classification(db) + + +def run_subscription_resync(db: Session) -> dict: + """Re-import every user's subscriptions so unsubscribes and new subscriptions on + YouTube are reflected automatically.""" + users = ( + db.execute( + select(User) + .join(OAuthToken) + .where(OAuthToken.refresh_token_enc.is_not(None)) + ) + .scalars() + .all() + ) + for user in users: + try: + import_subscriptions(db, user) + except Exception: + db.rollback() + log.exception("Subscription resync failed for user %s", user.id) + return {"users": len(users)} + + def run_recent_backfill( db: Session, channels: list[Channel] | None = None, max_channels: int | None = None ) -> dict: @@ -85,6 +117,7 @@ def run_recent_backfill( processed += 1 except Exception: db.rollback() + log.exception("Recent backfill failed for channel %s", channel.id) return {"channels_processed": processed, "videos_added": videos_added} @@ -113,4 +146,5 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - processed += 1 except Exception: db.rollback() + log.exception("Deep backfill failed for channel %s", channel.id) return {"channels_processed": processed, "videos_added": videos_added} diff --git a/backend/app/sync/subscriptions.py b/backend/app/sync/subscriptions.py index 653eba1..9c139e3 100644 --- a/backend/app/sync/subscriptions.py +++ b/backend/app/sync/subscriptions.py @@ -1,4 +1,5 @@ """Import the authenticated user's YouTube subscriptions and channel metadata.""" +import logging from datetime import datetime, timezone from sqlalchemy import select @@ -7,6 +8,8 @@ from sqlalchemy.orm import Session from app.models import Channel, Subscription, User from app.youtube.client import YouTubeClient, best_thumbnail +log = logging.getLogger("subfeed.sync") + def _to_int(value) -> int | None: try: @@ -107,9 +110,11 @@ def import_subscriptions(db: Session, user: User) -> dict: detailed = len(items) db.commit() - return { + result = { "subscriptions": len(fetched), "channels_new": new_channels, "channels_detailed": detailed, "removed_stale": removed, } + log.info("Subscription import (user %s): %s", user.id, result) + return result diff --git a/backend/app/sync/videos.py b/backend/app/sync/videos.py index 5818e5c..2ac9c56 100644 --- a/backend/app/sync/videos.py +++ b/backend/app/sync/videos.py @@ -2,9 +2,10 @@ uploads playlist, and enrichment via videos.list (duration, stats, category, Shorts and livestream classification).""" import re +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta, timezone -from sqlalchemy import select +from sqlalchemy import or_, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session @@ -12,6 +13,7 @@ from app.config import settings from app.models import Channel, Video from app.youtube.client import YouTubeClient, best_thumbnail from app.youtube.rss import fetch_channel_feed +from app.youtube.shorts import make_client, probe_is_short _DURATION_RE = re.compile( r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE @@ -195,11 +197,7 @@ def apply_video_details(video: Video, item: dict) -> None: "defaultAudioLanguage" ) video.live_status = _live_status(snippet, live) - video.is_short = bool( - video.live_status == "none" - and video.duration_seconds - and 0 < video.duration_seconds <= settings.shorts_max_seconds - ) + # is_short is decided separately by the youtube.com/shorts probe (run_shorts_classification). def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int: @@ -225,3 +223,60 @@ def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> enriched += 1 db.commit() return enriched + + +def run_shorts_classification(db: Session, limit: int | None = None) -> dict: + """Finalize Shorts classification: cheaply mark non-candidates, then probe the + youtube.com/shorts URL for short-enough, non-live, enriched videos.""" + limit = limit or settings.shorts_probe_batch + probe_max = settings.shorts_probe_max_seconds + + # 1) Anything enriched that can't be a Short -> finalize without a probe. + db.execute( + update(Video) + .where( + Video.shorts_probed.is_(False), + Video.enriched_at.is_not(None), + or_( + Video.duration_seconds.is_(None), + Video.duration_seconds > probe_max, + Video.live_status != "none", + ), + ) + .values(is_short=False, shorts_probed=True) + ) + db.commit() + + # 2) Probe the remaining candidates. + candidates = ( + db.execute( + select(Video).where( + Video.shorts_probed.is_(False), + Video.enriched_at.is_not(None), + Video.duration_seconds <= probe_max, + Video.live_status == "none", + ).limit(limit) + ) + .scalars() + .all() + ) + if not candidates: + return {"probed": 0, "shorts": 0} + + probed = 0 + shorts = 0 + with make_client() as client: + def work(video: Video): + return video, probe_is_short(client, video.id) + + with ThreadPoolExecutor(max_workers=16) as pool: + for video, result in pool.map(work, candidates): + if result is None: + continue # leave unprobed; retry on a later run + video.is_short = result + video.shorts_probed = True + probed += 1 + if result: + shorts += 1 + db.commit() + return {"probed": probed, "shorts": shorts} diff --git a/backend/app/youtube/client.py b/backend/app/youtube/client.py index 87305cd..274861c 100644 --- a/backend/app/youtube/client.py +++ b/backend/app/youtube/client.py @@ -5,6 +5,7 @@ Public reads (channels/videos/playlistItems) use the configured API key when ava so they don't depend on a specific user's token; subscriptions.list?mine=true always uses OAuth. """ +import logging from collections.abc import Iterator from datetime import datetime, timedelta, timezone @@ -15,6 +16,8 @@ from app.config import settings from app.models import User from app.security import decrypt +log = logging.getLogger("subfeed.youtube") + GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token" API_BASE = "https://www.googleapis.com/youtube/v3" @@ -76,6 +79,7 @@ class YouTubeClient: tok.expiry = now + timedelta(seconds=int(data.get("expires_in", 3600))) self.db.add(tok) self.db.commit() + log.info("Refreshed access token for user %s", self.user.id) return tok.access_token # --- core request --- @@ -89,6 +93,7 @@ class YouTubeClient: resp = self._http.get(f"{API_BASE}/{path}", params=p, headers=headers) quota.record_usage(self.db, cost) if resp.status_code != 200: + log.warning("YouTube API %s -> %s: %s", path, resp.status_code, resp.text[:200]) raise YouTubeError(f"GET {path} -> {resp.status_code}: {resp.text[:300]}") return resp.json() diff --git a/backend/app/youtube/shorts.py b/backend/app/youtube/shorts.py new file mode 100644 index 0000000..046380e --- /dev/null +++ b/backend/app/youtube/shorts.py @@ -0,0 +1,32 @@ +"""Confirm whether a video is a Short by probing youtube.com/shorts/. + +A real Short returns HTTP 200 at that URL; a regular video redirects to /watch. +This uses no API quota.""" +import httpx + +SHORTS_URL = "https://www.youtube.com/shorts/{video_id}" +# The SOCS cookie skips YouTube's cookie-consent interstitial, which otherwise +# redirects every server-side request to consent.youtube.com. +_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + "Cookie": "SOCS=CAI", +} + + +def probe_is_short(client: httpx.Client, video_id: str) -> bool | None: + """True if a Short, False if a regular video, None if undetermined (retry later).""" + try: + resp = client.get( + SHORTS_URL.format(video_id=video_id), follow_redirects=False + ) + except httpx.HTTPError: + return None + if resp.status_code == 200: + return True + if resp.status_code in (301, 302, 303, 307, 308): + return False + return None + + +def make_client() -> httpx.Client: + return httpx.Client(timeout=10.0, headers=_HEADERS) diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index 5d0809a..a340cf4 100644 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -5,4 +5,4 @@ echo "Applying database migrations..." alembic upgrade head echo "Starting Subfeed API..." -exec uvicorn app.main:app --host 0.0.0.0 --port 8000 +exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --log-config log_config.json diff --git a/backend/log_config.json b/backend/log_config.json new file mode 100644 index 0000000..47f8c1b --- /dev/null +++ b/backend/log_config.json @@ -0,0 +1,40 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "default": { + "()": "uvicorn.logging.DefaultFormatter", + "fmt": "%(asctime)s %(levelprefix)s [%(name)s] %(message)s" + }, + "access": { + "()": "uvicorn.logging.AccessFormatter", + "fmt": "%(asctime)s %(levelprefix)s %(client_addr)s - \"%(request_line)s\" %(status_code)s" + }, + "plain": { + "format": "%(asctime)s %(levelname)-5s [%(name)s] %(message)s" + } + }, + "handlers": { + "default": { + "class": "logging.StreamHandler", + "formatter": "default", + "stream": "ext://sys.stdout" + }, + "access": { + "class": "logging.StreamHandler", + "formatter": "access", + "stream": "ext://sys.stdout" + }, + "plain": { + "class": "logging.StreamHandler", + "formatter": "plain", + "stream": "ext://sys.stdout" + } + }, + "loggers": { + "uvicorn": { "handlers": ["default"], "level": "INFO", "propagate": false }, + "uvicorn.error": { "handlers": ["default"], "level": "INFO", "propagate": false }, + "uvicorn.access": { "handlers": ["access"], "level": "INFO", "propagate": false }, + "subfeed": { "handlers": ["plain"], "level": "INFO", "propagate": false } + } +} diff --git a/docker-compose.yml b/docker-compose.yml index ddc15d9..5ead8e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,8 @@ services: api: build: - context: ./backend + context: . + dockerfile: Dockerfile env_file: .env environment: DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-subfeed}:${POSTGRES_PASSWORD:-subfeed}@db:5432/${POSTGRES_DB:-subfeed} diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..8e21e5b --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,12 @@ + + + + + + Subfeed + + +
+ + + diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..8ef7c07 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,27 @@ +{ + "name": "subfeed-frontend", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "@tanstack/react-query": "^5.51.0", + "clsx": "^2.1.1", + "lucide-react": "^0.408.0", + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^4.3.1", + "autoprefixer": "^10.4.19", + "postcss": "^8.4.39", + "tailwindcss": "^3.4.6", + "typescript": "^5.5.3", + "vite": "^5.3.4" + } +} diff --git a/frontend/postcss.config.js b/frontend/postcss.config.js new file mode 100644 index 0000000..2aa7205 --- /dev/null +++ b/frontend/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +}; diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 0000000..ce58cda --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,106 @@ +import { useEffect, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { api, HttpError, type FeedFilters } from "./lib/api"; +import { + applyTheme, + DEFAULT_THEME, + loadLocalTheme, + saveLocalTheme, + type ThemePrefs, +} from "./lib/theme"; +import Login from "./components/Login"; +import Header from "./components/Header"; +import Sidebar from "./components/Sidebar"; +import Feed from "./components/Feed"; +import Toaster from "./components/Toaster"; + +const DEFAULT_FILTERS: FeedFilters = { + tags: [], + tagMode: "or", + q: "", + sort: "newest", + includeNormal: true, + includeShorts: false, + includeLive: false, + show: "unwatched", +}; + +const FILTERS_KEY = "subfeed.filters"; + +function loadFilters(): FeedFilters { + try { + return { ...DEFAULT_FILTERS, ...JSON.parse(localStorage.getItem(FILTERS_KEY) || "{}") }; + } catch { + return DEFAULT_FILTERS; + } +} + +export default function App() { + const [theme, setThemeState] = useState(() => loadLocalTheme()); + const [filters, setFiltersState] = useState(loadFilters); + const [view, setView] = useState<"grid" | "list">("grid"); + + function setFilters(next: FeedFilters) { + setFiltersState(next); + localStorage.setItem(FILTERS_KEY, JSON.stringify(next)); + } + + useEffect(() => applyTheme(theme), [theme]); + + const meQuery = useQuery({ queryKey: ["me"], queryFn: api.me }); + + // On login, adopt server-stored preferences. + useEffect(() => { + const prefs = meQuery.data?.preferences; + if (!prefs) return; + if (prefs.theme) { + const merged = { ...DEFAULT_THEME, ...prefs.theme }; + setThemeState(merged); + saveLocalTheme(merged); + } + if (prefs.view === "grid" || prefs.view === "list") setView(prefs.view); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [meQuery.data?.id]); + + function setTheme(next: ThemePrefs) { + setThemeState(next); + saveLocalTheme(next); + api.savePrefs({ theme: next }).catch(() => {}); + } + function changeView(v: "grid" | "list") { + setView(v); + api.savePrefs({ view: v }).catch(() => {}); + } + + if (meQuery.isLoading) + return
Loading…
; + if (meQuery.error instanceof HttpError && meQuery.error.status === 401) + return ; + if (meQuery.error) + return ( +
+ Something went wrong. +
+ ); + + return ( +
+
+
+ +
+ +
+
+ +
+ ); +} diff --git a/frontend/src/components/Feed.tsx b/frontend/src/components/Feed.tsx new file mode 100644 index 0000000..a1c8c92 --- /dev/null +++ b/frontend/src/components/Feed.tsx @@ -0,0 +1,132 @@ +import { useEffect, useRef, useState } from "react"; +import { useInfiniteQuery, useQuery, useQueryClient } from "@tanstack/react-query"; +import { api, type FeedFilters, type Video } from "../lib/api"; +import { toast } from "../lib/toast"; +import VideoCard from "./VideoCard"; + +const PAGE = 60; + +function matchesView(status: string, show: string): boolean { + switch (show) { + case "hidden": + return status === "hidden"; + case "watched": + return status === "watched"; + case "saved": + return status === "saved"; + case "unwatched": + return status !== "watched" && status !== "hidden"; + default: + return status !== "hidden"; // all + } +} + +export default function Feed({ + filters, + setFilters, + view, +}: { + filters: FeedFilters; + setFilters: (f: FeedFilters) => void; + view: "grid" | "list"; +}) { + const [overrides, setOverrides] = useState>({}); + const qc = useQueryClient(); + + const query = useInfiniteQuery({ + queryKey: ["feed", filters], + queryFn: ({ pageParam }) => api.feed(filters, pageParam as number, PAGE), + initialPageParam: 0, + getNextPageParam: (last) => (last.has_more ? last.offset + last.limit : undefined), + }); + + useEffect(() => setOverrides({}), [filters]); + + const countQuery = useQuery({ + queryKey: ["feed-count", filters], + queryFn: () => api.feedCount(filters), + staleTime: 30_000, + }); + + const sentinel = useRef(null); + const { hasNextPage, isFetchingNextPage, fetchNextPage } = query; + useEffect(() => { + const el = sentinel.current; + if (!el) return; + const io = new IntersectionObserver( + (entries) => { + if (entries[0].isIntersecting && hasNextPage && !isFetchingNextPage) { + fetchNextPage(); + } + }, + { rootMargin: "800px" } + ); + io.observe(el); + return () => io.disconnect(); + }, [hasNextPage, isFetchingNextPage, fetchNextPage]); + + function onState(id: string, status: string) { + setOverrides((o) => ({ ...o, [id]: status })); + // Refetch once the server has the change so other views (e.g. Hidden) are in sync. + api + .setState(id, status) + .then(() => qc.invalidateQueries({ queryKey: ["feed"] })) + .catch(() => {}); + if (status === "hidden") { + toast("Video hidden", { label: "Undo", onClick: () => onState(id, "new") }); + } + } + + function onChannelFilter(channelId: string, channelName: string) { + setFilters({ ...filters, channelId, channelName }); + } + + const items: Video[] = (query.data?.pages ?? []) + .flatMap((p) => p.items) + .map((v) => (overrides[v.id] ? { ...v, status: overrides[v.id] } : v)) + .filter((v) => matchesView(v.status, filters.show)); + + if (query.isLoading) return
Loading feed…
; + if (query.isError) return
Couldn't load the feed.
; + if (items.length === 0) + return
No videos match these filters.
; + + return ( +
+
+ {countQuery.data + ? `${countQuery.data.count.toLocaleString()} video${countQuery.data.count === 1 ? "" : "s"}` + : " "} +
+ {view === "grid" ? ( +
+ {items.map((v) => ( + + ))} +
+ ) : ( +
+ {items.map((v) => ( + + ))} +
+ )} +
+ {isFetchingNextPage && ( +
Loading more…
+ )} +
+ ); +} diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx new file mode 100644 index 0000000..2524b9f --- /dev/null +++ b/frontend/src/components/Header.tsx @@ -0,0 +1,147 @@ +import { useState } from "react"; +import { + LayoutGrid, + List, + LogOut, + Moon, + Palette, + Search, + Sun, +} from "lucide-react"; +import { SCHEMES, type Scheme, type ThemePrefs } from "../lib/theme"; +import type { FeedFilters, Me } from "../lib/api"; +import SyncStatus from "./SyncStatus"; + +function IconBtn(props: React.ButtonHTMLAttributes) { + const { className = "", ...rest } = props; + return ( +
+
Text size
+ setTheme({ ...theme, fontScale: Number(e.target.value) })} + className="w-full accent-accent" + /> +
+ {Math.round(theme.fontScale * 100)}% +
+ + ); +} + +export default function Header({ + me, + theme, + setTheme, + filters, + setFilters, + view, + setView, +}: { + me: Me; + theme: ThemePrefs; + setTheme: (t: ThemePrefs) => void; + filters: FeedFilters; + setFilters: (f: FeedFilters) => void; + view: "grid" | "list"; + setView: (v: "grid" | "list") => void; +}) { + const [menuOpen, setMenuOpen] = useState(false); + + async function logout() { + await fetch("/auth/logout", { method: "POST", credentials: "include" }); + location.reload(); + } + + return ( +
+
+ Subfeed +
+ + + +
+ + setFilters({ ...filters, q: e.target.value })} + placeholder="Search your subscriptions…" + className="w-full bg-card border border-border rounded-full pl-9 pr-4 py-2 text-sm outline-none focus:border-accent" + /> +
+ +
+ setView(view === "grid" ? "list" : "grid")} + title={view === "grid" ? "List view" : "Grid view"} + > + {view === "grid" ? : } + + setTheme({ ...theme, mode: theme.mode === "dark" ? "light" : "dark" })} + title="Toggle dark / light" + > + {theme.mode === "dark" ? : } + +
+ setMenuOpen((o) => !o)} title="Theme"> + + + {menuOpen && ( + <> +
setMenuOpen(false)} /> + + + )} +
+ +
+ {me.avatar_url ? ( + + ) : ( +
+ {me.email[0]?.toUpperCase()} +
+ )} + + + +
+
+
+ ); +} diff --git a/frontend/src/components/Login.tsx b/frontend/src/components/Login.tsx new file mode 100644 index 0000000..a79f418 --- /dev/null +++ b/frontend/src/components/Login.tsx @@ -0,0 +1,23 @@ +export default function Login() { + return ( +
+
+
+ Subfeed +
+

+ Your subscriptions, filtered your way. +
+ Sign in with your Google account. +

+ + Sign in with Google + +
Invite-only access.
+
+
+ ); +} diff --git a/frontend/src/components/Sidebar.tsx b/frontend/src/components/Sidebar.tsx new file mode 100644 index 0000000..dcc1a0a --- /dev/null +++ b/frontend/src/components/Sidebar.tsx @@ -0,0 +1,291 @@ +import { useQuery } from "@tanstack/react-query"; +import { X } from "lucide-react"; +import { api, type FeedFilters, type Tag } from "../lib/api"; + +const SORTS = [ + { id: "newest", label: "Newest" }, + { id: "oldest", label: "Oldest" }, + { id: "views", label: "Most viewed" }, + { id: "duration_desc", label: "Longest" }, + { id: "duration_asc", label: "Shortest" }, + { id: "title", label: "Name (A–Z)" }, + { id: "subscribers", label: "Channel subscribers" }, + { id: "shuffle", label: "Surprise me" }, +]; + +const SHOWS = [ + { id: "unwatched", label: "Unwatched" }, + { id: "all", label: "All" }, + { id: "watched", label: "Watched" }, + { id: "saved", label: "Saved" }, + { id: "hidden", label: "Hidden" }, +]; + +function TagChip({ + tag, + active, + onClick, +}: { + tag: Tag; + active: boolean; + onClick: () => void; +}) { + return ( + + ); +} + +export default function Sidebar({ + filters, + setFilters, +}: { + filters: FeedFilters; + setFilters: (f: FeedFilters) => void; +}) { + const tagsQuery = useQuery({ queryKey: ["tags"], queryFn: api.tags }); + const tags = tagsQuery.data ?? []; + const languages = tags.filter((t) => t.category === "language"); + const topics = tags.filter((t) => t.category === "topic"); + + function toggleTag(id: number) { + const has = filters.tags.includes(id); + setFilters({ + ...filters, + tags: has ? filters.tags.filter((t) => t !== id) : [...filters.tags, id], + }); + } + + const active = + filters.tags.length > 0 || + !filters.includeNormal || + filters.includeShorts || + filters.includeLive || + filters.show !== "unwatched" || + filters.sort !== "newest" || + !!filters.channelId || + !!filters.dateFrom || + !!filters.dateTo; + + return ( + + ); +} + +function Section({ + title, + right, + children, +}: { + title: string; + right?: React.ReactNode; + children: React.ReactNode; +}) { + return ( +
+
+
{title}
+ {right} +
+ {children} +
+ ); +} + +function Toggle({ + label, + checked, + onChange, +}: { + label: string; + checked: boolean; + onChange: (v: boolean) => void; +}) { + return ( + + ); +} diff --git a/frontend/src/components/SyncStatus.tsx b/frontend/src/components/SyncStatus.tsx new file mode 100644 index 0000000..b79fb7b --- /dev/null +++ b/frontend/src/components/SyncStatus.tsx @@ -0,0 +1,49 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Database, Loader2, Pause, Play } from "lucide-react"; +import { api } from "../lib/api"; +import { formatViews } from "../lib/format"; + +export default function SyncStatus() { + const qc = useQueryClient(); + const { data } = useQuery({ + queryKey: ["sync-status"], + queryFn: api.status, + refetchInterval: 30_000, + staleTime: 25_000, + }); + + const toggle = useMutation({ + mutationFn: () => (data?.paused ? api.resumeSync() : api.pauseSync()), + onSuccess: () => qc.invalidateQueries({ queryKey: ["sync-status"] }), + }); + + if (!data) return null; + + return ( +
+ + {formatViews(data.videos_total)} videos + · + {data.paused ? ( + paused + ) : data.channels_backfilling > 0 ? ( + + + {data.channels_backfilling} syncing + + ) : ( + all synced + )} + {data.is_admin && ( + + )} +
+ ); +} diff --git a/frontend/src/components/Toaster.tsx b/frontend/src/components/Toaster.tsx new file mode 100644 index 0000000..58a653a --- /dev/null +++ b/frontend/src/components/Toaster.tsx @@ -0,0 +1,29 @@ +import { useSyncExternalStore } from "react"; +import { dismiss, getToasts, subscribe } from "../lib/toast"; + +export default function Toaster() { + const toasts = useSyncExternalStore(subscribe, getToasts, getToasts); + return ( +
+ {toasts.map((t) => ( +
+ {t.message} + {t.action && ( + + )} +
+ ))} +
+ ); +} diff --git a/frontend/src/components/VideoCard.tsx b/frontend/src/components/VideoCard.tsx new file mode 100644 index 0000000..c364fdf --- /dev/null +++ b/frontend/src/components/VideoCard.tsx @@ -0,0 +1,204 @@ +import { Bookmark, Check, Eye, EyeOff, ListFilter } from "lucide-react"; +import clsx from "clsx"; +import type { Video } from "../lib/api"; +import { formatDuration, formatViews, relativeTime } from "../lib/format"; + +function Actions({ + video, + onState, + onChannelFilter, +}: { + video: Video; + onState: (id: string, status: string) => void; + onChannelFilter?: (channelId: string, channelName: string) => void; +}) { + const act = (status: string) => (e: React.MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + onState(video.id, video.status === status ? "new" : status); + }; + return ( +
+ + + + {onChannelFilter && ( + + )} +
+ ); +} + +function Thumb({ video, className }: { video: Video; className?: string }) { + return ( + video.status === "new" && undefined} + className={clsx( + "block relative rounded-xl overflow-hidden bg-surface border border-border shadow-md group-hover:shadow-2xl transition-shadow", + className + )} + > + {video.thumbnail_url ? ( + + ) : ( +
+ )} + {video.duration_seconds != null && ( + + {formatDuration(video.duration_seconds)} + + )} + {video.live_status === "was_live" && ( + + stream + + )} + {video.status === "saved" && ( + + + + )} + + ); +} + +export default function VideoCard({ + video, + view, + onState, + onChannelFilter, +}: { + video: Video; + view: "grid" | "list"; + onState: (id: string, status: string) => void; + onChannelFilter?: (channelId: string, channelName: string) => void; +}) { + const watched = video.status === "watched"; + const meta = ( + <> + {video.view_count != null && <>{formatViews(video.view_count)} views · } + {relativeTime(video.published_at)} + + ); + const title = ( + + {video.title} + + ); + + if (view === "list") { + return ( + + ); + } + + return ( + + ); +} diff --git a/frontend/src/index.css b/frontend/src/index.css new file mode 100644 index 0000000..1d4ce62 --- /dev/null +++ b/frontend/src/index.css @@ -0,0 +1,129 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +:root { + --font-scale: 1.06; +} + +/* Make native controls (date picker, scrollbars) follow the theme. */ +html[data-theme="dark"] { + color-scheme: dark; +} +html[data-theme="light"] { + color-scheme: light; +} + +html { + font-size: calc(16px * var(--font-scale)); + background: var(--bg); +} + +body { + margin: 0; + background: var(--bg); + color: var(--fg); +} + +/* ===== Color schemes (accent + neutrals), each with dark + light ===== */ + +/* Midnight — deep navy/slate with blue accent (default) */ +html[data-scheme="midnight"][data-theme="dark"] { + --bg: #0b1020; + --surface: #121a2e; + --card: #161f38; + --border: #243049; + --fg: #e6e9f0; + --muted: #97a3c0; + --accent: #6d8cff; + --accent-fg: #0b1020; +} +html[data-scheme="midnight"][data-theme="light"] { + --bg: #f4f6fc; + --surface: #ffffff; + --card: #ffffff; + --border: #dde3f0; + --fg: #1a2238; + --muted: #5b6685; + --accent: #3b5bdb; + --accent-fg: #ffffff; +} + +/* Forest — dark slate with teal/green accent */ +html[data-scheme="forest"][data-theme="dark"] { + --bg: #0a1512; + --surface: #0f1f1a; + --card: #12241e; + --border: #1f3a30; + --fg: #e6f0ec; + --muted: #90b1a4; + --accent: #2dd4bf; + --accent-fg: #04110d; +} +html[data-scheme="forest"][data-theme="light"] { + --bg: #f1f7f4; + --surface: #ffffff; + --card: #ffffff; + --border: #d6e7e0; + --fg: #102a22; + --muted: #4d6b60; + --accent: #0d9488; + --accent-fg: #ffffff; +} + +/* Slate — neutral grey with warm orange accent (muted in dark) */ +html[data-scheme="slate"][data-theme="dark"] { + --bg: #15171c; + --surface: #1b1e25; + --card: #1f232b; + --border: #2c313b; + --fg: #e7e9ee; + --muted: #9aa1ad; + --accent: #e8913c; + --accent-fg: #1a1206; +} +html[data-scheme="slate"][data-theme="light"] { + --bg: #f6f7f9; + --surface: #ffffff; + --card: #ffffff; + --border: #e3e6eb; + --fg: #1c1f26; + --muted: #5d636e; + --accent: #d97706; + --accent-fg: #ffffff; +} + +/* YouTube — near-black with red accent */ +html[data-scheme="youtube"][data-theme="dark"] { + --bg: #0f0f0f; + --surface: #181818; + --card: #1f1f1f; + --border: #303030; + --fg: #f1f1f1; + --muted: #aaaaaa; + --accent: #ff3b46; + --accent-fg: #ffffff; +} +html[data-scheme="youtube"][data-theme="light"] { + --bg: #ffffff; + --surface: #f9f9f9; + --card: #ffffff; + --border: #e5e5e5; + --fg: #0f0f0f; + --muted: #606060; + --accent: #ff0033; + --accent-fg: #ffffff; +} + +/* Thin, theme-aware scrollbars */ +* { + scrollbar-color: var(--border) transparent; +} +*::-webkit-scrollbar { + width: 10px; + height: 10px; +} +*::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 8px; +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts new file mode 100644 index 0000000..3fec3fe --- /dev/null +++ b/frontend/src/lib/api.ts @@ -0,0 +1,128 @@ +export interface Me { + id: number; + email: string; + display_name: string | null; + avatar_url: string | null; + role: string; + preferences: Record; +} + +export interface Tag { + id: number; + name: string; + color: string | null; + category: string; + system: boolean; + channel_count: number; +} + +export interface Video { + id: string; + title: string | null; + channel_id: string; + channel_title: string | null; + channel_thumbnail: string | null; + channel_url: string; + published_at: string | null; + thumbnail_url: string | null; + duration_seconds: number | null; + view_count: number | null; + is_short: boolean; + live_status: string; + status: string; + watch_url: string; +} + +export interface FeedResponse { + items: Video[]; + has_more: boolean; + offset: number; + limit: number; +} + +export interface FeedFilters { + tags: number[]; + tagMode: "or" | "and"; + q: string; + sort: string; + includeNormal: boolean; + includeShorts: boolean; + includeLive: boolean; + show: string; + channelId?: string; + channelName?: string; + maxAgeDays?: number; + minDuration?: number; + maxDuration?: number; + dateFrom?: string; + dateTo?: string; +} + +class HttpError extends Error { + status: number; + constructor(status: number) { + super(`HTTP ${status}`); + this.status = status; + } +} + +async function req(url: string, opts: RequestInit = {}): Promise { + const r = await fetch(url, { + credentials: "include", + headers: { "Content-Type": "application/json" }, + ...opts, + }); + if (!r.ok) throw new HttpError(r.status); + return r.status === 204 ? null : r.json(); +} + +function feedQuery(f: FeedFilters, offset: number, limit: number): string { + const p = new URLSearchParams(); + f.tags.forEach((t) => p.append("tags", String(t))); + p.set("tag_mode", f.tagMode); + if (f.q) p.set("q", f.q); + p.set("sort", f.sort); + p.set("show_normal", String(f.includeNormal)); + p.set("include_shorts", String(f.includeShorts)); + p.set("include_live", String(f.includeLive)); + p.set("show", f.show); + if (f.channelId) p.set("channel_id", f.channelId); + if (f.maxAgeDays) p.set("max_age_days", String(f.maxAgeDays)); + if (f.dateFrom) p.set("published_after", f.dateFrom); + if (f.dateTo) p.set("published_before", f.dateTo); + if (f.minDuration != null) p.set("min_duration", String(f.minDuration)); + if (f.maxDuration != null) p.set("max_duration", String(f.maxDuration)); + p.set("offset", String(offset)); + p.set("limit", String(limit)); + return p.toString(); +} + +export interface SyncStatus { + subscriptions: number; + channels_total: number; + channels_backfilling: number; + videos_total: number; + pending_enrich: number; + quota_used_today: number; + quota_remaining_today: number; + paused: boolean; + is_admin: boolean; +} + +export const api = { + me: (): Promise => req("/api/me"), + tags: (): Promise => req("/api/tags"), + status: (): Promise => req("/api/sync/status"), + feed: (f: FeedFilters, offset: number, limit: number): Promise => + req(`/api/feed?${feedQuery(f, offset, limit)}`), + feedCount: (f: FeedFilters): Promise<{ count: number }> => + req(`/api/feed/count?${feedQuery(f, 0, 0)}`), + setState: (id: string, status: string) => + req(`/api/videos/${id}/state`, { method: "POST", body: JSON.stringify({ status }) }), + pauseSync: () => req("/api/sync/pause", { method: "POST" }), + resumeSync: () => req("/api/sync/resume", { method: "POST" }), + savePrefs: (p: Record) => + req("/api/me/preferences", { method: "PUT", body: JSON.stringify(p) }), +}; + +export { HttpError }; diff --git a/frontend/src/lib/format.ts b/frontend/src/lib/format.ts new file mode 100644 index 0000000..4f7b89f --- /dev/null +++ b/frontend/src/lib/format.ts @@ -0,0 +1,43 @@ +export function relativeTime(iso: string | null): string { + if (!iso) return ""; + const then = new Date(iso).getTime(); + const secs = Math.max(0, (Date.now() - then) / 1000); + const units: [number, string][] = [ + [60, "s"], + [3600, "min"], + [86400, "h"], + [604800, "d"], + [2592000, "wk"], + [31536000, "mo"], + ]; + if (secs < 60) return "just now"; + for (let i = 0; i < units.length - 1; i++) { + const [, label] = units[i]; + const next = units[i + 1][0]; + if (secs < next) { + const v = Math.floor(secs / units[i][0]); + return `${v} ${label} ago`; + } + } + const years = Math.floor(secs / 31536000); + if (years >= 1) return `${years} yr ago`; + const months = Math.floor(secs / 2592000); + return `${months} mo ago`; +} + +export function formatDuration(sec: number | null): string { + if (sec == null) return ""; + const h = Math.floor(sec / 3600); + const m = Math.floor((sec % 3600) / 60); + const s = Math.floor(sec % 60); + const pad = (n: number) => String(n).padStart(2, "0"); + return h > 0 ? `${h}:${pad(m)}:${pad(s)}` : `${m}:${pad(s)}`; +} + +export function formatViews(n: number | null): string { + if (n == null) return ""; + if (n >= 1e9) return `${(n / 1e9).toFixed(1).replace(/\.0$/, "")}B`; + if (n >= 1e6) return `${(n / 1e6).toFixed(1).replace(/\.0$/, "")}M`; + if (n >= 1e3) return `${(n / 1e3).toFixed(1).replace(/\.0$/, "")}K`; + return String(n); +} diff --git a/frontend/src/lib/theme.ts b/frontend/src/lib/theme.ts new file mode 100644 index 0000000..436cad4 --- /dev/null +++ b/frontend/src/lib/theme.ts @@ -0,0 +1,42 @@ +export type Mode = "dark" | "light"; +export type Scheme = "midnight" | "forest" | "slate" | "youtube"; + +export const SCHEMES: { id: Scheme; name: string; swatch: string }[] = [ + { id: "midnight", name: "Midnight", swatch: "#6d8cff" }, + { id: "forest", name: "Forest", swatch: "#2dd4bf" }, + { id: "slate", name: "Slate", swatch: "#e8913c" }, + { id: "youtube", name: "YouTube", swatch: "#ff3b46" }, +]; + +export interface ThemePrefs { + mode: Mode; + scheme: Scheme; + fontScale: number; +} + +export const DEFAULT_THEME: ThemePrefs = { + mode: "dark", + scheme: "midnight", + fontScale: 1.06, +}; + +export function applyTheme(t: ThemePrefs): void { + const el = document.documentElement; + el.dataset.theme = t.mode; + el.dataset.scheme = t.scheme; + el.style.setProperty("--font-scale", String(t.fontScale)); +} + +const KEY = "subfeed.theme"; + +export function loadLocalTheme(): ThemePrefs { + try { + return { ...DEFAULT_THEME, ...JSON.parse(localStorage.getItem(KEY) || "{}") }; + } catch { + return DEFAULT_THEME; + } +} + +export function saveLocalTheme(t: ThemePrefs): void { + localStorage.setItem(KEY, JSON.stringify(t)); +} diff --git a/frontend/src/lib/toast.ts b/frontend/src/lib/toast.ts new file mode 100644 index 0000000..8a35638 --- /dev/null +++ b/frontend/src/lib/toast.ts @@ -0,0 +1,41 @@ +export interface ToastAction { + label: string; + onClick: () => void; +} +export interface ToastItem { + id: number; + message: string; + action?: ToastAction; +} + +let items: ToastItem[] = []; +let listeners: Array<() => void> = []; +let counter = 1; + +function emit() { + listeners.forEach((l) => l()); +} + +export function toast(message: string, action?: ToastAction): number { + const id = counter++; + items = [...items, { id, message, action }]; + emit(); + setTimeout(() => dismiss(id), 9000); + return id; +} + +export function dismiss(id: number) { + items = items.filter((i) => i.id !== id); + emit(); +} + +export function getToasts(): ToastItem[] { + return items; +} + +export function subscribe(listener: () => void): () => void { + listeners.push(listener); + return () => { + listeners = listeners.filter((l) => l !== listener); + }; +} diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx new file mode 100644 index 0000000..17aa397 --- /dev/null +++ b/frontend/src/main.tsx @@ -0,0 +1,17 @@ +import React from "react"; +import { createRoot } from "react-dom/client"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import App from "./App"; +import "./index.css"; + +const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false, staleTime: 30_000, refetchOnWindowFocus: false } }, +}); + +createRoot(document.getElementById("root")!).render( + + + + + +); diff --git a/frontend/tailwind.config.js b/frontend/tailwind.config.js new file mode 100644 index 0000000..7f8b621 --- /dev/null +++ b/frontend/tailwind.config.js @@ -0,0 +1,22 @@ +/** @type {import('tailwindcss').Config} */ +export default { + content: ["./index.html", "./src/**/*.{ts,tsx}"], + theme: { + extend: { + colors: { + bg: "var(--bg)", + surface: "var(--surface)", + card: "var(--card)", + border: "var(--border)", + fg: "var(--fg)", + muted: "var(--muted)", + accent: "var(--accent)", + "accent-fg": "var(--accent-fg)", + }, + fontFamily: { + sans: ["Inter", "system-ui", "-apple-system", "Segoe UI", "Roboto", "sans-serif"], + }, + }, + }, + plugins: [], +}; diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json new file mode 100644 index 0000000..79a2287 --- /dev/null +++ b/frontend/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": false, + "noUnusedParameters": false + }, + "include": ["src"] +} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts new file mode 100644 index 0000000..77d2204 --- /dev/null +++ b/frontend/vite.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +// During `vite dev` (host with Node), proxy API calls to the backend container. +const proxy = { + "/api": "http://localhost:8080", + "/auth": "http://localhost:8080", + "/healthz": "http://localhost:8080", +}; + +export default defineConfig({ + plugins: [react()], + server: { port: 5173, proxy }, + build: { outDir: "dist" }, +});