chore(feed): Phase 2 #2 backend cleanup — dead return, dup helpers, shared const
- feed.py _filtered_query: drop the dead 2nd return element (status_expr was
used only internally for WHERE filters; all 3 callers discarded it as _status).
Now returns (query, rank_expr); fixed the stale docstring.
- youtube/client.py: extract _iter_playlist_items() — iter_my_playlist_video_ids
and iter_playlist_items_with_ids were near-identical playlistItems paging loops
(jscpd [173-187]≈[267-281]); both now map over the shared generator.
- sync/videos.py: extract _apply_video_batch() with an on_missing callback —
enrich_pending and refresh_live shared the fetch-map-apply skeleton, differing
only in the query and how they retire rows YouTube omits.
- models.py: add LIVE_OR_UPCOMING = ("live","upcoming"); replace the 4 duplicated
copies (feed.HIDDEN_LIVE, search._LIVE_HIDDEN, videos.py + channels.py inline).
Behavior-neutral. ruff clean on touched files, localdev boots, feed/worker healthy.
This commit is contained in:
parent
477056bfa8
commit
505f0e5650
6 changed files with 75 additions and 71 deletions
|
|
@ -2,6 +2,7 @@
|
|||
uploads playlist, and enrichment via videos.list (duration, stats, category, Shorts
|
||||
and livestream classification)."""
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
|
|
@ -10,7 +11,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, sysconfig
|
||||
from app.models import Channel, Video
|
||||
from app.models import LIVE_OR_UPCOMING, Channel, Video
|
||||
from app.titles import normalize_title
|
||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||
from app.youtube.rss import fetch_channel_feed
|
||||
|
|
@ -260,6 +261,37 @@ def apply_video_details(video: Video, item: dict) -> None:
|
|||
# is_short is decided separately by the youtube.com/shorts probe (run_shorts_classification).
|
||||
|
||||
|
||||
def _apply_video_batch(
|
||||
db: Session,
|
||||
yt: YouTubeClient,
|
||||
videos: list[Video],
|
||||
on_missing: Callable[[Video], None],
|
||||
) -> int:
|
||||
"""Fetch videos.list details for the given rows, apply them (stamping enriched_at), and
|
||||
return how many were applied. Rows YouTube omits (deleted/private) are handled by the
|
||||
caller-supplied `on_missing` so each caller decides how to retire them."""
|
||||
if not videos:
|
||||
return 0
|
||||
items = {it["id"]: it for it in yt.get_videos([v.id for v in videos])}
|
||||
now = _now()
|
||||
applied = 0
|
||||
for video in videos:
|
||||
item = items.get(video.id)
|
||||
if item is None:
|
||||
on_missing(video)
|
||||
continue
|
||||
apply_video_details(video, item)
|
||||
video.enriched_at = now
|
||||
applied += 1
|
||||
db.commit()
|
||||
return applied
|
||||
|
||||
|
||||
def _mark_enriched(video: Video) -> None:
|
||||
# Unavailable (deleted/private): stamp so we don't keep retrying.
|
||||
video.enriched_at = _now()
|
||||
|
||||
|
||||
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
|
||||
limit = limit or sysconfig.get_int(db, "enrich_batch_size")
|
||||
videos = (
|
||||
|
|
@ -267,22 +299,7 @@ def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) ->
|
|||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not videos:
|
||||
return 0
|
||||
items = {it["id"]: it for it in yt.get_videos([v.id for v in videos])}
|
||||
now = _now()
|
||||
enriched = 0
|
||||
for video in videos:
|
||||
item = items.get(video.id)
|
||||
if item is None:
|
||||
# Unavailable (deleted/private): stamp so we don't keep retrying.
|
||||
video.enriched_at = now
|
||||
continue
|
||||
apply_video_details(video, item)
|
||||
video.enriched_at = now
|
||||
enriched += 1
|
||||
db.commit()
|
||||
return enriched
|
||||
return _apply_video_batch(db, yt, list(videos), _mark_enriched)
|
||||
|
||||
|
||||
def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
|
||||
|
|
@ -300,7 +317,7 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
|
|||
select(Video)
|
||||
.where(
|
||||
or_(
|
||||
Video.live_status.in_(("live", "upcoming")),
|
||||
Video.live_status.in_(LIVE_OR_UPCOMING),
|
||||
and_(
|
||||
Video.live_status == "was_live",
|
||||
Video.duration_seconds.is_(None),
|
||||
|
|
@ -312,23 +329,13 @@ def refresh_live(db: Session, yt: YouTubeClient, limit: int | None = None) -> in
|
|||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not videos:
|
||||
return 0
|
||||
items = {it["id"]: it for it in yt.get_videos([v.id for v in videos])}
|
||||
now = _now()
|
||||
updated = 0
|
||||
for video in videos:
|
||||
item = items.get(video.id)
|
||||
if item is None:
|
||||
# The broadcast vanished (deleted/private). Stop treating it as live so it
|
||||
# doesn't get refreshed forever; it just becomes an ordinary (dead) row.
|
||||
video.live_status = "none"
|
||||
continue
|
||||
apply_video_details(video, item)
|
||||
video.enriched_at = now
|
||||
updated += 1
|
||||
db.commit()
|
||||
return updated
|
||||
|
||||
def _retire(video: Video) -> None:
|
||||
# The broadcast vanished (deleted/private). Stop treating it as live so it
|
||||
# doesn't get refreshed forever; it just becomes an ordinary (dead) row.
|
||||
video.live_status = "none"
|
||||
|
||||
return _apply_video_batch(db, yt, list(videos), _retire)
|
||||
|
||||
|
||||
def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue