feat: M2 (part 1) — subscription import, YouTube client, quota guard
- Channel/Subscription/Video/ApiQuotaUsage models + migration 0002 - Synchronous YouTube Data API client with OAuth token refresh and per-call quota accounting (API key preferred for public reads when configured) - Subscription import: upsert channels + subscription links, prune unsubscribed, fetch channel details (uploads playlist, topics, stats, handle, country) - Central quota guard tracking units per US-Pacific day against a daily budget - POST /api/sync/subscriptions and GET /api/sync/status endpoints
This commit is contained in:
npeter83
parent
3a774cf7d6
commit
24b6e0026d
10 changed files with 603 additions and 3 deletions
127
backend/alembic/versions/0002_ingest_core.py
Normal file
127
backend/alembic/versions/0002_ingest_core.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
"""ingest core: channels, subscriptions, videos, api_quota_usage
|
||||
|
||||
Revision ID: 0002_ingest_core
|
||||
Revises: 0001_initial
|
||||
Create Date: 2026-06-11
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = "0002_ingest_core"
|
||||
down_revision: Union[str, None] = "0001_initial"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"channels",
|
||||
sa.Column("id", sa.String(length=32), primary_key=True),
|
||||
sa.Column("title", sa.String(length=255), nullable=True),
|
||||
sa.Column("handle", sa.String(length=255), nullable=True),
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
sa.Column("thumbnail_url", sa.String(length=1024), nullable=True),
|
||||
sa.Column("uploads_playlist_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("subscriber_count", sa.BigInteger(), nullable=True),
|
||||
sa.Column("video_count", sa.BigInteger(), nullable=True),
|
||||
sa.Column("topic_categories", sa.JSON(), nullable=True),
|
||||
sa.Column("default_language", sa.String(length=16), nullable=True),
|
||||
sa.Column("country", sa.String(length=8), nullable=True),
|
||||
sa.Column("details_synced_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("recent_synced_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_rss_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("backfill_cursor", sa.String(length=128), nullable=True),
|
||||
sa.Column("backfill_done", sa.Boolean(), nullable=False, server_default="false"),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
nullable=False,
|
||||
),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"subscriptions",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"channel_id",
|
||||
sa.String(length=32),
|
||||
sa.ForeignKey("channels.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("yt_subscription_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("priority", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("hidden", sa.Boolean(), nullable=False, server_default="false"),
|
||||
sa.Column("subscribed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.UniqueConstraint("user_id", "channel_id", name="uq_user_channel"),
|
||||
)
|
||||
op.create_index("ix_subscriptions_user_id", "subscriptions", ["user_id"])
|
||||
op.create_index("ix_subscriptions_channel_id", "subscriptions", ["channel_id"])
|
||||
|
||||
op.create_table(
|
||||
"videos",
|
||||
sa.Column("id", sa.String(length=16), primary_key=True),
|
||||
sa.Column(
|
||||
"channel_id",
|
||||
sa.String(length=32),
|
||||
sa.ForeignKey("channels.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("title", sa.Text(), nullable=True),
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
sa.Column("published_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("thumbnail_url", sa.String(length=1024), nullable=True),
|
||||
sa.Column("duration_seconds", sa.Integer(), nullable=True),
|
||||
sa.Column("view_count", sa.BigInteger(), nullable=True),
|
||||
sa.Column("like_count", sa.BigInteger(), nullable=True),
|
||||
sa.Column("category_id", sa.Integer(), nullable=True),
|
||||
sa.Column("topic_categories", sa.JSON(), nullable=True),
|
||||
sa.Column("default_language", sa.String(length=16), nullable=True),
|
||||
sa.Column("detected_language", sa.String(length=16), nullable=True),
|
||||
sa.Column("is_short", sa.Boolean(), nullable=False, server_default="false"),
|
||||
sa.Column("live_status", sa.String(length=16), nullable=False, server_default="none"),
|
||||
sa.Column("enriched_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
nullable=False,
|
||||
),
|
||||
)
|
||||
op.create_index("ix_videos_channel_id", "videos", ["channel_id"])
|
||||
op.create_index("ix_videos_published_at", "videos", ["published_at"])
|
||||
op.create_index("ix_videos_is_short", "videos", ["is_short"])
|
||||
op.create_index("ix_videos_live_status", "videos", ["live_status"])
|
||||
|
||||
op.create_table(
|
||||
"api_quota_usage",
|
||||
sa.Column("day", sa.Date(), primary_key=True),
|
||||
sa.Column("units_used", sa.Integer(), nullable=False, server_default="0"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("api_quota_usage")
|
||||
op.drop_index("ix_videos_live_status", table_name="videos")
|
||||
op.drop_index("ix_videos_is_short", table_name="videos")
|
||||
op.drop_index("ix_videos_published_at", table_name="videos")
|
||||
op.drop_index("ix_videos_channel_id", table_name="videos")
|
||||
op.drop_table("videos")
|
||||
op.drop_index("ix_subscriptions_channel_id", table_name="subscriptions")
|
||||
op.drop_index("ix_subscriptions_user_id", table_name="subscriptions")
|
||||
op.drop_table("subscriptions")
|
||||
op.drop_table("channels")
|
||||
Loading…
Add table
Add a link
Reference in a new issue