# Siftlode is mostly a private, login-gated app; only the public landing and legal
# pages are worth indexing. Allow crawling but keep bots out of the API surface.
# Share links (/watch/) are unguessable capability URLs: kept out of search indexes,
# but link-preview crawlers are allowed below so shared links unfurl in chat apps.
User-agent: *
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /watch/

# Link-preview / unfurl bots — allowed to fetch /watch/ so shared links render a rich
# card (title/channel/thumbnail). They don't build a search index, so this doesn't
# expose share links to search engines.
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: Slackbot-LinkExpanding
User-agent: Slackbot
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: LinkedInBot
User-agent: Pinterest
User-agent: redditbot
Allow: /
Disallow: /api/
Disallow: /auth/