# Jobbase robots.txt
#
# Search engines and link previews: welcome, on every public page.
#
# AI systems and their operators: do not use anything on this site to
# train, fine-tune or evaluate a model, or to build a dataset. This is a
# reservation of text and data mining rights under Article 4(3) of EU
# Directive 2019/790, also sent as the tdm-reservation header and in
# /.well-known/tdmrep.json. An assistant fetching one public page because
# a person asked it to may read it for that person and link to it; it may
# not keep or republish it.
#
# Terms: https://remotelink-africa-timmy-mathebulas-projects.vercel.app/legal/terms#automated-access

User-agent: *
Allow: /
Allow: /listings/
Allow: /search
Allow: /remote-jobs
Disallow: /apply/
Disallow: /api/
Disallow: /dashboard
Disallow: /account
Disallow: /admin
Disallow: /checkout
Disallow: /referrals
Disallow: /saved
Disallow: /verify
Disallow: /reset
Disallow: /reauth
Disallow: /r/
Disallow: /sponsor/
Disallow: /alerts/
Disallow: /*?cursor=

# Google Jobs and Bing read listings through these two.
User-agent: Googlebot
User-agent: Bingbot
Allow: /
Allow: /listings/
Allow: /search
Allow: /remote-jobs
Disallow: /apply/
Disallow: /api/
Disallow: /admin
Disallow: /r/
Disallow: /sponsor/
Disallow: /alerts/

# AI training crawlers: nothing here, at any path.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Omgili
User-agent: Timpibot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: PanguBot
User-agent: img2dataset
User-agent: Webzio-Extended
User-agent: VelenPublicWebCrawler
User-agent: ICC-Crawler
User-agent: Kangaroo Bot
Disallow: /

# SEO data miners.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: BLEXBot
Disallow: /

Sitemap: https://remotelink-africa-timmy-mathebulas-projects.vercel.app/sitemap.xml
Sitemap: https://remotelink-africa-timmy-mathebulas-projects.vercel.app/sitemaps/listings/sitemap/0.xml
Host: https://remotelink-africa-timmy-mathebulas-projects.vercel.app