# ==============================================================================
# Flyer International Travel Agency (Pvt) Ltd - Official Robots.txt
# Domain: https://flyerint.com/
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. Search Engine Crawlers (Googlebot, Bingbot, etc.)
# ------------------------------------------------------------------------------
User-agent: Googlebot
Allow: /
Disallow: /search
Disallow: /search/
Disallow: /*?s=
Disallow: /*?q=
Disallow: /*?search=
Disallow: /*search
Disallow: /*?*
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-json/
Disallow: /wp-json/*
Disallow: /?rest_route=
Disallow: /?rest_route=*
Disallow: /*?rest_route=
Disallow: /*?rest_route=*
Disallow: /*&rest_route=
Disallow: /*&rest_route=*
Disallow: /index.php?rest_route=
Disallow: /index.php?rest_route=*
Disallow: /*rest_route*
Disallow: /*rest_route=

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /
Disallow: /?rest_route=
Disallow: /*?rest_route=
Disallow: /*&rest_route=
Disallow: /*rest_route*

User-agent: *
Allow: /

# Prevent indexing of internal search queries & parameters
Disallow: /search
Disallow: /search/
Disallow: /*?s=
Disallow: /*?q=
Disallow: /*?search=
Disallow: /*search

# Prevent crawling of common URL parameters & query strings
Disallow: /*?*
Disallow: /*?
Disallow: /*?ref=
Disallow: /*?source=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?sessionid=
Disallow: /*?replytocom=

# Protect against CMS admin endpoints & vulnerabilities
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/

# Block WordPress JSON API & REST route parameters
Disallow: /wp-json/
Disallow: /wp-json/*
Disallow: /?rest_route=
Disallow: /?rest_route=*
Disallow: /*?rest_route=
Disallow: /*?rest_route=*
Disallow: /*&rest_route=
Disallow: /*&rest_route=*
Disallow: /index.php?rest_route=
Disallow: /index.php?rest_route=*
Disallow: /*rest_route*
Disallow: /*rest_route=
Disallow: /rest_route/
Disallow: /rest-api/

# Allow static CSS, JS, and media assets
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.ico$

# ------------------------------------------------------------------------------
# 2. Block Commercial SEO Crawlers (Ahrefs, Semrush, Moz, Majestic)
# ------------------------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: SemrushBot-SBM
Disallow: /

User-agent: dotbot
User-agent: rogerbot
Disallow: /

User-agent: MJ12bot
Disallow: /

# ------------------------------------------------------------------------------
# 3. Block Wayback Machine / Archive.org Crawler
# ------------------------------------------------------------------------------
User-agent: ia_archiver
Disallow: /

# ------------------------------------------------------------------------------
# 4. Block Known Aggressive Scrapers & Bad Bots
# ------------------------------------------------------------------------------
User-agent: BLEXBot
User-agent: PetalBot
User-agent: Bytespider
User-agent: MegaIndex
User-agent: MegaIndex.ru
User-agent: DataForSeoBot
User-agent: ZoominfoBot
User-agent: Baiduspider
User-agent: Sogou web spider
User-agent: Scrapy
User-agent: Spinn3r
User-agent: proximic
User-agent: SeznamBot
User-agent: Serpstatbot
Disallow: /

# ------------------------------------------------------------------------------
# XML Sitemap Directives
# ------------------------------------------------------------------------------
Sitemap: https://flyerint.com/sitemap.xml