# ----------------------------------------------------------------------- # Primary robots.txt for WordPress + large listing site # Optimized for SEO-safe crawling, image discovery, and reduced junk crawl # ----------------------------------------------------------------------- # ========================= # 1) Block AI / unwanted bots # ========================= User-agent: Amazonbot Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: ClaudeBot Disallow: / User-agent: GPTBot Disallow: / User-agent: PetalBot Disallow: / User-agent: PerplexityBot Disallow: / User-agent: CCBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: DotBot Disallow: / User-agent: Yandex Disallow: / User-agent: YaK Disallow: / User-agent: EtaoSpider Disallow: / User-agent: viberbot Disallow: / User-agent: uptimerobot Disallow: / # ========================= # 2) Google AI policy # ========================= # Allow Google Search, but block Google AI training/use User-agent: Google-Extended Disallow: / # ========================= # 3) Apple search crawler # ========================= # Allow Apple search indexing, but rate control should be handled at server/WAF level User-agent: Applebot Allow: / # ========================= # 4) Google image crawler # ========================= # Restrict image crawling to the folders/files you actually want surfaced User-agent: Googlebot-Image Disallow: / Allow: /favicon.ico Allow: /wp-content/uploads/ Allow: /images/agents/ Allow: /images/udx/ Allow: /images/listings/ # ========================= # 5) Main search crawlers # ========================= User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / # ========================= # 6) General rules for all crawlers # ========================= User-agent: * Allow: / Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # Query cleanup / crawl waste reduction Disallow: /*?attachment_id= Disallow: /*?*q= # Faceted / sort-style URLs Disallow: /*/hga Disallow: /*/hga*?*q= Disallow: /*/lga Disallow: /*/lga*?*q= Disallow: /*/oga Disallow: /*/oga*?*q= Disallow: /*/nga Disallow: /*/nga*?*q= # Error / junk patterns Disallow: /*/A Disallow: /*/Array Disallow: /*?*afghanistan* Disallow: /*?*cheapfifa23coins* # Important crawlable pages Allow: /favicon.ico Allow: /en/d/ Allow: /en/d/* Allow: /fr/d/ Allow: /fr/d/* Allow: /nl/d/ Allow: /nl/d/* Allow: /es/d/ Allow: /es/d/* # Account: 1000433 (orig) Disallow: /en/s/ Allow: /en/s/for-sale Allow: /en/s/for-sale/condos-apartments Allow: /en/s/for-sale/condos-apartments/pool Allow: /en/s/for-sale/condos-apartments/water-front Allow: /en/s/for-sale/condos-apartments/beach-oceanfront Allow: /en/s/for-sale/condos-apartments/bedrooms-1 Allow: /en/s/for-sale/condos-apartments/bedrooms-2 Allow: /en/s/for-sale/condos-apartments/bedrooms-3 Allow: /en/s/for-sale/condos-apartments/ambergris-cays Allow: /en/s/for-sale/condos-apartments/dellis-cay Allow: /en/s/for-sale/condos-apartments/east-caicos Allow: /en/s/for-sale/condos-apartments/grand-turk Allow: /en/s/for-sale/condos-apartments/middle-caicos Allow: /en/s/for-sale/condos-apartments/north-caicos Allow: /en/s/for-sale/condos-apartments/pine-cay Allow: /en/s/for-sale/condos-apartments/providenciales Allow: /en/s/for-sale/condos-apartments/salt-cay Allow: /en/s/for-sale/condos-apartments/south-caicos Allow: /en/s/for-sale/condos-apartments/water-cay Allow: /en/s/for-sale/condos-apartments/west-caicos Allow: /en/s/for-sale/single-family-homes Allow: /en/s/for-sale/single-family-homes/pool Allow: /en/s/for-sale/single-family-homes/water-front Allow: /en/s/for-sale/single-family-homes/beach-oceanfront Allow: /en/s/for-sale/single-family-homes/bedrooms-2 Allow: /en/s/for-sale/single-family-homes/bedrooms-3 Allow: /en/s/for-sale/single-family-homes/bedrooms-4 Allow: /en/s/for-sale/single-family-homes/bedrooms-5 Allow: /en/s/for-sale/single-family-homes/ambergris-cays Allow: /en/s/for-sale/single-family-homes/dellis-cay Allow: /en/s/for-sale/single-family-homes/east-caicos Allow: /en/s/for-sale/single-family-homes/grand-turk Allow: /en/s/for-sale/single-family-homes/middle-caicos Allow: /en/s/for-sale/single-family-homes/north-caicos Allow: /en/s/for-sale/single-family-homes/pine-cay Allow: /en/s/for-sale/single-family-homes/providenciales Allow: /en/s/for-sale/single-family-homes/salt-cay Allow: /en/s/for-sale/single-family-homes/south-caicos Allow: /en/s/for-sale/single-family-homes/water-cay Allow: /en/s/for-sale/single-family-homes/west-caicos Allow: /en/s/for-sale/vacant-land Allow: /en/s/for-sale/vacant-land/water-front Allow: /en/s/for-rent Allow: /en/s/for-rent/condos-apartments Allow: /en/s/for-rent/condos-apartments/pool Allow: /en/s/for-rent/condos-apartments/water-front Allow: /en/s/for-rent/condos-apartments/beach-oceanfront Allow: /en/s/for-rent/condos-apartments/bedrooms-1 Allow: /en/s/for-rent/condos-apartments/bedrooms-2 Allow: /en/s/for-rent/condos-apartments/bedrooms-3 Allow: /en/s/for-rent/condos-apartments/ambergris-cays Allow: /en/s/for-rent/condos-apartments/dellis-cay Allow: /en/s/for-rent/condos-apartments/east-caicos Allow: /en/s/for-rent/condos-apartments/grand-turk Allow: /en/s/for-rent/condos-apartments/middle-caicos Allow: /en/s/for-rent/condos-apartments/north-caicos Allow: /en/s/for-rent/condos-apartments/pine-cay Allow: /en/s/for-rent/condos-apartments/providenciales Allow: /en/s/for-rent/condos-apartments/salt-cay Allow: /en/s/for-rent/condos-apartments/south-caicos Allow: /en/s/for-rent/condos-apartments/water-cay Allow: /en/s/for-rent/condos-apartments/west-caicos Allow: /en/s/for-rent/single-family-homes Allow: /en/s/commercial Allow: /en/s/commercial/business Allow: /en/s/commercial/commercial-mixed-use Allow: /en/s/commercial/commercial-building-office Allow: /en/s/commercial/vacant-land-agricultural Allow: /en/s/commercial/vacant-land-commercial Allow: /en/s/commercial/water-front Allow: /en/s/commercial/ambergris-cays Allow: /en/s/commercial/dellis-cay Allow: /en/s/commercial/east-caicos Allow: /en/s/commercial/grand-turk Allow: /en/s/commercial/middle-caicos Allow: /en/s/commercial/north-caicos Allow: /en/s/commercial/pine-cay Allow: /en/s/commercial/providenciales Allow: /en/s/commercial/salt-cay Allow: /en/s/commercial/south-caicos Allow: /en/s/commercial/water-cay Allow: /en/s/commercial/west-caicos