# --------------------------------------------------------------------------- # ALL CRAWLERS — default rules # --------------------------------------------------------------------------- User-agent: * Allow: / # WordPress system files Disallow: /wp-admin/ Disallow: /wp-login.php Disallow: /wp-signup.php Disallow: /wp-cron.php Disallow: /wp-json/ Disallow: /xmlrpc.php Disallow: /readme.html Disallow: /license.txt # Private / authenticated pages Disallow: /account/ Disallow: /my-account/ Disallow: /dashboard/ Disallow: /message-chat/ Disallow: /private/ Disallow: /checkout/ Disallow: /cart/ Disallow: /register/ # Feed / comment URLs Disallow: /feed/ Disallow: /comments/feed/ Disallow: /trackback/ # Junk / duplicate parameter URLs (tracking & session IDs only) Disallow: /*?s= Disallow: /*?search= Disallow: /*?ref= Disallow: /*?utm_ Disallow: /*?fbclid= Disallow: /*?gclid= Disallow: /*?sessionid= Disallow: /*?PHPSESSID= Disallow: /*?attachment_id= # --------------------------------------------------------------------------- # GOOGLEBOT — explicit rendering permissions # --------------------------------------------------------------------------- User-agent: Googlebot Allow: / Allow: /wp-content/uploads/ Allow: /wp-content/themes/ Allow: /wp-includes/*.js Allow: /wp-includes/*.css Allow: /*.css$ Allow: /*.js$ # --------------------------------------------------------------------------- # BINGBOT # --------------------------------------------------------------------------- User-agent: Bingbot Allow: / # --------------------------------------------------------------------------- # SEO TOOL BOTS — allowed with crawl delay # --------------------------------------------------------------------------- User-agent: AhrefsBot Crawl-delay: 5 Allow: / User-agent: SemrushBot Crawl-delay: 5 Allow: / User-agent: MJ12bot Crawl-delay: 10 Allow: / User-agent: DotBot Crawl-delay: 10 Allow: / # --------------------------------------------------------------------------- # AI SEARCH BOTS — allowed (surface listings in AI answers) # --------------------------------------------------------------------------- User-agent: OAI-SearchBot Allow: / User-agent: PerplexityBot Allow: / # --------------------------------------------------------------------------- # AI TRAINING & SCRAPER BOTS — blocked # --------------------------------------------------------------------------- User-agent: GPTBot Disallow: / User-agent: Google-Extended Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: meta-externalagent Disallow: / User-agent: Amazonbot Disallow: / User-agent: Bytespider Disallow: / User-agent: CCBot Disallow: / User-agent: Diffbot Disallow: / User-agent: omgili Disallow: / User-agent: BLEXBot Disallow: / User-agent: Baiduspider Disallow: / User-agent: YandexBot Disallow: / User-agent: PetalBot Disallow: / User-agent: Sogou web spider Disallow: / User-agent: 360Spider Disallow: / User-agent: CloudflareBrowserRenderingCrawler Disallow: / User-agent: Scrapy Disallow: / User-agent: Python-urllib Disallow: / User-agent: Java Disallow: / User-agent: wget Disallow: / # --------------------------------------------------------------------------- Sitemap: https://roomforrent.rent/sitemap.xml