# YellowKey Qatar - robots.txt # https://www.yellowkey.qa User-agent: * # Allow property listings (SEO critical) Allow: /properties/ Allow: /property/ Allow: /listing/ Allow: /guides/ Allow: /blog/ # Allow access to public pages Allow: / Allow: /about/ Allow: /contact/ Allow: /services/ Allow: /property_marketing/ Allow: /property_management/ Allow: /realtor_services/ Allow: /workman_services/ Allow: /whatsapp_group/ Allow: /careers/ Allow: /faq/ Allow: /testimonials/ Allow: /privacy-policy/ Allow: /map/ Allow: /help/ Allow: /en/ Allow: /ar/ # Allow crawlers to access static files for rendering Allow: /static/ Allow: /media/ # Disallow admin and user-specific dashboard areas Disallow: /admin/ Disallow: /accounts/ Disallow: /account/ Disallow: /realtor/ Disallow: /clients/ Disallow: /workman/ Disallow: /staff/ Disallow: /dashboard/ Disallow: /social/ # Disallow API endpoints (internal use only) Disallow: /api/ # Disallow specific actions Disallow: */join/ Disallow: */add/ Disallow: */submit/ Disallow: */update/ Disallow: */delete/ Disallow: */approve/ Disallow: */reject/ Disallow: */edit/ # Block duplicate content query strings (allow page= for pagination) Disallow: /*?*sort= Disallow: /*?*filter= Disallow: /*?*search= Disallow: /*?*status= # Specific rules for AI crawlers (ChatGPT, Perplexity, Claude, Gemini, etc.) User-agent: GPTBot Allow: / Allow: /properties/ Allow: /property/ Allow: /guides/ Allow: /blog/ Allow: /about/ Allow: /contact/ Allow: /services/ Allow: /faq/ Allow: /testimonials/ Allow: /property_marketing/ Allow: /property_management/ Allow: /realtor_services/ Allow: /workman_services/ Allow: /careers/ Allow: /help/ User-agent: ChatGPT-User Allow: / Allow: /properties/ Allow: /property/ Allow: /guides/ Allow: /blog/ Allow: /about/ Allow: /contact/ Allow: /services/ Allow: /faq/ Allow: /testimonials/ Allow: /property_marketing/ Allow: /property_management/ Allow: /realtor_services/ Allow: /workman_services/ Allow: /careers/ Allow: /help/ User-agent: PerplexityBot Allow: / Allow: /properties/ Allow: /property/ Allow: /guides/ Allow: /blog/ Allow: /about/ Allow: /contact/ Allow: /services/ Allow: /faq/ Allow: /testimonials/ Allow: /property_marketing/ Allow: /property_management/ Allow: /realtor_services/ Allow: /workman_services/ Allow: /careers/ Allow: /help/ User-agent: Claude-Web Allow: / Allow: /properties/ Allow: /property/ Allow: /guides/ Allow: /blog/ Allow: /about/ Allow: /contact/ Allow: /services/ Allow: /faq/ Allow: /testimonials/ Allow: /property_marketing/ Allow: /property_management/ Allow: /realtor_services/ Allow: /workman_services/ Allow: /careers/ Allow: /help/ User-agent: anthropic-ai Allow: / Allow: /properties/ Allow: /property/ Allow: /guides/ Allow: /blog/ Allow: /about/ Allow: /contact/ Allow: /services/ Allow: /faq/ Allow: /testimonials/ Allow: /property_marketing/ Allow: /property_management/ Allow: /realtor_services/ Allow: /workman_services/ Allow: /careers/ Allow: /help/ User-agent: Google-Extended Allow: / Allow: /properties/ Allow: /property/ Allow: /guides/ Allow: /blog/ Allow: /about/ Allow: /contact/ Allow: /services/ Allow: /faq/ Allow: /testimonials/ Allow: /property_marketing/ Allow: /property_management/ Allow: /realtor_services/ Allow: /workman_services/ Allow: /careers/ Allow: /help/ User-agent: Googlebot Allow: / Allow: /static/ Allow: /media/ Crawl-delay: 0 User-agent: Bingbot Allow: / Allow: /static/ Allow: /media/ # Sitemaps Sitemap: https://www.yellowkey.qa/sitemap.xml # Host directive Host: https://www.yellowkey.qa # Crawl-delay for polite crawling (prevents server overload) Crawl-delay: 1