# Vanguard Yachts - robots.txt # Generated for comprehensive SEO optimization # Allow all bots to crawl the entire site User-agent: * Allow: / # Allow specific search engines for better indexing User-agent: Googlebot Allow: / Disallow: /api/ Disallow: /*.json$ Crawl-delay: 0 User-agent: Bingbot Allow: / Disallow: /api/ Crawl-delay: 0 User-agent: Slurp Allow: / Disallow: /api/ Crawl-delay: 1 # Block bad bots User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: DotBot Disallow: / User-agent: MJ12bot Disallow: / # Sitemap location Sitemap: https://vanguard.yachts/sitemap.xml # Specify crawl-delay for non-specific bots (in seconds) Request-rate: 1/1s