# =====================================
# Google (SEO critical)
# =====================================
User-agent: Googlebot
Allow: /
Disallow: /catalogsearch/result/
Disallow: /catalogsearch/advanced/
Disallow: /catalogsearch/advanced
User-agent: Googlebot-Image
Allow: /
User-agent: AdsBot-Google
Allow: /
# =====================================
# Bing
# =====================================
User-agent: bingbot
Allow: /
Disallow: /catalogsearch/result/
Disallow: /catalogsearch/advanced/
Disallow: /catalogsearch/advanced
# =====================================
# Block aggressive / AI crawlers
# =====================================
User-agent: Sogou
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-SearchBot
Disallow: /
User-agent: CCBot
Disallow: /
# =====================================
# Default rules (Magento safe)
# =====================================
User-agent: *
Disallow: /checkout/
Disallow: /customer/
Disallow: /cart/
Disallow: /admin/
Disallow: /search/
Disallow: /catalogsearch/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /index.php/
Disallow: /media/catalog/product/cache/
Disallow: /var/
Allow: /*.css$
Allow: /*.js$
Sitemap: https://www.hlcwholesale.com/media/sitemap.xml
Sitemap: https://www.hlcwholesale.com/media/sitemap.xml