User-agent: *
Crawl-delay: 10
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Googlebot
Allow: / 
User-agent: Adsbot-Google
Disallow: 
User-agent: Googlebot-Image
Disallow: 
User-agent: Googlebot-Mobile
Allow: / 
User-agent: MSNBot
Disallow: 
User-agent: bingbot
Crawl-delay: 300
Allow: /
User-agent: Slurp
Allow: /
User-Agent: Yahoo! Slurp
Allow: /

User-agent: rogerbot
Crawl-delay: 10

User-agent: AhrefsBot
Disallow: / 

#Yandex Spider
User-agent: Yandex
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

#Block Common Crawl.org
User-agent: CCBot
Disallow: /

#Block OpenLinkProfiler
User-agent: spbot
Disallow: /
 
# Accessibility Audit
User-agent: PowerMapper
Allow: /
 
# Directories
User-agent: *
Disallow: /
Disallow: /cgi-bin/
Disallow: /ads/
Disallow: /assets/
Disallow: /cgi-bin/
Disallow: /phone/
Disallow: /scripts/


User-agent: www.deadlinkchecker.com
Allow: / 

User-agent: *
Disallow: /silver/*.jsp

User-agent: *
Disallow: /custom/*.jsp

User-agent: *
Disallow: /api/*.jsp

User-agent: *
Disallow: /theme/*/*.jsp

Sitemap: https://shop.forbergsmith.com/sitemap.xml