User-agent: *
Disallow: /search/				# Block search results for site search or learning navigation with facets chosen (Temporarily block the whole directory)
Disallow: /search/search*			# Block site search results 
Disallow: /search/go*				# Block site search results 
Disallow: /search/popular/			# Block popular site search results 
Disallow: /business/search		# Block search results for Where to Buy or Find a Service Provider
Disallow: /*.swf$					# Block files ending .swf (Flash media)
Disallow: /*.doc$					# Block files ending .doc (Word document)
Disallow: /*.dwg$					# Block files ending .dwg
Disallow: /sitecore/content/		# Don't allow access to Sitecore directory (Include meta robots noindex,nofolow tag)
Disallow: /thd-connect                        # Block /thd-connect from being indexed
Disallow: /api/                                       # Block /api from being indexed

#Block crawlers from accessing facet pages
Disallow: *?frameColor
Disallow: *?exteriorTrim
Disallow: *?exteriorTrimColor
Disallow: *?frameColorExt
Disallow: *?frameStain
Disallow: *?hardware
Disallow: *?grilleStyle
Disallow: *?hardwareOptions
Disallow: *?side
Disallow: *?exteriorSash
Disallow: *?reside
Disallow: *?widIn
Disallow: *?hgtIn
Disallow: *?grilleLightsWide
Disallow: *?grilleLightsHigh
Disallow: *?grilleSpacing
Disallow: *?glass

#AWEB-353:  Allow SEMRush to be able to crawl our site
User-agent: SemrushBot
Allow: /
Disallow: /search/
Disallow: /api/
User-agent: SemrushBot-SI
Allow: /

#Block malicious bots
User-agent: AhrefsBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: SiteBot
Disallow: /
User-agent: MegaIndex
Disallow: /

# SITEMAPS
Sitemap: https://www.andersenwindows.com/sitemap.xml