# ===================================== # Global rules for all user agents # ===================================== User-agent: * Disallow: /*.wav Disallow: /*.mp3 Disallow: /*captcha.png* Disallow: /*?tl= Disallow: /*?sl= Clean-param: ts /* # ===================================== # Blocklists for known aggressive / unwanted bots # (Does NOT affect SEO) # ===================================== # Advertising crawlers User-agent: ias_crawler Disallow: / User-agent: CCBot Disallow: / # Chinese crawlers known for heavy crawling User-agent: Sogou web spider Disallow: / User-agent: Sogou inst spider Disallow: / User-agent: Sogou Disallow: / # SEO tools (not helpful for your site) User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: AhrefsBot Disallow: / User-agent: dotbot Disallow: / # Other high-load or low-value crawlers User-agent: MJ12bot Disallow: / User-agent: proximic Disallow: / User-agent: PetalBot Disallow: / User-agent: Bytespider Disallow: / # Common scrapers User-agent: DataForSeoBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: ZoominfoBot Disallow: / User-agent: MauiBot Disallow: / User-agent: Mail.RU_Bot Disallow: / User-agent: trendemon.com Disallow: / User-agent: Seekport Crawler Disallow: / User-agent: 360Spider Disallow: /