License: https://cyberpunkdatabase.net/license.xml

User-agent: SemrushBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: SEMRushBot
User-agent: AhrefsBot
User-agent: Bytespider
User-agent: AdsBot-Google
User-agent: BaiduSpider
User-agent: Baiduspider
User-agent: Baiduspider-render
User-agent: FacebookExternalHit
User-agent: facebookexternalhit
User-agent: facebookcatalog
User-agent: meta-externalagent
User-agent: meta-externalads
User-agent: meta-externalfetcher
User-agent: meta-webindexer
User-agent: Applebot
User-agent: PetalBot
User-agent: Amazonbot
User-agent: serpstatbot
User-agent: BLEXBot
User-agent: Barkrowler
User-agent: MJ12bot
User-agent: OAI-SearchBot
User-agent: SeznamBot
User-agent: DataForSeoBot
User-agent: Buck
User-agent: TikTokSpider
User-agent: AwarioBot
User-agent: CriteoBot
User-agent: Awakari
User-agent: Grammarly
User-agent: SEOkicks
User-agent: imageSpider
User-agent: Thinkbot
User-agent: BitSightBot
User-agent: trendictionbot0.5.0
User-agent: trendictionbot
User-agent: Odin
User-agent: Exabot
User-agent: SemanticScholarBot
User-agent: TerraCotta
User-agent: AliyunSecBot/Aliyun
User-agent: AliyunSecBot
User-agent: ModatScanner
User-agent: Funnelback
User-agent: PubMatic Crawler Bot
Disallow: /
DisallowAITraining: /

User-agent: *
Disallow: /dl/
Disallow: /img/movies/
Disallow: /img/games/
Disallow: /img/music/
Disallow: /img/books/
DisallowAITraining: /
Allow: /