User-agent: *
Crawl-delay: 30

# Google Image Crawler Setup
#User-agent: Googlebot-Image
#Disallow:

# Crawlers Setup
User-agent: *
#Disallow: /*/media/catalog/product/cache/

#Images:
#Disallow: /*/media/catalog/product/cache/

# Directories
Disallow: /*/404/
Disallow: /*/app/
Disallow: /*/cgi-bin/
Disallow: /*/downloader/
Disallow: /*/errors/
Disallow: /*/includes/
#Disallow: /*/js/
#Disallow: /*/lib/
Disallow: /*/magento/
#Disallow: /*/media/
Disallow: /*/pkginfo/
Disallow: /*/report/
Disallow: /*/scripts/
Disallow: /*/shell/
#Disallow: /*/skin/
Disallow: /*/stats/
Disallow: /*/var/

# Paths (clean URLs)
Disallow: /*/index.php/
Disallow: /*/catalog/product_compare/
Disallow: /*/catalog/category/view/
Disallow: /*/catalog/product/view/
#Enable search to allow brand pages?
#Disallow: /*/catalogsearch/
#Disallow: /*/checkout/
Disallow: /*/control/
Disallow: /*/contacts/
Disallow: /*/customer/
Disallow: /*/customize/
Disallow: /*/newsletter/
Disallow: /*/poll/
#Disallow: /*/review/
Disallow: /*/sendfriend/
Disallow: /*/tag/
Disallow: /*/wishlist/
Disallow: /*/catalog/product/gallery/

# Filters
Disallow: /*?dir*
Disallow: /*?limit*
Disallow: /*?order=*
Disallow: /*?mode=*
Disallow: /*/l/*

# Files
Disallow: /*/.gitignore
Disallow: /*/api.php
Disallow: /*/cron.php
Disallow: /*/mage$
Disallow: /*/cron.sh
Disallow: /*/error_log
Disallow: /*/get.php
Disallow: /*/install.php
Disallow: /*/affiliate.php
Disallow: /*/firegento.php
Disallow: /*/LICENSE.html
Disallow: /*/LICENSE.txt
Disallow: /*/LICENSE_AFL.txt
Disallow: /*/README.txt
Disallow: /*/RELEASE_NOTES.txt
Disallow: /*/STATUS.txt 

# Paths (no clean URLs)
#Disallow: /*.js$
#Disallow: /*.css$
Disallow: /*.php$
Disallow: /*?SID=

# # Do not allow indexing files and folders that are required during development: CVS, SVN directory and dump files
Disallow: /CVS
Disallow: /*.Svn$
Disallow: /*.Idea$
Disallow: /*.Sql$
Disallow: /*.Tgz$

## Other Crawlers
User-Agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Baiduspider
User-agent: baiduspider
User-agent: Baiduspider+ 
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Disallow: /