User-agent: AhrefsBot
Crawl-Delay: 60

User-agent: *
Crawl-Delay: 10
Disallow: /cgimodel.py
Disallow: /cgimodel.py*
Disallow: /*.tsv
Disallow: /*.xml
Disallow: /admin
Disallow: /downloads
Disallow: /instances.gff
Disallow: /instances.fasta
Disallow: /instances.pir
Disallow: /instances.tsv
Disallow: /ajax
Disallow: /media
Disallow: /interactions/as_mitab/