Sitemap: http://www.marki.fr/sitemap.html

# please do not crawl these pages
User-agent: *
Disallow: /cgi-bin/ 
Disallow: /admin/
Disallow: /stats/

# seconds to wait between successive requests = 30
Crawl-delay: 30

# maximum rate is one page every 20 seconds
Request-rate: 1/20 

# these agents is not allowed to crawl pages
User-Agent: MJ12bot
Disallow:   /

User-Agent: Yanga
Disallow:   /

User-Agent: Yanga WorldSearch Bot
Disallow:   /

User-Agent: YodaoBot
Disallow:   /