User-agent: Go-http-client
User-agent: survey
User-agent: link
Disallow: /


User-agent: *
Crawl-delay: 10