User-agent: * Disallow: /cgi-bin/ # Allow only specific directories User-agent: Teoma Disallow: / Allow: /public_html