# All robots will spider the domain

User-agent: *
Disallow: /cgi-bin/
Disallow: /assets/
Disallow: /assets_c/
Disallow: /templates_c/
Disallow: /mt-static/
Disallow: /directory/
Disallow: /tube/
Disallow: /free/
Disallow: /tgp/
Disallow: /tgp2/