# All robots will spider the domain User-agent: * Disallow: # Disallow directory /files/ User-agent: * Disallow: /files/ # Disallow directory /scripts/ User-agent: * Disallow: /scripts/ # Disallow directory /source/ User-agent: * Disallow: /source/ # Disallow directory /stats/ User-agent: * Disallow: /stats/ # Disallow directory /subdomains/ User-agent: * Disallow: /subdomains/ # Disallow directory /templates/ User-agent: * Disallow: /templates/