############################### # # sample robots.txt file for this website # # addresses all robots by using wild card * # User-agent: * Crawl-Delay: 60 # list folders robots are not allowed to index Disallow: /CSS/ Disallow: /customer/ Disallow: /WEB-INF/ Disallow: /META-INF/ Disallow: /_templates/ Disallow: /onlinestore/ Disallow: /onlinestore/payment/ Disallow: /includes/ Disallow: /resources/ Disallow: /stocklists/stockrequests/ Disallow: /stocklists/shippingcalc/ # # list specific files robots are not allowed to index # End of robots.txt file # ###############################