User-agent: * # list folders robots are not allowed to index Disallow: /assets/ Disallow: /navimages/ Disallow: /dogs/ Disallow: /dogtalk/ Disallow: /thumbnails/ Disallow: /xybase/ # # list specific files robots are not allowed to index # Disallow: /family.htm Disallow: /404error.html # # End of robots.txt file