# robots.txt file for webtrees # # Copyright (C) Greg Roach 2014 # # This file needs to be placed in the domain root folder, # such as "www.example.com/robots.txt". It will not work in a # subfolder, such as "www.example.com/webtrees/robots.txt" # If you need to move it, then remember to adjust the paths as well. # e.g. "Allow: /index.php" becomes "Allow: /webtrees/index.php". # # Well-behaved robots understand "Allow" directives. # Google (plus some others?) understands the "$" pattern match User-Agent: bingbot # Bing User-Agent: Googlebot # Google User-Agent: ia_archiver # Internet Archive "wayback" machine User-Agent: msnbot # Microsoft User-Agent: robozilla # DMOZ User-Agent: slurp # Yahoo! User-Agent: teoma # Ask Jeeves User-Agent: yandex # Popular Russian search engine User-Agent: baiduspider # Popular Chinese search engine Allow: /ged/family.php Allow: /ged/index.php Allow: /ged/indilist.php Allow: /ged/individual.php Allow: /ged/mediafirewall.php Allow: /ged/mediaviewer.php Allow: /ged/module.php?mod=sitemap Allow: /ged/note.php Allow: /ged/repo.php Allow: /ged/site-offline.php Allow: /ged/site-unavailable.php Allow: /ged/source.php Allow: /ged/$ Disallow: /ged/ # Disallow all other robots User-Agent: * Disallow: / # Put your sitemap here: Sitemap: http://www.markdionne.com/ged/module.php?mod=sitemap&mod_action=generate&file=sitemap.xml