31 lines
588 B
Plaintext
31 lines
588 B
Plaintext
User-agent: *
|
|
Allow: /$
|
|
Allow: /index.php$
|
|
Allow: /sitemap*
|
|
Allow: /rss*
|
|
Allow: /*.html
|
|
Allow: /news/*
|
|
Allow: /films/*
|
|
Allow: /video/*
|
|
Allow: /music/*
|
|
Allow: /games/*
|
|
Allow: /soft/*
|
|
Allow: /library/*
|
|
Allow: /wallpapers/*
|
|
Allow: /forum/$
|
|
Allow: /forum/forum_
|
|
Allow: /forum/topic_
|
|
Disallow: /
|
|
Crawl-delay: 0.5
|
|
Host: files-sib.net
|
|
Sitemap: http://files-sib.net/sitemap.xml
|
|
|
|
User-agent: Adsbot-Google
|
|
User-agent: Googlebot-Image
|
|
User-agent: Googlebot-Mobile
|
|
User-agent: Mediapartners-Google
|
|
User-agent: msnbot
|
|
User-agent: msnbot-media
|
|
User-agent: Slurp
|
|
User-agent: Yahoo-MMCrawler
|
|
Disallow: / |