# -----------------------------------------------------------------------------
# This Robots.txt file is for http://www.os2site.com/
# see http://www.robotstxt.org/wc/exclusion.html
# -----------------------------------------------------------------------------

User-agent: *
Disallow: /

User-agent: SmartViper
Crawl-delay: 50
SITEMAP: http://www.os2site.com/sitemap.xml
Disallow: /sw/incoming/
Disallow: /sw/new/
Disallow: /sw/unknown/
Disallow: /cgi-bin/
Allow: /sitemap.xml$
Allow: /*.html$
Allow: /*.htm$
Allow: /*.txt$
Allow: /*.link$
Disallow: /*.*$

User-agent: Teoma
Crawl-delay: 50
SITEMAP: http://www.os2site.com/sitemap.xml
Disallow: /sw/incoming/
Disallow: /sw/new/
Disallow: /sw/unknown/
Disallow: /cgi-bin/
Allow: /sitemap.xml$
Allow: /*.html$
Allow: /*.htm$
Allow: /*.txt$
Allow: /*.link$
Disallow: /*.*$

User-agent: ia_archiver
Disallow: /cgi-bin/
Disallow: /dec/
Disallow: /sw/

User-agent: archive.org_bot
Disallow: /cgi-bin/
Disallow: /dec/
Disallow: /sw/

User-agent: msnbot
Crawl-delay: 100
Disallow: /sw/incoming/
Disallow: /sw/new/
Disallow: /sw/unknown/
Disallow: /cgi-bin/
Disallow: /*.mpg$
Disallow: /*.zip$
Disallow: /*.lzh$
Disallow: /*.jar$
Disallow: /*.exe$
Disallow: /*.wpi$
Disallow: /*.pdf$
Disallow: /*.arc$
Allow: /sitemap.xml$
Allow: /*.html$
Allow: /*.htm$
Allow: /*.link$
Disallow: /*.*$

User-agent: Slurp
Crawl-delay: 200
Disallow: /sw/incoming/
Disallow: /sw/new/
Disallow: /sw/unknown/
Disallow: /cgi-bin/
Allow: /sitemap.xml$
Allow: /*.html$
Allow: /*.htm$
Allow: /*.txt$
Allow: /*.link$
Disallow: /*.*$

User-agent: Googlebot
Disallow: /sw/incoming/
Disallow: /sw/new/
Disallow: /sw/unknown/
Disallow: /cgi-bin/
Allow: /sitemap.xml$
Allow: /*.html$
Allow: /*.htm$
Allow: /*.txt$
Allow: /*.link$
Allow: /*.*.link$
Disallow: /*.*$

User-agent: Mediapartners-Google
Disallow: /cgi-bin/
Allow: /*.html$
Allow: /*.htm$
Allow: /*.txt$
Allow: /*.link$
Allow: /*.*.link$
Disallow: /*.*$

User-agent: Ocelli
Crawl-delay: 50
Disallow: /sw/incoming/
Disallow: /sw/new/
Disallow: /sw/unknown/
Disallow: /cgi-bin/
Allow: /sitemap.xml$
Allow: /*.html$
Allow: /*.htm$
Allow: /*.txt$
Allow: /*.link$
Allow: /*.*.link$
Disallow: /*.*$

User-agent: http://www.almaden.ibm.com/cs/crawler
Disallow: /cgi-bin/
Disallow: /sw/incoming/
Disallow: /sw/new/

User-agent: Adsbot-Google
Disallow: /cgi-bin/