User-agent: twiceler
Crawl-delay: 120
Disallow: /bin/attach/
Disallow: /bin/changes/
Disallow: /bin/edit/
Disallow: /bin/geturl/
Disallow: /bin/installpasswd/
Disallow: /bin/mailnotify/
Disallow: /bin/manage/
Disallow: /bin/oops/
Disallow: /bin/passwd/
Disallow: /bin/preview/
Disallow: /bin/rdiff/
Disallow: /bin/rdiffauth/
Disallow: /bin/register/
Disallow: /bin/rename/
Disallow: /bin/save/
Disallow: /bin/savemulti/
Disallow: /bin/search/
Disallow: /bin/setlib.cfg/
Disallow: /bin/statistics/
Disallow: /bin/testenv/
Disallow: /bin/configure/
Disallow: /bin/upload/
#Disallow: /bin/viewauth/
Disallow: /bin/viewfile/
Disallow: /list/
Disallow: /~calice/
Disallow: /install/
Disallow: /bin/login/

# From logs this seems set up from the Computer Center
# logs: gsa-crawler (Enterprise; MID-02703; robert.turner rhul.ac.uk)

User-agent: gsa-crawler
Disallow: /bin/attach/
Disallow: /bin/changes/
Disallow: /bin/edit/
Disallow: /bin/geturl/
Disallow: /bin/installpasswd/
Disallow: /bin/mailnotify/
Disallow: /bin/manage/
Disallow: /bin/oops/
Disallow: /bin/passwd/
Disallow: /bin/preview/
Disallow: /bin/rdiff/
Disallow: /bin/rdiffauth/
Disallow: /bin/register/
Disallow: /bin/rename/
Disallow: /bin/save/
Disallow: /bin/savemulti/
Disallow: /bin/search/
Disallow: /bin/setlib.cfg/
Disallow: /bin/statistics/
Disallow: /bin/testenv/
Disallow: /bin/configure/
Disallow: /bin/upload/
#Disallow: /bin/viewauth/
Disallow: /bin/viewfile/
Disallow: /list/
Disallow: /~calice/
Disallow: /install/
Disallow: /bin/login/
Disallow: /bin/view/TWiki/
Disallow: /twiki/bin/edit/

User-agent: *
Disallow: /bin/attach/
Disallow: /bin/changes/
Disallow: /bin/edit/
Disallow: /bin/geturl/
Disallow: /bin/installpasswd/
Disallow: /bin/mailnotify/
Disallow: /bin/manage/
Disallow: /bin/oops/
Disallow: /bin/passwd/
Disallow: /bin/preview/
Disallow: /bin/rdiff/
Disallow: /bin/rdiffauth/
Disallow: /bin/register/
Disallow: /bin/rename/
Disallow: /bin/save/
Disallow: /bin/savemulti/
Disallow: /bin/search/
Disallow: /bin/setlib.cfg
Disallow: /bin/statistics/
Disallow: /bin/testenv/
Disallow: /bin/configure
Disallow: /bin/upload/
#Disallow: /bin/viewauth/
Disallow: /bin/viewfile
Disallow: /list/
Disallow: /~calice/
Disallow: /install/
Disallow: /bin/login/

#User-agent: *
#Disallow: /