# www.worldlii.org

# updated 2012/6/6 -JO
# Allow google access to /catalog


# updated 2011/5/26 - JO
# Allowed access to treaties, /int/cases still blocked

# Google adsbot ignores robots.txt unless specifically named!
User-agent: adsbot-google
Disallow: /

User-agent: Gromit
Disallow: /cgi-bin/
Disallow: /fcgi-bin/
Disallow: /do/
Disallow: /do2/
Disallow: /lawcite/
Disallow: /LawCite/
Disallow: /inCite/
Disallow: /incite/
Disallow: /uscode/

# Exclude all directories except the WorldLII Catalog /catalog
# and WorldLII help pages /worldlii
# (20030826): PTHC: allow robots to the /int databases
# (20041203): PTHC: exclude robots to the /int/cases/GENDND databases
# (20060315): PTHC: exclude robots to the /int/cases databases
User-agent: *
Disallow: /ad/
Disallow: /af/
Disallow: /ag/
Disallow: /ai/
Disallow: /al/
Disallow: /am/
Disallow: /ao/
Disallow: /apec/
Disallow: /aq/
Disallow: /ar/
Disallow: /as/
Disallow: /asean/
Disallow: /asia/
Disallow: /at/
Disallow: /au/
Disallow: /az/
Disallow: /ba/
Disallow: /bb/
Disallow: /bd/
Disallow: /be/
Disallow: /bf/
Disallow: /bg/
Disallow: /bh/
Disallow: /bi/
Disallow: /bj/
Disallow: /bk/
Disallow: /bm/
Disallow: /bn/
Disallow: /bo/
Disallow: /bq/
Disallow: /br/
Disallow: /bs/
Disallow: /bt/
Disallow: /bw/
Disallow: /by/
Disallow: /bz/
Disallow: /ca/
Disallow: /canlii/
Disallow: /cc/
Disallow: /cd/
Disallow: /cf/
Disallow: /cg/
Disallow: /fcgi-bin/
Disallow: /cgi-bin/
Disallow: /ch/
Disallow: /ci/
Disallow: /ck/
Disallow: /cl/
Disallow: /cm/
Disallow: /cn/
Disallow: /co/
Disallow: /cr/
Disallow: /cu/
Disallow: /cv/
Disallow: /cy/
Disallow: /cz/
Disallow: /de/
Disallow: /dj/
Disallow: /dk/
Disallow: /dm/
Disallow: /do/
Disallow: /do2/
Disallow: /droit/
Disallow: /dz/
Disallow: /ea/
Disallow: /ec/
Disallow: /editors/
Disallow: /ee/
Disallow: /eg/
Disallow: /eh/
Disallow: /er/
Disallow: /es/
Disallow: /et/
Disallow: /eu/
Disallow: /ew/
Disallow: /fcgi-bin/
Disallow: /fi/
Disallow: /fj/
Disallow: /fk/
Disallow: /fm/
Disallow: /fo/
Disallow: /form/
Disallow: /forms/
Disallow: /fr/
Disallow: /ga/
Disallow: /gb/
Disallow: /gd/
Disallow: /ge/
Disallow: /gf/
Disallow: /gg/
Disallow: /gh/
Disallow: /gi/
Disallow: /gl/
Disallow: /globalex/
Disallow: /gm/
Disallow: /gn/
Disallow: /gq/
Disallow: /gr/
Disallow: /gs/
Disallow: /gt/
Disallow: /gu/
Disallow: /gw/
Disallow: /gy/
Disallow: /hk/
Disallow: /hn/
Disallow: /hr/
Disallow: /ht/
Disallow: /hu/
Disallow: /id/
Disallow: /idlii/
Disallow: /ie/
Disallow: /IHLRes/
Disallow: /il/
Disallow: /im/
Disallow: /in/
Disallow: /inCite/
Disallow: /incite/
Disallow: /ind/
Disallow: /int/cases/
Disallow: /io/
Disallow: /iq/
Disallow: /ir/
Disallow: /is/
Disallow: /it/
Disallow: /je/
Disallow: /jm/
Disallow: /jo/
Disallow: /jp/
Disallow: /ke/
Disallow: /kg/
Disallow: /kh/
Disallow: /ki/
Disallow: /kl/
Disallow: /km/
Disallow: /kn/
Disallow: /kp/
Disallow: /kr/
Disallow: /kw/
Disallow: /ky/
Disallow: /kz/
Disallow: /la/
Disallow: /LawCite/
Disallow: /lawcite/
Disallow: /lb/
Disallow: /lc/
Disallow: /li/
Disallow: /links/
Disallow: /lk/
Disallow: /lr/
Disallow: /ls/
Disallow: /lt/
Disallow: /lu/
Disallow: /lv/
Disallow: /ly/
Disallow: /ma/
Disallow: /make_database.css/
Disallow: /mc/
Disallow: /md/
Disallow: /mg/
Disallow: /mh/
Disallow: /mk/
Disallow: /ml/
Disallow: /mm/
Disallow: /mn/
Disallow: /mo/
Disallow: /mp/
Disallow: /mr/
Disallow: /ms/
Disallow: /mt/
Disallow: /mu/
Disallow: /mv/
Disallow: /mw/
Disallow: /mx/
Disallow: /my/
Disallow: /mz/
Disallow: /na/
Disallow: /ne/
Disallow: /nf/
Disallow: /ng/
Disallow: /ni/
Disallow: /nie/
Disallow: /nl/
Disallow: /no/
Disallow: /np/
Disallow: /nr/
Disallow: /nu/
Disallow: /nz/
Disallow: /om/
Disallow: /pa/
Disallow: /paclii/
Disallow: /pe/
Disallow: /pf/
Disallow: /pg/
Disallow: /ph/
Disallow: /pk/
Disallow: /pl/
Disallow: /plates/
Disallow: /pn/
Disallow: /pr/
Disallow: /ps/
Disallow: /pt/
Disallow: /pub/
Disallow: /pw/
Disallow: /py/
Disallow: /qa/
Disallow: /ro/
Disallow: /rs/
Disallow: /ru/
Disallow: /rw/
Disallow: /sa/
Disallow: /sb/
Disallow: /sc/
Disallow: /scot/
Disallow: /sd/
Disallow: /se/
Disallow: /sg/
Disallow: /sh/
Disallow: /si/
Disallow: /sk/
Disallow: /sl/
Disallow: /sm/
Disallow: /sn/
Disallow: /so/
Disallow: /sr/
Disallow: /st/
Disallow: /styles/
Disallow: /sv/
Disallow: /sy/
Disallow: /sz/
Disallow: /tc/
Disallow: /td/
Disallow: /test/
Disallow: /tg/
Disallow: /th/
Disallow: /tj/
Disallow: /tk/
Disallow: /tm/
Disallow: /tn/
Disallow: /to/
Disallow: /tp/
Disallow: /tr/
Disallow: /tt/
Disallow: /tv/
Disallow: /tw/
Disallow: /tz/
Disallow: /ua/
Disallow: /ue/
Disallow: /ug/
Disallow: /uk/
Disallow: /us/
Disallow: /uscode/
Disallow: /uy/
Disallow: /uz/
Disallow: /va/
Disallow: /vc/
Disallow: /ve/
Disallow: /vg/
Disallow: /vi/
Disallow: /vm/
Disallow: /vn/
Disallow: /vu/
Disallow: /wales/
Disallow: /worldlii/
Disallow: /ws/
Disallow: /ye/
Disallow: /za/
Disallow: /zm/
Disallow: /zw/

# Riddler 
User-agent: Riddler
Disallow: /