# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /
#
# Also be aware that this robots.txt is only served for charanga.com and the
# principle partner sites. We have a dynamically generated robots.txt for
# assets1.charanga.com, assets2.charanga.com etc to prevent us getting duplicate
# content penalties, but it might interfere with e.g. redbot.org testing

User-Agent: *
Disallow: /hnyp0t

# redbot is very useful for testing, but will only work if we
# specifically allow it
User-agent: RED
Allow: /

# Don't index any pdfs in the resource_library; there are brochures we do want
# indexed, hence the /resource_library path
User-Agent: *
Disallow: /resource_library/*.pdf$
#Noindex: /resource_library/*.pdf$

# And really, we should just stop any crawling of the resource_library
User-Agent: *
Disallow: /resource_library
#Noindex: /resource_library

# Stop Bingpreview from invalidating links
User-Agent: BingPreview
Disallow: /pupil_reports
#Noindex: /pupil_reports

# The AdsBot-Google has gone mental and is following links that
# deliver converting users to us. It seems to particularly like
# the login page, so  BAN THIS FILTH
User-agent: AdsBot-Google
Disallow: /user/login

# Some SEO spider that's all over us
User-agent: MJ12bot
Disallow: /

# Google, Yahoo, MSN et al all seem to be trying to index this (non-existant) url. God knows why
# Tell them to sod off
User-Agent: *
Disallow: /~magnolia/

# It was public, now it's login-only. We need this here so we can purge
# cached copies with Google Webmaster Tools
Disallow: /music_service_admin/letters/pupil_information_printable

# These have to be fully qualified URLs, and because this robots.txt is shared
# let's add them all here...
# Ours
Sitemap: https://charanga.com/site/sitemap_index.xml
Sitemap: https://www.charanga.cz/site/sitemap_index.xml
Sitemap: https://www.charanga.dk/site/sitemap_index.xml
Sitemap: https://www.charanga.com.au/site/sitemap_index.xml
Sitemap: https://www.charanga.co.za/site/sitemap_index.xml
Sitemap: https://www.charanga.hk/site/sitemap_index.xml
Sitemap: https://www.charanga.in/site/sitemap_index.xml
Sitemap: https://www.charanga.vn/site/sitemap_index.xml

# Partners
Sitemap: https://www.banesmusiconline.co.uk/site/sitemapindex.xml
Sitemap: https://www.bradfordmusiconline.co.uk/site/sitemap_index.xml
Sitemap: https://www.essexmusichub.org.uk/site/sitemap_index.xml
Sitemap: https://www.lancashiremusichub.co.uk/site/sitemap_index.xml
Sitemap: https://www.norfolkmusichub.org.uk/site/sitemap_index.xml
Sitemap: https://www.richmondmusictrust.org.uk/site/sitemap_index.xml
Sitemap: https://www.wakefieldmusicservices.org/site/sitemap_index.xml

# stop dumb robots submitting this with no values
Disallow: /school_quotes
#Noindex: /school_quotes

Disallow: /vip_session_accounts
#Noindex: /vip_session_accounts

Disallow: /quotes
#Noindex: /school_quotes

Disallow: /musicalschoolfreetrial
Disallow: /assets/record_usage

Disallow: /admin
#Noindex:  /admin

Disallow: /music_service_admin
#Noindex: /music_service_admin

Disallow: /training_events
#Noindex: /training_events