# File Generated by CF Worker: Last Updated 12 April 2024
User-agent: *
Allow:    / # added 30.09.2019
# removed /ads.txt 30.09.2019
# removed following Disallows on 23/05/2020
# /jobs/british-islands*
# /jobs/england*
# /search-jobs?*
# removed following Disallows on 02.06.2020
# /*?q=* added 18.12.2019
Disallow: /*?order*  # added 02.06.2020
Disallow: /*&order*  # added 02.06.2020
Disallow: /*?salary*  # added 02.06.2020
Disallow: /*&salary*  # added 02.06.2020
Disallow: /*?perpage*  # added 02.06.2020
Disallow: /*&perpage*  # added 02.06.2020
Disallow: /*advertise-jobs? # added 22.10.2019
Disallow: /aff/*
Disallow: /alerts/check-existence
Disallow: /apply/ # added 18.12.2019
Disallow: /*?ar= # added 22.10.2019
Disallow: /candidate/
Disallow: /cgi-bin/ # updated 30.09.2019
Disallow: /candidate/savejob*
Disallow: /codetrial/
Allow:    /companies/employers # added 09.10.2019
Allow:    /companies/agencies # added 09.10.2019
Disallow: /companies/employers/by-industry/ # added 26.02.2020
Disallow: /companies/employers/by-county/ # added 26.02.2020
Disallow: /companies/agencies/by-industry/ # added 26.02.2020
Disallow: /companies/agencies/by-county/ # added 26.02.2020
Disallow: /companies/agencies? # added 22.10.2019
Disallow: /*cvdb7n= # added 22.10.2019
Disallow: /*cvdb14n= #added 18.12.2019
Disallow: /*?distance=* # added  18.12.2019
Disallow: /*&distance=*
Disallow: /*?posted=* # added  04.06.2020
Disallow: /*&posted=* # added  04.06.2020
Disallow: /domain-spellcheck
Disallow: /employer_trial/
Disallow: /entercode/
Disallow: /external/ # added 18.12.2019
Disallow: /*featured=* # added 30.09.2019
Allow:    /job/ # added 30.09.2019
Disallow: /job/*/*/sanitise
Disallow: /job/*/*/apply
Disallow: /job/*/*/apply/external
Disallow: /job/*/*/email
Allow:    /jobs/ # added 30.09.2019
Disallow: /jobs/*?id=* # added 30.09.2019
Disallow: /*/key/ # added 04/11/2019
Disallow: /login/
Disallow: /*?hl= # added 18.12.2019
Disallow: /r/
Disallow: /recruiter-profile/ # added 22.10.2019
Disallow: /refer/ # added 22.10.2019
Disallow: /register?* # added 22.10.2019
Disallow: /register?id=* # added 09.10.2019
Disallow: /register/*?id=* # added 09.10.2019
Disallow: /refer/ #added 09.10.2019
Disallow: /recruiter/
Disallow: /trial/
Disallow: /trial-code/
Disallow: /trialcode/
Disallow: /*?ts= # added 22.10.2019
Disallow: /*/*/page/*/ # added 04/11/2019
Disallow: /press/*
Disallow: /*preview_id= # added 22.10.2019
Disallow: /*search-cvs? # added 22.10.2019
Disallow: /*search?q= # added 22.10.2019
Disallow: /*sid= #added 30.09.2019
Disallow: /*?sj= # added 22.10.2019
Disallow: /*?Subject= # added 18.12.2019
Disallow: */?utm_source=* # added 09.10.2019 / Updated 18.12.2019
Disallow: /welcome-back?id=* # added 09.10.2019

# The following user-agents ignore the global (*) directives
# so we need to specify them individually.
# These disallow directives are non-rendering JS endpoints.
User-agent: Mediapartners-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google
Disallow: /alerts/check-existence
Disallow: /job/*/*/sanitise
Disallow: /collect$

# added 30.09.2019
User-agent: Applebot
User-agent: Bingbot
User-agent: Twitterbot
Allow:    /
# removed following Disallows on 23/05/2020
# /jobs/british-islands*
# /jobs/england*
# /search-jobs?*
# removed following Disallows on 02.06.2020
# /*?q=* added 18.12.2019
Disallow: /*?order* # added 02.06.2020
Disallow: /*&order*  # added 02.06.2020
Disallow: /*?salary*  # added 02.06.2020
Disallow: /*&salary*  # added 02.06.2020
Disallow: /*?perpage*  # added 02.06.2020
Disallow: /*&perpage*  # added 02.06.2020
Disallow: /ads.txt
Disallow: /*advertise-jobs? # added 22.10.2019
Disallow: /aff/*
Disallow: /apply/ # added 18.12.2019
Disallow: /*?ar= # added 22.10.2019
Disallow: /candidate/
Disallow: /cgi-bin/
Disallow: /candidate/savejob*
Disallow: /codetrial/
Disallow: /alerts/check-existence
Allow:    /companies/employers # added 09.10.2019
Allow:    /companies/agencies # added 09.10.2019
Disallow: /companies/employers/ # added 09.10.2019
Disallow: /companies/agencies/ # added 09.10.2019
Disallow: /companies/agencies? # added 22.10.2019
Disallow: /*cvdb7n= # added 22.10.2019
Disallow: /*cvdb14n= #added 18.12.2019
Disallow: /*?distance=* # added  18.12.2019
Disallow: /*&distance=*
Disallow: /*?posted=* # added  04.06.2020
Disallow: /*&posted=* # added  04.06.2020
Disallow: /domain-spellcheck
Disallow: /employer_trial/
Disallow: /entercode/
Disallow: /external/ # added 18.12.2019
Disallow: /*featured=* # added 04/11/2019
Disallow: /job/*/*/sanitise
Disallow: /job/*/*/apply
Disallow: /job/*/*/apply/external
Disallow: /job/*/*/email
Allow:    /jobs/
Disallow: /jobs/*?id=* # added 04/11/2019
Disallow: /*/key/ # added 04/11/2019
Disallow: /login/
Disallow: /*?hl= # added 18.12.2019
Disallow: /r/
Disallow: /recruiter-profile/ # added 22.10.2019
Disallow: /refer/ # added 22.10.2019
Disallow: /register?* # added 22.10.2019
Disallow: /register?id=* # added 09.10.2019
Disallow: /register/*?id=* # added 09.10.2019
Disallow: /refer/ # added 09.10.2019
Disallow: /recruiter/
Disallow: /trial/
Disallow: /trial-code/
Disallow: /trialcode/
Disallow: /*?ts= # added 22.10.2019
Disallow: /*/*/page/*/ # added 04/11/2019
Disallow: /press/*
Disallow: /*preview_id= # added 22.10.2019
Disallow: /search/ # added 25.11.19
Disallow: /*searchcvs? # added 22.10.2019
Disallow: /*search?q= # added 22.10.2019
Disallow: /*sid= # added 09.10.2019
Disallow: /*?sj= # added 22.10.2019
Disallow: /*?Subject= # added 18.12.2019
Disallow: */?utm_source=* # added 09.10.2019 / Updated 18.12.2019
Disallow: /welcome-back?id=* # added 09.10.2019

# Baidu Chinese search engine
User-agent: Baiduspider
Disallow: /

#Yandex RU search engine
User-agent: Yandex
Disallow: /

# Yahoo
User-agent: Slurp
Disallow: /

# Bulk Disallow added 04/11/2019
User-Agent: OmniExplorer_Bot
User-Agent: Verticrawlbot
User-agent: AddSearchBot
User-agent: admantx
User-agent: AhrefsBot
User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
User-agent: ChangeDetection
User-agent: dotbot
User-agent: Exabot
User-agent: GozaikBot
User-agent: grapeshot
User-Agent: Ichiro
User-agent: sogou spider
User-agent: WeSEE
User-agent: WijuBot
User-agent: XoviBot
User-agent: Cliqzbot
User-agent: PiplBot
User-agent: CCBot
Disallow: /

Sitemap: https://www.cv-library.co.uk/sitemaps/browse-jobs-index.xml
Sitemap: https://www.cv-library.co.uk/sitemaps/jobs-industry-by-location-index.xml
Sitemap: https://www.cv-library.co.uk/sitemaps/jobs-industry-index.xml
Sitemap: https://www.cv-library.co.uk/sitemaps/jobs-location-index.xml
Sitemap: https://www.cv-library.co.uk/sitemaps/jobs-parent-index.xml
Sitemap: https://www.cv-library.co.uk/sitemaps/job-posts-index.xml
Sitemap: https://www.cv-library.co.uk/sitemaps/recruiters-index.xml
Sitemap: https://www.cv-library.co.uk/sitemaps/salary-guide-index.xml