# completely block these user-agents
User-agent: PetalBot
User-agent: SemrushBot
Disallow: /

# have to specifically block AdsBot-Google etc as they ignore the User-agent*
User-agent: Googlebot
User-agent: AdsBot-Google
User-agent: Mediapartners-Google
User-agent: Googlebot-Image
User-agent: *

Disallow: /api/
Disallow: /filter-count/
Disallow: /admin/
Disallow: /compare/
Disallow: /favourites/
Disallow: /listings/*/near/*
Disallow: /includes/
Disallow: /list/add-watch-list.cfm
Disallow: /list/view_image.cfm
Disallow: /list/view_image-print.cfm
Disallow: /list/view_image-new.cfm
Disallow: /list/emailpop-up.cfm
Disallow: /pages/index.cfm
Disallow: /pages/?PortalID=
Disallow: /pages/?ClientID=
Disallow: /pages/special.cfm
Disallow: /pages/custom-content.cfm
Disallow: /pages/view-listing.cfm
Disallow: /display.cfm
Disallow: /t/tracking.cfm

Crawl-delay: 10

Sitemap: https://www.agdealer.com/storage/sitemaps/en/sitemap.xml
Sitemap: https://www.agdealer.com/storage/sitemaps/fr/sitemap.xml