# --------------------------------------------------------------- # 1. Crawlers that are not welcome at all. # (They are also blocked with a 403 in httpd.conf, because # some of them ignore robots.txt.) # --------------------------------------------------------------- User-agent: Bytespider User-agent: meta-externalagent User-agent: PetalBot User-agent: DotBot User-agent: MJ12bot Disallow: / # --------------------------------------------------------------- # 2. Heavy crawlers that are allowed, but must slow down # (one request every 10 seconds at most). # A crawler only reads ONE group, so the rules from group 3 # are repeated here. # --------------------------------------------------------------- User-agent: SemrushBot User-agent: AhrefsBot User-agent: Amazonbot User-agent: YandexBot Crawl-delay: 10 Disallow: /action/ Disallow: /api/ Disallow: /tableau-de-bord Disallow: /en/dashboard Disallow: /en/admin-dashboard Disallow: /notifications Disallow: /en/notifications Disallow: /messagerie Disallow: /en/messages Disallow: /favoris Disallow: /en/favorites Disallow: /gestion-abonnement Disallow: /en/manage-subscription Disallow: /mes-factures Disallow: /en/my-bills Disallow: /modification-du-profil Disallow: /en/profile-modification Disallow: /manage-genlists Disallow: /en/manage-genlists Disallow: /mes-artistes Disallow: /en/my-artists Disallow: /sauvegardes Disallow: /fiche-des-sauvegarde Disallow: /en/save-single Disallow: /en/playlists Disallow: /ajouter-une-audition Disallow: /en/add-an-audition Disallow: /modifier-une-audition Disallow: /en/edit-an-audition Disallow: /audition-invitations Disallow: /en/audition-invitations Disallow: /trouver-une-voix? Disallow: /en/find-a-voice? Disallow: /recherche? Disallow: /en/search? Disallow: /*?auditionfw= # --------------------------------------------------------------- # 3. Everyone else (Google, Bing, Apple, Facebook previews, ...) # Public pages, artist/agency profiles and the blog stay open. # --------------------------------------------------------------- User-agent: * # Internal actions, cron jobs, AJAX and API endpoints. # sessionVariables stays open: search pages call it while loading, # and Google needs it to render them correctly. Disallow: /action/ Disallow: /api/ # Member-only pages (they redirect to the login page anyway) Disallow: /tableau-de-bord Disallow: /en/dashboard Disallow: /en/admin-dashboard Disallow: /notifications Disallow: /en/notifications Disallow: /messagerie Disallow: /en/messages Disallow: /favoris Disallow: /en/favorites Disallow: /gestion-abonnement Disallow: /en/manage-subscription Disallow: /mes-factures Disallow: /en/my-bills Disallow: /modification-du-profil Disallow: /en/profile-modification Disallow: /manage-genlists Disallow: /en/manage-genlists Disallow: /mes-artistes Disallow: /en/my-artists Disallow: /sauvegardes Disallow: /fiche-des-sauvegarde Disallow: /en/save-single Disallow: /en/playlists Disallow: /ajouter-une-audition Disallow: /en/add-an-audition Disallow: /modifier-une-audition Disallow: /en/edit-an-audition Disallow: /audition-invitations Disallow: /en/audition-invitations # Search result variations (page=, order=random, filters). # The main search pages themselves stay open; every profile is # already listed in the sitemap, so crawlers don't need to page through. Disallow: /trouver-une-voix? Disallow: /en/find-a-voice? Disallow: /recherche? Disallow: /en/search? # Login links carrying an audition parameter Disallow: /*?auditionfw= Sitemap: https://www.voilacasting.com/sitemap.xml