disallow weird AI and SEO crawlers

breunigs · Dec 2, 2024 · b1ea25e · b1ea25e
1 parent f5b2999
commit b1ea25e
Showing 1 changed file with 16 additions and 6 deletions.
diff --git a/lib/mix/tasks/velo.assets.prepare.ex b/lib/mix/tasks/velo.assets.prepare.ex
@@ -33,17 +33,27 @@ defmodule Mix.Tasks.Velo.Assets.Prepare do
     User-agent: *
     Allow: /
 
-    User-agent: Amazonbot
-    Disallow: /images/thumbnails/
+    User-agent: AhrefsBot
+    Crawl-Delay: 600
 
-    User-agent: GPTBot-User
-    Allow: /
+    User-agent: AhrefsBot
+    Disallow: /live/websocket
+    Disallow: /images/thumbnails/
+    Disallow: /map/___static/
+    Disallow: /assets/basemap/
 
+    User-agent: Amazonbot
+    User-agent: Applebot-Extended
+    User-agent: barkrowler
+    User-agent: DataForSeoBot
     User-agent: GPTBot
+    User-agent: meta-externalagent
+    User-agent: PerplexityBot
+    User-agent: turnitinbot
     Disallow: /
 
-    User-agent: Applebot-Extended
-    Disallow: /
+    User-agent: GPTBot-User
+    Allow: /
     """)
   end
 end