Upload updated robots.txt

StevenPG · Oct 19, 2024 · 2a4bb8f · 2a4bb8f
1 parent 1e6bfd6
commit 2a4bb8f
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 1 deletion.
diff --git a/astro/src/pages/robots.txt.ts b/astro/src/pages/robots.txt.ts
@@ -7,10 +7,15 @@ Disallow: /nogooglebot/
 
 User-agent: *
 Allow: /
+Allow: /posts
+Allow: /posts/*
 
 Sitemap: ${new URL("sitemap-index.xml", SITE.website).href}
+Sitemap: ${new URL("sitemap-0.xml", SITE.website).href}
+
 `.trim();
 
+
 export const GET: APIRoute = () =>
   new Response(robots, {
     headers: { "Content-Type": "text/plain" },

diff --git a/robots.txt b/robots.txt
@@ -3,5 +3,8 @@ Disallow: /nogooglebot/
 
 User-agent: *
 Allow: /
+Allow: /posts
+Allow: /posts/*
 
-Sitemap: https://stevenpg.com/sitemap-index.xml
+Sitemap: https://stevenpg.com/sitemap-index.xml
+Sitemap: https://stevenpg.com/sitemap-0.xml