From 2a4bb8f2e9474da86e492894c9bcafcf008c345b Mon Sep 17 00:00:00 2001 From: Steven Gantz Date: Sat, 19 Oct 2024 16:18:54 -0400 Subject: [PATCH] Upload updated robots.txt --- astro/src/pages/robots.txt.ts | 5 +++++ robots.txt | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/astro/src/pages/robots.txt.ts b/astro/src/pages/robots.txt.ts index 0475e016..3c631aea 100644 --- a/astro/src/pages/robots.txt.ts +++ b/astro/src/pages/robots.txt.ts @@ -7,10 +7,15 @@ Disallow: /nogooglebot/ User-agent: * Allow: / +Allow: /posts +Allow: /posts/* Sitemap: ${new URL("sitemap-index.xml", SITE.website).href} +Sitemap: ${new URL("sitemap-0.xml", SITE.website).href} + `.trim(); + export const GET: APIRoute = () => new Response(robots, { headers: { "Content-Type": "text/plain" }, diff --git a/robots.txt b/robots.txt index 4eed84ac..ef376fc1 100644 --- a/robots.txt +++ b/robots.txt @@ -3,5 +3,8 @@ Disallow: /nogooglebot/ User-agent: * Allow: / +Allow: /posts +Allow: /posts/* -Sitemap: https://stevenpg.com/sitemap-index.xml \ No newline at end of file +Sitemap: https://stevenpg.com/sitemap-index.xml +Sitemap: https://stevenpg.com/sitemap-0.xml \ No newline at end of file