Merge pull request JetBrains#3819 from JetBrains/ktl-1180-exclude-noi…

…ndex fix; KTL-1180: Exclude hidden pages from search
hltj · Oct 12, 2023 · 80239f9 · 80239f9
2 parents 73e4ece + 8b70a93
commit 80239f9
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/dist.py b/src/dist.py
@@ -52,6 +52,9 @@ def get_dist_page_type(url):
         if parsed.find("meta", {"http-equiv": "refresh"}):
             page_type = 'Redirect'
 
+        if parsed.find("meta", {"name": "robots", "content": "noindex"}):
+            page_type = 'Hidden'
+
     if url.endswith('pdf'):
         page_type = 'File_Pdf'