Improve autocalibration

maurosoria · Oct 24, 2024 · f2793f5 · f2793f5
1 parent 52338ef
commit f2793f5
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 35 deletions.
diff --git a/db/403_blacklist.txt b/db/403_blacklist.txt
@@ -1,21 +0,0 @@
-.hta
-.htaccess
-.htaccess-dev
-.htaccess-local
-.htaccess-marco
-.htaccess.BAK
-.htaccess.bak
-.htaccess.old
-.htaccess.inc
-.htaccess.txt
-.htaccess~
-.htaccess/
-.htpasswd
-.htpasswd-old
-.htpasswd.bak
-.htpasswd.inc
-.htpa55wd
-.htpasswd/
-.htpasswrd
-.htgroup
-.htusers

diff --git a/lib/core/fuzzer.py b/lib/core/fuzzer.py
@@ -155,29 +155,24 @@ def __init__(
 
     def setup_scanners(self) -> None:
         # Default scanners (wildcard testers)
-        self.scanners["default"].update(
-            {
-                "index": Scanner(self._requester, path=self._base_path),
-                "random": Scanner(
-                    self._requester, path=self._base_path + WILDCARD_TEST_POINT_MARKER
-                ),
-            }
+        self.scanners["default"]["random"] = Scanner(
+            self._requester, path=self._base_path + WILDCARD_TEST_POINT_MARKER
         )
 
         if options["exclude_response"]:
             self.scanners["default"]["custom"] = Scanner(
                 self._requester, tested=self.scanners, path=options["exclude_response"]
             )
 
-        for prefix in options["prefixes"] + DEFAULT_TEST_PREFIXES:
+        for prefix in set(options["prefixes"] + DEFAULT_TEST_PREFIXES):
             self.scanners["prefixes"][prefix] = Scanner(
                 self._requester,
                 tested=self.scanners,
                 path=f"{self._base_path}{prefix}{WILDCARD_TEST_POINT_MARKER}",
                 context=f"/{self._base_path}{prefix}***",
             )
 
-        for suffix in options["suffixes"] + DEFAULT_TEST_SUFFIXES:
+        for suffix in set(options["suffixes"] + DEFAULT_TEST_SUFFIXES):
             self.scanners["suffixes"][suffix] = Scanner(
                 self._requester,
                 tested=self.scanners,

diff --git a/lib/core/settings.py b/lib/core/settings.py
@@ -69,9 +69,9 @@
 
 STANDARD_PORTS = {"http": 80, "https": 443}
 
-DEFAULT_TEST_PREFIXES = (".",)
+DEFAULT_TEST_PREFIXES = (".", ".ht")
 
-DEFAULT_TEST_SUFFIXES = ("/",)
+DEFAULT_TEST_SUFFIXES = ("/", "~")
 
 DEFAULT_TOR_PROXIES = ("socks5://127.0.0.1:9050", "socks5://127.0.0.1:9150")
 

diff --git a/lib/utils/diff.py b/lib/utils/diff.py
@@ -50,14 +50,19 @@ def compare_to(self, content):
 
         i = -1
         splitted_content = content.split()
+        # Allow one miss, see https://github.com/maurosoria/dirsearch/issues/1279
+        misses = 0
         for pattern in self._static_patterns:
             try:
                 i = splitted_content.index(pattern, i + 1)
             except ValueError:
-                return False
+                if misses or len(self._static_patterns) < 20:
+                    return False
 
-        # The number of static patterns is not big enough to say it's a reliable method
-        if len(self._static_patterns) < 20 and len(content.split()) > len(self._base_content.split()):
+                misses += 1
+
+        # Static patterns doesn't seem to be a reliable enough method
+        if len(content.split()) > len(self._base_content.split()) and len(self._static_patterns) < 20:
             return difflib.SequenceMatcher(None, self._base_content, content).ratio() > 0.75
 
         return True