Merge branch 'master' into typing

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
         pip install codespell flake8 -r requirements.txt
     - name: Test
       run: |
-        python3 dirsearch.py -w ./tests/static/wordlist.txt -u https://example.com -o tmp_report.json --format json --force-recursive -R 3 --full-url -q -O
+        python3 dirsearch.py -w ./tests/static/wordlist.txt -u https://example.com -o tmp_report.json --output-formats json --force-recursive -R 3 --full-url -q
         python3 dirsearch.py -w ./tests/static/wordlist.txt -l ./tests/static/targets.txt --subdirs /,admin/ --exclude-extensions conf -q -L -f -i 200 --user-agent a --log tmp_log.log
         python3 dirsearch.py -w ./tests/static/wordlist.txt --nmap-report ./tests/static/nmap.xml --max-rate 2 -H K:V --random-agent --overwrite-extensions --no-color
         python3 dirsearch.py -w ./tests/static/wordlist.txt --raw ./tests/static/raw.txt --prefixes . --suffixes ~ --skip-on-status 404 -m POST -d test=1 --crawl --min-response-size 9

CHANGELOG.md

@@ -1,10 +1,13 @@
 # Changelog
 
 ## [Unreleased]
+- Ability to use multiple output formats
+- MySQL and PostgreSQL report formats
+- Support variables in file path and SQL table name for saving results
 - Support non-default network interface
-- Remove unused dependencies (urllib3, cryptography, cffi, idna, chardet)
 - Load targets from a Nmap XML report
 - Added --async option to enable asynchronous mode (use coroutines instead of threads)
+- Added option to disable CLI output entirely
 
 ## [0.4.3] - October 2nd, 2022
 - Automatically detect the URI scheme (`http` or `https`) if no scheme is provided

README.md

@@ -31,6 +31,7 @@ Table of Contents
   * [Pausing progress](#pausing-progress)
   * [Recursion](#recursion)
   * [Threads](#threads)
+  * [Asynchronous](#asynchronous)
   * [Prefixes / Suffixes](#prefixes--suffixes)
   * [Blacklist](#blacklist)
   * [Filters](#filters)
@@ -129,12 +130,15 @@ Options:
 
   Mandatory:
     -u URL, --url=URL   Target URL(s), can use multiple flags
-    -l PATH, --url-file=PATH
+    -l PATH, --urls-file=PATH
                         URL list file
     --stdin             Read URL(s) from STDIN
     --cidr=CIDR         Target CIDR
     --raw=PATH          Load raw HTTP request from file (use '--scheme' flag
                         to set the scheme)
+    --nmap-report=PATH  Load targets from nmap report (Ensure the inclusion of
+                        the -sV flag during nmap scan for comprehensive
+                        results)
     -s SESSION_FILE, --session=SESSION_FILE
                         Session file
     --config=PATH       Path to configuration file (Default:
@@ -143,7 +147,8 @@ Options:
 
   Dictionary Settings:
     -w WORDLISTS, --wordlists=WORDLISTS
-                        Customize wordlists (separated by commas)
+                        Wordlist files or directories contain wordlists
+                        (separated by commas)
     -e EXTENSIONS, --extensions=EXTENSIONS
                         Extension list separated by commas (e.g. php,asp)
     -f, --force-extensions
@@ -172,6 +177,7 @@ Options:
   General Settings:
     -t THREADS, --threads=THREADS
                         Number of threads
+    --async             Enable asynchronous mode
     -r, --recursive     Brute-force recursively
     --deep-recursive    Perform recursive scan on every directory depth (e.g.
                         api/users -> api/)
@@ -224,14 +230,14 @@ Options:
     --data-file=PATH    File contains HTTP request data
     -H HEADERS, --header=HEADERS
                         HTTP request header, can use multiple flags
-    --header-file=PATH  File contains HTTP request headers
+    --headers-file=PATH
+                        File contains HTTP request headers
     -F, --follow-redirects
                         Follow HTTP redirects
     --random-agent      Choose a random User-Agent for each request
     --auth=CREDENTIAL   Authentication credential (e.g. user:password or
                         bearer token)
-    --auth-type=TYPE    Authentication type (basic, digest, bearer, ntlm, jwt,
-                        oauth2)
+    --auth-type=TYPE    Authentication type (basic, digest, bearer, ntlm, jwt)
     --cert-file=PATH    File contains client-side certificate
     --key-file=PATH     File contains client-side certificate private key
                         (unencrypted)
@@ -241,8 +247,10 @@ Options:
   Connection Settings:
     --timeout=TIMEOUT   Connection timeout
     --delay=DELAY       Delay between requests
-    --proxy=PROXY       Proxy URL (HTTP/SOCKS), can use multiple flags
-    --proxy-file=PATH   File contains proxy servers
+    -p PROXY, --proxy=PROXY
+                        Proxy URL (HTTP/SOCKS), can use multiple flags
+    --proxies-file=PATH
+                        File contains proxy servers
     --proxy-auth=CREDENTIAL
                         Proxy authentication credential
     --replay-proxy=PROXY
@@ -268,10 +276,12 @@ Options:
     -q, --quiet-mode    Quiet mode
 
   Output Settings:
-    -o PATH, --output=PATH
-                        Output file
+    -o PATH/URL, --output=PATH/URL
+                        Output file or MySQL/PostgreSQL URL (Format:
+                        scheme://[username:password@]host[:port]/database-
+                        name)
     --format=FORMAT     Report format (Available: simple, plain, json, xml,
-                        md, csv, html, sqlite)
+                        md, csv, html, sqlite, mysql, postgresql)
     --log=PATH          Log file
 ```
 
@@ -288,6 +298,7 @@ By default, `config.ini` inside your dirsearch directory is used as the configur
 
 [general]
 threads = 25
+async = False
 recursive = False
 deep-recursive = False
 force-recursive = False
@@ -414,6 +425,12 @@ In spite of that, the speed still depends a lot on the response time of the serv
 python3 dirsearch.py -e php,htm,js,bak,zip,tgz,txt -u https://target -t 20
 ```
 
+----
+### Asynchronous
+You can switch to asynchronous mode by `--async`, let dirsearch use coroutines instead of threads to handle concurrent requests.
+
+In theory, asynchronous mode offers better performance and lower CPU usage since it doesn't require switching between different thread contexts. Additionally, pressing CTRL+C will immediately pause progress without needing to wait for threads to suspend.
+
 ----
 ### Prefixes / Suffixes
 - **--prefixes**: Add custom prefixes to all entries
@@ -567,7 +584,7 @@ python3 dirsearch.py -e php,html,js -u https://target --proxylist proxyservers.t
 
 ----
 ### Reports
-Supported report formats: **simple**, **plain**, **json**, **xml**, **md**, **csv**,  **html**, **sqlite**
+Supported report formats: **simple**, **plain**, **json**, **xml**, **md**, **csv**,  **html**, **sqlite**, **mysql**, **postgresql**
 
 ```
 python3 dirsearch.py -e php -l URLs.txt --format plain -o report.txt

config.ini

@@ -4,6 +4,7 @@
 
 [general]
 threads = 25
+async = False
 recursive = False
 deep-recursive = False
 force-recursive = False
@@ -13,53 +14,53 @@ exclude-subdirs = %%ff/,.;/,..;/,;/,./,../,%%2e/,%%2e%%2e/
 random-user-agents = False
 max-time = 0
 exit-on-error = False
-# subdirs = /,api/
-# include-status = 200-299,401
-# exclude-status = 400,500-999
-# exclude-sizes = 0b,123gb
-# exclude-texts = [
-#   "Not found",
-#   "404"
-# ]
-# exclude-regex = "^403$"
-# exclude-redirect = "*/error.html"
-# exclude-response = 404.html
-# skip-on-status = 429,999
+#subdirs = /,api/
+#include-status = 200-299,401
+#exclude-status = 400,500-999
+#exclude-sizes = 0b,123gb
+#exclude-texts = [
+#  "Not found",
+#  "404"
+#]
+#exclude-regex = "^403$"
+#exclude-redirect = "*/error.html"
+#exclude-response = 404.html
+#skip-on-status = 429,999
 
 [dictionary]
 default-extensions = php,aspx,jsp,html,js
 force-extensions = False
 overwrite-extensions = False
 lowercase = False
 uppercase = False
-capitalization = False
-# exclude-extensions = old,log
-# prefixes = .,admin
-# suffixes = ~,.bak
-# wordlists = /path/to/wordlist1.txt,/path/to/wordlist2.txt
+capital = False
+#exclude-extensions = old,log
+#prefixes = .,admin
+#suffixes = ~,.bak
+#wordlists = /path/to/wordlist1.txt,/path/to/wordlist2.txt
 
 [request]
 http-method = get
 follow-redirects = False
-# headers = [
-#   "Header1: Value",
-#   "Header2: Value"
-# ]
-# headers-file = /path/to/headers.txt
-# user-agent = MyUserAgent
-# cookie = SESSIONID=123
+#headers = [
+#  "Header1: Value",
+#  "Header2: Value"
+#]
+#headers-file = /path/to/headers.txt
+#user-agent = MyUserAgent
+#cookie = SESSIONID=123
 
 [connection]
 timeout = 7.5
 delay = 0
 max-rate = 0
 max-retries = 1
-## By disabling `scheme` variable, dirsearch will automatically identify the URI scheme
-# scheme = http
-# proxies = ["localhost:8080"]
-# proxies-file = /path/to/proxies.txt
-# replay-proxy = localhost:8000
-# network-interface = eth0
+# By disabling `scheme` variable, dirsearch will automatically identify the URI scheme
+#scheme = http
+#proxies = ["localhost:8080"]
+#proxies-file = /path/to/proxies.txt
+#replay-proxy = localhost:8000
+#network-interface = eth0
 
 [advanced]
 crawl = False
@@ -69,11 +70,24 @@ full-url = False
 quiet-mode = False
 color = True
 show-redirects-history = False
+disable-cli = False
 
 [output]
-## Support: plain, simple, json, xml, md, csv, html, sqlite, mysql, postgresql
-report-format = plain
-autosave-report = True
-autosave-report-folder = reports/
-# log-file = /path/to/dirsearch.log
-# log-file-size = 50000000
+# Available: simple, plain, json, xml, md, csv, html, sqlite, mysql, postgresql
+output-formats = plain
+# Supported variables for 'output-file and 'output-sql-table':
+# - {extension}: File extension of the report, for 'output-file' only (e.g. txt, json)
+# - {format}: Output format (e.g. plain, simple, xml)
+# - {host}: Target hostname or IP (e.g. example.com)
+# - {scheme}: URI scheme (http or https)
+# - {port}: Port number (e.g. 443)
+# - {date}: Scan date, format: DD-MM-YYYY (e.g. 07-10-2022)
+#
+# For output formats other than PostgreSQL and MySQL
+#output-file = reports/{host}/{scheme}_{port}.{extension}
+#mysql-url = mysql://user:password@localhost/database
+#postgres-url = postgres://user:password@localhost/database
+# Table to be used for SQL output
+output-sql-table = {scheme}_{host}:{port}
+#log-file = /path/to/dirsearch.log
+#log-file-size = 50000000

db/dicc.txt

@@ -9434,6 +9434,7 @@ wp-content/plugins/jrss-widget/proxy.php?url=
 wp-content/plugins/super-forms/
 wp-content/plugins/wp-publication-archive/includes/openfile.php?file=
 wp-content/plugins/wpengine-snapshot/snapshots/
+wp-content/themes/
 wp-content/updraft/
 wp-content/upgrade/
 wp-content/uploads/

lib/connection/response.py

@@ -20,6 +20,7 @@
 
 from typing import Any
 
+import time
 import httpx
 import requests
 
@@ -30,11 +31,12 @@
     UNKNOWN,
 )
 from lib.parse.url import clean_path, parse_path
-from lib.utils.common import is_binary
+from lib.utils.common import get_readable_size, is_binary
 
 
 class BaseResponse:
     def __init__(self, response: requests.Response | httpx.Response) -> None:
+        self.datetime = time.strftime("%Y-%m-%d %H:%M:%S")
         self.url = str(response.url)
         self.full_path = parse_path(self.url)
         self.path = clean_path(self.full_path)
@@ -59,6 +61,10 @@ def length(self) -> int:
 
         return len(self.body)
 
+    @property
+    def size(self) -> str:
+        return get_readable_size(self.length)
+
     def __hash__(self) -> int:
         return hash(self.body)