From 65ddf73c2b703b9519e4f4c5977b5b190c32663f Mon Sep 17 00:00:00 2001 From: Phan Date: Sun, 17 Sep 2023 16:23:10 +1000 Subject: [PATCH 1/3] Allow dirsearch to use a non-default network interface --- CHANGELOG.md | 1 + CONTRIBUTORS.md | 1 + README.md | 2 ++ lib/connection/requester.py | 14 +++++++++++++- lib/parse/cmdline.py | 1 + requirements.txt | 1 + 6 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd1f9030c..fbdba80f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## [Unreleased] +- Support non-default network interface ## [0.4.3] - October 2nd, 2022 - Automatically detect the URI scheme (`http` or `https`) if no scheme is provided diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index c2357077d..fa655469e 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -73,6 +73,7 @@ - [Akshay Ravi](https://www.linkedin.com/in/c09yc47/) - [kosyan62](https://https://github.com/kosyan62) - [Maxence Zolnieurck](https://github.com/mxcezl) +- [huyphan](https://github.com/huyphan) Special thanks to all the people who are named here! diff --git a/README.md b/README.md index 6fe9f50c9..1404b9bdd 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,8 @@ Options: --max-rate=RATE Max requests per second --retries=RETRIES Number of retries for failed requests --ip=IP Server IP address + --interface=NETWORK_INTERFACE + Network interface to use Advanced Settings: --crawl Crawl for new paths in responses diff --git a/lib/connection/requester.py b/lib/connection/requester.py index 8c9f57930..d9ab20b99 100755 --- a/lib/connection/requester.py +++ b/lib/connection/requester.py @@ -28,6 +28,7 @@ from requests.auth import AuthBase, HTTPBasicAuth, HTTPDigestAuth from requests.packages import urllib3 from requests_ntlm import HttpNtlmAuth +from requests_toolbelt.adapters.socket_options import SocketOptionsAdapter from urllib.parse import urlparse from lib.core.data import options @@ -83,9 +84,20 @@ def __init__(self): if options["data"] and "content-type" not in self.headers: self.set_header("content-type", guess_mimetype(options["data"])) + socket_options = [] + if options.get('network_interface'): + socket_options.append( + (socket.SOL_SOCKET, socket.SO_BINDTODEVICE, options['network_interface'].encode("utf-8")) + ) + for scheme in ("http://", "https://"): self.session.mount( - scheme, HTTPAdapter(max_retries=0, pool_maxsize=options["thread_count"]) + scheme, + SocketOptionsAdapter( + max_retries=0, + pool_maxsize=options["thread_count"], + socket_options=socket_options + ) ) def _fetch_agents(self): diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index 4fdea4001..8d30da9dc 100755 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -450,6 +450,7 @@ def parse_arguments(): help="Number of retries for failed requests", ) connection.add_option("--ip", action="store", dest="ip", help="Server IP address") + connection.add_option("--interface", action="store", dest="network_interface", help="Network interface to use") # Advanced Settings advanced = OptionGroup(parser, "Advanced Settings") diff --git a/requirements.txt b/requirements.txt index 8a14e598d..7dea7a29f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ pyparsing>=2.4.7 beautifulsoup4>=4.8.0 mysql-connector-python>=8.0.20 psycopg[binary]>=3.0 +requests-toolbelt>=1.0.0 From a45de031dc0c739bcfc693da720696c4a959419f Mon Sep 17 00:00:00 2001 From: Pham Sy Minh <59408894+shelld3v@users.noreply.github.com> Date: Tue, 19 Sep 2023 19:52:49 +0700 Subject: [PATCH 2/3] Linting --- lib/connection/requester.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/connection/requester.py b/lib/connection/requester.py index d9ab20b99..78409bc30 100755 --- a/lib/connection/requester.py +++ b/lib/connection/requester.py @@ -85,9 +85,9 @@ def __init__(self): self.set_header("content-type", guess_mimetype(options["data"])) socket_options = [] - if options.get('network_interface'): + if options["network_interface"]: socket_options.append( - (socket.SOL_SOCKET, socket.SO_BINDTODEVICE, options['network_interface'].encode("utf-8")) + (socket.SOL_SOCKET, socket.SO_BINDTODEVICE, options["network_interface"].encode("utf-8")) ) for scheme in ("http://", "https://"): @@ -96,7 +96,7 @@ def __init__(self): SocketOptionsAdapter( max_retries=0, pool_maxsize=options["thread_count"], - socket_options=socket_options + socket_options=socket_options, ) ) From 1bd24902987528ec8d5d113e99656a817c5696c8 Mon Sep 17 00:00:00 2001 From: shelld3v <59408894+shelld3v@users.noreply.github.com> Date: Tue, 19 Sep 2023 20:08:01 +0700 Subject: [PATCH 3/3] Fix --- config.ini | 1 + lib/connection/requester.py | 1 - lib/core/data.py | 1 + lib/core/options.py | 1 + 4 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config.ini b/config.ini index 14c449603..21b4413a6 100644 --- a/config.ini +++ b/config.ini @@ -59,6 +59,7 @@ max-retries = 1 # proxies = ["localhost:8080"] # proxies-file = /path/to/proxies.txt # replay-proxy = localhost:8000 +# network-interface = eth0 [advanced] crawl = False diff --git a/lib/connection/requester.py b/lib/connection/requester.py index 78409bc30..b39c678b9 100755 --- a/lib/connection/requester.py +++ b/lib/connection/requester.py @@ -24,7 +24,6 @@ import threading import time -from requests.adapters import HTTPAdapter from requests.auth import AuthBase, HTTPBasicAuth, HTTPDigestAuth from requests.packages import urllib3 from requests_ntlm import HttpNtlmAuth diff --git a/lib/core/data.py b/lib/core/data.py index 367660bb9..bb6754b98 100755 --- a/lib/core/data.py +++ b/lib/core/data.py @@ -78,6 +78,7 @@ "scheme": None, "max_rate": 0, "max_retries": 1, + "network_interface": None, "ip": None, "exit_on_error": False, "crawl": False, diff --git a/lib/core/options.py b/lib/core/options.py index f4866c9e2..16c3c2c08 100755 --- a/lib/core/options.py +++ b/lib/core/options.py @@ -305,6 +305,7 @@ def parse_config(opt): "connection", "scheme", None, ("http", "https") ) opt.replay_proxy = opt.replay_proxy or config.safe_get("connection", "replay-proxy") + opt.network_interface = opt.network_interface or config.safe_get("connection", "network-interface") # Advanced opt.crawl = opt.crawl or config.safe_getboolean("advanced", "crawl")