diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..c057b0f --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length=119 diff --git a/.github/workflows/mdbook.yml b/.github/workflows/mdbook.yml index 56c5dd8..511303f 100644 --- a/.github/workflows/mdbook.yml +++ b/.github/workflows/mdbook.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - wip-hydra pull_request: jobs: diff --git a/.gitignore b/.gitignore index 539274a..f1becf5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ __pycache__/ # C extensions *.so +# VIM extensions +*.swo +*.swp + # Distribution / packaging .Python build/ diff --git a/checks.sh b/checks.sh index ce68a68..f7c33fe 100755 --- a/checks.sh +++ b/checks.sh @@ -1,6 +1,6 @@ #!/bin/bash flake8 src test live-tests -coverage run --source=src --omit=src/synack/db/alembic/env.py,src/synack/db/alembic/versions/649443e08834_initial.py -m unittest discover test +coverage run --source=src --omit=src/synack/db/alembic/env.py,src/synack/db/alembic/versions/*.py -m unittest discover test coverage report | egrep -v "^[^T].*100%" coverage html diff --git a/docs/src/usage/examples/mission-bot.md b/docs/src/usage/examples/mission-bot.md index 026fb39..a3dba6c 100644 --- a/docs/src/usage/examples/mission-bot.md +++ b/docs/src/usage/examples/mission-bot.md @@ -21,9 +21,9 @@ known_missions = 0 while True: time.sleep(30) curr_missions = h.missions.get_count() - if curr_missions > known_missions: + if curr_missions and curr_missions > known_missions: known_missions = curr_missions - msns = h.missions.get_available() + missions = h.missions.get_available() for m in missions: time.sleep(1) h.missions.set_claimed(m) diff --git a/docs/src/usage/examples/mission-templates.md b/docs/src/usage/examples/mission-templates.md index 9750337..76a8c2c 100644 --- a/docs/src/usage/examples/mission-templates.md +++ b/docs/src/usage/examples/mission-templates.md @@ -23,7 +23,7 @@ def replace_placeholders(template, mission): """ for k in template.keys(): template[k] = template[k].replace('__CODENAME__', - mission['listingCodename'] + mission['listingCodename']) h = synack.Handler() diff --git a/docs/src/usage/plugins/db.md b/docs/src/usage/plugins/db.md index d305288..ac572c1 100644 --- a/docs/src/usage/plugins/db.md +++ b/docs/src/usage/plugins/db.md @@ -14,9 +14,11 @@ Additionally, some properties can be overridden by the State, which allows you t | email | No | Yes | The email used to log into Synack | http_proxy | No | Yes | The http web proxy (Burp, etc.) to use for requests | https_proxy | No | Yes | The https web proxy (Burp, etc.) to use for requests +| ips | Yes | No | All cached IPs | notifications_token | No | No | Synack Notifications Token used to authenticate requests | otp_secret | No | Yes | Synack OTP Secret | password | No | Yes | The password used to log into Synack +| ports | Yes | No | All cached Ports | proxies | Yes | Yes | A dict built from http_proxy and https_proxy | targets | Yes | No | All cached Targets | template_dir | No | Yes | The path to a directory where your templates are stored @@ -64,6 +66,48 @@ Additionally, some properties can be overridden by the State, which allows you t >> >>> h.db.add_targets([{...}, {...}, {...}]) >> ``` +## db.find_ips(ip, **kwargs) + +> Filters through all the ips to return ones which match a given criteria +> +> | Argument | Type | Description +> | --- | --- | --- +> | `ip` | str | IP Address to search for +> | `kwargs` | kwargs | Any attribute of the Target Database Model (codename, slug, is_active, etc.) +> +>> Examples +>> ```python3 +>> >>> h.db.find_ips(codename="SLEEPYPUPPY") +>> [{'ip': '1.1.1.1, 'target': '12398h21'}, ... ] +>> ``` + +## db.find_ports(port, protocol, source, ip, **kwargs) + +> Filters through all the ports to return ones which match a given criteria +> +> | Argument | Type | Description +> | --- | --- | --- +> | `port` | int | Port number to search for (443, 80, 25, etc.) +> | `protocol` | str | Protocol to search for (tcp, udp, etc.) +> | `source` | str | Source to search for (hydra, nmap, etc.) +> | `ip` | str | IP Address to search for +> | `kwargs` | kwargs | Any attribute of the Target Database Model (codename, slug, is_active, etc.) +> +>> Examples +>> ```python3 +>> >>> h.db.find_ports(codename="SLEEPYPUPPY") +>> [ +>> { +>> 'ip': '1.2.3.4', 'source': 'hydra', 'target': '123hg912', +>> 'ports': [ +>> { 'open': True, 'port': '443', 'protocol': 'tcp', 'screenshot_url': '', 'service': 'https - Wordpress', 'updated': 1654840021 }, +>> ... +>> ] +>> }, +>> ... +>> ] +>> ``` + ## db.find_targets(**kwargs) > Filters through all the targets to return ones which match a given criteria diff --git a/docs/src/usage/plugins/hydra.md b/docs/src/usage/plugins/hydra.md new file mode 100644 index 0000000..fce8fd3 --- /dev/null +++ b/docs/src/usage/plugins/hydra.md @@ -0,0 +1,38 @@ +# Hydra + +## hydra.get_hydra(page, max_page, update_db, **kwargs) + +> Returns information from Synack Hydra Service +> +> | Arguments | Type | Description +> | --- | --- | --- +> | `page` | int | Page of the Hydra Service to start on (Default: 1) +> | `max_page` | int | Highest page that should be queried (Default: 5) +> | `update_db` | bool | Store the results in the database +> +>> Examples +>> ```python3 +>> >>> h.hydra.get_hydra(codename='SLEEPYPUPPY') +>> [{'host_plugins': {}, 'ip': '1.2.3.4', 'last_changed_dt': '2022-01-01T01:02:03Z', ... }, ... ] +>> >>> h.hydra.get_hydra(codename='SLEEPYPUPPY', page=3, max_page=5, update_db=False) +>> [{'host_plugins': {}, 'ip': '3.4.5.6', 'last_changed_dt': '2022-01-01T01:02:03Z', ... }, ... ] +>> ``` + +## hydra.build_db_input() + +> Builds a list of ports ready to be ingested by the Database from Hydra output +> +>> Examples +>> ```python3 +>> >>> h.hydra.build_db_input(h.hydra.get_hydra(codename='SLEEPYPUPPY', update_db=False)) +>> [ +>> { +>> 'ip': '1.2.3.4', 'source': 'hydra', 'target': '123hg912', +>> 'ports': [ +>> { 'open': True, 'port': '443', 'protocol': 'tcp', 'screenshot_url': '', 'service': 'https - Wordpress', 'updated': 1654840021 }, +>> ... +>> ] +>> }, +>> ... +>> ] +>> ``` diff --git a/src/synack/_state.py b/src/synack/_state.py index 4f3cef0..1060460 100644 --- a/src/synack/_state.py +++ b/src/synack/_state.py @@ -29,8 +29,7 @@ def __init__(self): @property def config_dir(self) -> pathlib.PosixPath: if self._config_dir is None: - self._config_dir = pathlib.Path('~/.config/synack').\ - expanduser().resolve() + self._config_dir = pathlib.Path('~/.config/synack').expanduser().resolve() if self._config_dir: self._config_dir.mkdir(parents=True, exist_ok=True) return self._config_dir diff --git a/src/synack/db/__init__.py b/src/synack/db/__init__.py index b6b1374..e9e38f5 100644 --- a/src/synack/db/__init__.py +++ b/src/synack/db/__init__.py @@ -3,4 +3,6 @@ from .models import Target from .models import Config from .models import Category +from .models import IP from .models import Organization +from .models import Port diff --git a/src/synack/db/alembic/versions/649443e08834_initial.py b/src/synack/db/alembic/versions/649443e08834_initial.py index 954d7bc..7c549a0 100644 --- a/src/synack/db/alembic/versions/649443e08834_initial.py +++ b/src/synack/db/alembic/versions/649443e08834_initial.py @@ -21,15 +21,11 @@ def upgrade(): sa.Column('api_token', sa.VARCHAR(200), server_default=""), sa.Column('debug', sa.BOOLEAN, server_default='f'), sa.Column('email', sa.VARCHAR(150), server_default=""), - sa.Column('http_proxy', sa.VARCHAR(50), - server_default='http://localhost:8080'), - sa.Column('https_proxy', sa.VARCHAR(50), - server_default='http://localhost:8080'), + sa.Column('http_proxy', sa.VARCHAR(50), server_default='http://localhost:8080'), + sa.Column('https_proxy', sa.VARCHAR(50), server_default='http://localhost:8080'), sa.Column('login', sa.BOOLEAN, server_default='f'), - sa.Column('template_dir', sa.VARCHAR(250), - server_default='~/Templates'), - sa.Column('notifications_token', sa.VARCHAR(1000), - server_default=""), + sa.Column('template_dir', sa.VARCHAR(250), server_default='~/Templates'), + sa.Column('notifications_token', sa.VARCHAR(1000), server_default=""), sa.Column('otp_secret', sa.VARCHAR(50), server_default=""), sa.Column('password', sa.VARCHAR(150), server_default=""), sa.Column('user_id', sa.VARCHAR(20), server_default=""), @@ -38,20 +34,16 @@ def upgrade(): op.create_table('categories', sa.Column('id', sa.INTEGER, primary_key=True), sa.Column('name', sa.VARCHAR(100)), - sa.Column('passed_practical', sa.BOOLEAN, - server_default='f'), - sa.Column('passed_written', sa.BOOLEAN, - server_default='f')) + sa.Column('passed_practical', sa.BOOLEAN, server_default='f'), + sa.Column('passed_written', sa.BOOLEAN, server_default='f')) op.create_table('organizations', sa.Column('slug', sa.VARCHAR(20), primary_key=True)) op.create_table('targets', sa.Column('slug', sa.VARCHAR(20), primary_key=True), - sa.Column('category', sa.INTEGER, - sa.ForeignKey('categories.id')), - sa.Column('organization', sa.VARCHAR(20), - sa.ForeignKey('organizations.slug')), + sa.Column('category', sa.INTEGER, sa.ForeignKey('categories.id')), + sa.Column('organization', sa.VARCHAR(20), sa.ForeignKey('organizations.slug')), sa.Column('average_payout', sa.REAL, server_default='0.0'), sa.Column('codename', sa.VARCHAR(100)), sa.Column('date_updated', sa.INTEGER, default=0), @@ -62,10 +54,8 @@ def upgrade(): sa.Column('is_updated', sa.BOOLEAN, default='f'), sa.Column('last_submitted', sa.INTEGER, default=0), sa.Column('start_date', sa.INTEGER, default=0), - sa.Column('vulnerability_discovery', sa.BOOLEAN, - default='f'), - sa.Column('workspace_access_missing', sa.BOOLEAN, - default='f')) + sa.Column('vulnerability_discovery', sa.BOOLEAN, default='f'), + sa.Column('workspace_access_missing', sa.BOOLEAN, default='f')) def downgrade(): diff --git a/src/synack/db/alembic/versions/deb7dd07212c_added_ip_port_tables.py b/src/synack/db/alembic/versions/deb7dd07212c_added_ip_port_tables.py new file mode 100644 index 0000000..3c7e6ae --- /dev/null +++ b/src/synack/db/alembic/versions/deb7dd07212c_added_ip_port_tables.py @@ -0,0 +1,39 @@ +"""Added IP/Port tables + +Revision ID: deb7dd07212c +Revises: 649443e08834 +Create Date: 2022-05-23 00:26:08.257745 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'deb7dd07212c' +down_revision = '649443e08834' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('ips', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('ip', sa.VARCHAR(40)), + sa.Column('target', sa.VARCHAR(20), sa.ForeignKey('targets.slug'))) + op.create_table('ports', + sa.Column('id', sa.INTEGER, autoincrement=True, primary_key=True), + sa.Column('ip', sa.VARCHAR(40), sa.ForeignKey('ips.id')), + sa.Column('port', sa.INTEGER), + sa.Column('protocol', sa.VARCHAR(10)), + sa.Column('source', sa.VARCHAR(50)), + sa.Column('open', sa.BOOLEAN, server_default='f'), + sa.Column('service', sa.VARCHAR(200), server_default=''), + sa.Column('updated', sa.INTEGER, server_default='0'), + sa.Column('url', sa.VARCHAR(200), server_default=''), + sa.Column('screenshot_url', sa.VARCHAR(1000), server_default='')) + + +def downgrade(): + op.drop_table('ips') + op.drop_table('ports') diff --git a/src/synack/db/models/__init__.py b/src/synack/db/models/__init__.py index 45ef2d1..645563b 100644 --- a/src/synack/db/models/__init__.py +++ b/src/synack/db/models/__init__.py @@ -3,4 +3,6 @@ from .target import Target from .config import Config from .category import Category +from .ip import IP from .organization import Organization +from .port import Port diff --git a/src/synack/db/models/ip.py b/src/synack/db/models/ip.py new file mode 100644 index 0000000..a45be8c --- /dev/null +++ b/src/synack/db/models/ip.py @@ -0,0 +1,17 @@ +"""db/models/ip.py + +Database Model for the IP item +""" + +import sqlalchemy as sa +from sqlalchemy.orm import declarative_base +from .target import Target + +Base = declarative_base() + + +class IP(Base): + __tablename__ = 'ips' + id = sa.Column(sa.Integer, autoincrement=True, primary_key=True) + ip = sa.Column(sa.VARCHAR(40)) + target = sa.Column(sa.VARCHAR(20), sa.ForeignKey(Target.slug)) diff --git a/src/synack/db/models/port.py b/src/synack/db/models/port.py new file mode 100644 index 0000000..a592b0e --- /dev/null +++ b/src/synack/db/models/port.py @@ -0,0 +1,24 @@ +"""db/models/port.py + +Database Model for the Port item +""" + +import sqlalchemy as sa +from sqlalchemy.orm import declarative_base +from .ip import IP + +Base = declarative_base() + + +class Port(Base): + __tablename__ = 'ports' + id = sa.Column(sa.INTEGER, autoincrement=True, primary_key=True) + ip = sa.Column(sa.VARCHAR(40), sa.ForeignKey(IP.id)) + port = sa.Column(sa.INTEGER) + protocol = sa.Column(sa.VARCHAR(10)) + source = sa.Column(sa.VARCHAR(50)) + open = sa.Column(sa.BOOLEAN, default=False) + service = sa.Column(sa.VARCHAR(200), default="") + updated = sa.Column(sa.INTEGER, default=0) + url = sa.Column(sa.VARCHAR(200), default="") + screenshot_url = sa.Column(sa.VARCHAR(1000), default="") diff --git a/src/synack/plugins/__init__.py b/src/synack/plugins/__init__.py index a1576c5..08484c9 100644 --- a/src/synack/plugins/__init__.py +++ b/src/synack/plugins/__init__.py @@ -4,6 +4,7 @@ from .auth import Auth from .db import Db from .debug import Debug +from .hydra import Hydra from .missions import Missions from .notifications import Notifications from .targets import Targets diff --git a/src/synack/plugins/api.py b/src/synack/plugins/api.py index 3124b09..9751bfc 100644 --- a/src/synack/plugins/api.py +++ b/src/synack/plugins/api.py @@ -12,9 +12,7 @@ class Api(Plugin): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) for plugin in ['Debug', 'Db']: - setattr(self, - plugin.lower(), - self.registry.get(plugin)(self.state)) + setattr(self, plugin.lower(), self.registry.get(plugin)(self.state)) def login(self, method, path, **kwargs): """Modify API Request for Login diff --git a/src/synack/plugins/db.py b/src/synack/plugins/db.py index af0284b..9c842e7 100644 --- a/src/synack/plugins/db.py +++ b/src/synack/plugins/db.py @@ -12,7 +12,9 @@ from synack.db.models import Target from synack.db.models import Config from synack.db.models import Category +from synack.db.models import IP from synack.db.models import Organization +from synack.db.models import Port from .base import Plugin @@ -46,7 +48,32 @@ def add_categories(self, categories): session.commit() session.close() - def add_organizations(self, targets, session): + def add_ips(self, results, session=None): + close = False + if session is None: + session = self.Session() + close = True + q = session.query(IP) + for result in results: + filt = sa.and_( + IP.ip.like(result.get('ip')), + IP.target.like(result.get('target')) + ) + db_ip = q.filter(filt).first() + if not db_ip: + db_ip = IP( + ip=result.get('ip'), + target=result.get('target')) + session.add(db_ip) + if close: + session.commit() + session.close() + + def add_organizations(self, targets, session=None): + close = False + if session is None: + session = self.Session() + close = True q = session.query(Organization) for t in targets: if t.get('organization'): @@ -57,6 +84,48 @@ def add_organizations(self, targets, session): if not db_o: db_o = Organization(slug=slug) session.add(db_o) + if close: + session.commit() + session.close() + + def add_ports(self, results, **kwargs): + self.add_ips(results) + session = self.Session() + q = session.query(Port) + ips = session.query(IP) + for result in results: + ip = ips.filter_by(ip=result.get('ip')) + if ip: + ip = ip.first() + for port in result.get('ports', []): + filt = sa.and_( + Port.port.like(port.get('port')), + Port.protocol.like(port.get('protocol')), + Port.ip.like(ip.id), + Port.source.like(result.get('source'))) + db_port = q.filter(filt) + if not db_port: + db_port = Port( + port=port.get('port'), + protocol=port.get('protocol'), + service=port.get('service'), + screenshot_url=port.get('screenshot_url'), + url=port.get('url'), + ip=ip.id, + source=result.get('source'), + open=port.get('open'), + updated=port.get('updated') + ) + else: + db_port = db_port.first() + db_port.service = port.get('service', db_port.service) + db_port.screenshot_url = port.get('screenshot_url', db_port.screenshot_url) + db_port.url = port.get('url', db_port.url) + db_port.open = port.get('open', db_port.open) + db_port.updated = port.get('updated', db_port.updated) + session.add(db_port) + session.commit() + session.close() def add_targets(self, targets, **kwargs): session = self.Session() @@ -94,6 +163,78 @@ def find_targets(self, **kwargs): session.close() return targets + def find_ports(self, port=None, protocol=None, source=None, ip=None, **kwargs): + session = self.Session() + query = session.query(Port) + if port: + query = query.filter_by(port=port) + if protocol: + query = query.filter_by(protocol=protocol) + if source: + query = query.filter_by(source=source) + + query = query.join(IP) + if ip: + query = query.filter_by(ip=ip) + + query = query.join(Target) + if kwargs: + query = query.filter_by(**kwargs) + + ports = query.all() + + ips = dict() + for port in ports: + ips[port.ip] = ips.get(port.ip, list()) + ips[port.ip].append({ + "port": port.port, + "protocol": port.protocol, + "service": port.service, + "source": port.source, + "open": port.open, + "updated": port.updated, + "url": port.url, + "screenshot_url": port.screenshot_url + }) + + ret = list() + for ip_id in ips.keys(): + ip = session.query(IP).filter_by(id=ip_id).first() + ret.append({ + "ip": ip.ip, + "target": ip.target, + "ports": ips[ip_id] + }) + + session.expunge_all() + session.close() + return ret + + def find_ips(self, ip=None, **kwargs): + session = self.Session() + query = session.query(IP) + + if ip: + query = query.filter_by(ip=ip) + + query = query.join(Target) + if kwargs: + query = query.filter_by(**kwargs) + + ips = query.all() + + session.expunge_all() + session.close() + + ret = list() + for ip in ips: + ret.append({ + "ip": ip.ip, + "target": ip.target + }) + + return ret + def get_config(self, name=None): session = self.Session() config = session.query(Config).filter_by(id=1).first() @@ -161,6 +302,20 @@ def targets(self): session.close() return targets + @property + def ports(self): + session = self.Session() + ports = session.query(Port).all() + session.close() + return ports + + @property + def ips(self): + session = self.Session() + ips = session.query(IP).all() + session.close() + return ips + @property def api_token(self): return self.get_config('api_token') diff --git a/src/synack/plugins/hydra.py b/src/synack/plugins/hydra.py new file mode 100644 index 0000000..125ce14 --- /dev/null +++ b/src/synack/plugins/hydra.py @@ -0,0 +1,81 @@ +"""plugins/hydra.py + +Functions dealing with hydra +""" + +import json +import time + +from .base import Plugin +from datetime import datetime + + +class Hydra(Plugin): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + for plugin in ['Api', 'Db']: + setattr(self, + plugin.lower(), + self.registry.get(plugin)(self.state)) + + def get_hydra(self, page=1, max_page=5, update_db=True, **kwargs): + """Get Hydra results for target identified using kwargs (codename='x', slug='x', etc.)""" + max_page = 1000 if max_page == 0 else max_page + target = self.db.find_targets(**kwargs)[0] + results = list() + if target: + query = { + 'page': page, + 'listing_uids': target.slug, + 'q': '+port_is_open:true' + } + time.sleep(page*0.01) + res = self.api.request('GET', + 'hydra_search/search', + query=query) + if res.status_code == 200: + curr_results = json.loads(res.content) + results.extend(curr_results) + if len(curr_results) == 10 and page < max_page: + results.extend(self.get_hydra(page=page+1, max_page=max_page, **kwargs)) + if update_db: + self.db.add_ports(self.build_db_input(results)) + return results + + def build_db_input(self, results): + """Format the Hydra output so that it can be ingested into the DB""" + db_input = list() + for result in results: + ports = list() + for port in result.get('ports').keys(): + for protocol in result['ports'][port].keys(): + for hydra_src in result['ports'][port][protocol].keys(): + h_src = result['ports'][port][protocol][hydra_src] + service = h_src.get('verified_service', {'parsed': 'unknown'})['parsed'] + \ + ' - ' + \ + h_src.get('product', {'parsed': 'unknown'})['parsed'] + service = service.strip(' - ') + screenshot_url = result['ports'][port][protocol][hydra_src].get('screenshot_key', '') + port_open = result['ports'][port][protocol][hydra_src]['open']['parsed'] + epoch = datetime(1970, 1, 1) + try: + last_changed_dt = datetime.strptime(result['last_changed_dt'], "%Y-%m-%dT%H:%M:%SZ") + except ValueError: + last_changed_dt = datetime.strptime(result['last_changed_dt'], "%Y-%m-%dT%H:%M:%S.%fZ") + updated = int((last_changed_dt - epoch).total_seconds()) + + ports.append({ + "port": port, + "protocol": protocol, + "service": service, + "screenshot_url": screenshot_url, + "open": port_open, + "updated": updated + }) + db_input.append({ + "ip": result["ip"], + "target": result["listing_uid"], + "source": "hydra", + "ports": ports + }) + return db_input diff --git a/src/synack/plugins/missions.py b/src/synack/plugins/missions.py index 8eba1ee..e2b314a 100644 --- a/src/synack/plugins/missions.py +++ b/src/synack/plugins/missions.py @@ -58,7 +58,7 @@ def build_summary(self, missions): utc = datetime.utcnow() claimed_on = datetime.strptime(m['claimedOn'], "%Y-%m-%dT%H:%M:%S.%fZ") - elapsed = (utc - claimed_on).seconds + elapsed = int((utc - claimed_on).total_seconds()) time = m['maxCompletionTimeInSecs'] - elapsed if time < ret['time'] or ret['time'] == 0: ret['time'] = time diff --git a/test/test_db.py b/test/test_db.py index a577bf1..fd7b8c0 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -6,6 +6,7 @@ import alembic.command import alembic.config import os +import sqlalchemy import sys import pathlib import unittest @@ -54,8 +55,7 @@ def test_set_migration(self): mock_config.return_value = mock with patch.object(alembic.command, 'upgrade') as mock_upgrade: self.db.set_migration() - mock_config.return_value.set_main_option.\ - assert_has_calls(calls) + mock_config.return_value.set_main_option.assert_has_calls(calls) mock_upgrade.assert_called_with(mock, 'head') def test_get_config(self): @@ -71,6 +71,254 @@ def test_get_config(self): query.return_value.filter_by.return_value.first.assert_called_with() self.db.Session.return_value.close.assert_called_with() + def test_find_ips(self): + """Should return a list of IPs""" + self.db.Session = MagicMock() + + self.db.Session.return_value.query.return_value.join.return_value.all.return_value = [ + synack.db.models.IP(ip='1.2.3.4', target='487egfue'), + synack.db.models.IP(ip='4.3.2.1', target='487egfue') + ] + + returned = self.db.find_ips() + expected = [ + {'ip': '1.2.3.4', 'target': '487egfue'}, + {'ip': '4.3.2.1', 'target': '487egfue'} + ] + self.assertTrue(returned, expected) + self.db.Session.assert_called() + self.db.Session.return_value.expunge_all.assert_called() + self.db.Session.return_value.close.assert_called() + + def test_find_ips_filters(self): + """Should apply filters to IPs search""" + self.db.Session = MagicMock() + + self.db.Session.return_value.query.return_value.join.return_value.all.return_value = [] + + self.db.find_ips(ip='1.2.3.4', codename='SLEEPYPUPPY') + self.db.Session.return_value.query.return_value.filter_by.assert_called_with(ip='1.2.3.4') + self.db.Session.return_value.query.return_value.filter_by.return_value.join.return_value. \ + filter_by.assert_called_with(codename='SLEEPYPUPPY') + + self.db.Session.assert_called() + self.db.Session.return_value.expunge_all.assert_called() + self.db.Session.return_value.close.assert_called() + + def test_find_ports(self): + """Should return a list of Ports""" + self.db.Session = MagicMock() + + self.db.Session.return_value.query.return_value.join.return_value.join.return_value.all.return_value = [ + synack.db.models.Port(ip='1', port='443', protocol='tcp'), + synack.db.models.Port(ip='1', port='53', protocol='udp') + ] + + returned = self.db.find_ports() + expected = [ + {'ip': '1.2.3.4', 'target': '487egfue', 'ports': {'port': '443', 'protocol': 'tcp'}}, + {'ip': '4.3.2.1', 'target': '487egfue', 'ports': {'port': '53', 'protocol': 'udp'}} + ] + self.assertTrue(returned, expected) + self.db.Session.assert_called() + self.db.Session.return_value.expunge_all.assert_called() + self.db.Session.return_value.close.assert_called() + + def test_find_ports_filters(self): + """Should apply filters to Ports search""" + self.db.Session = MagicMock() + + self.db.Session.return_value.query.return_value.join.return_value.all.return_value = [] + + self.db.find_ports(port=25, ip='1.2.3.4', codename='SLEEPYPUPPY') + self.db.Session.return_value.query.return_value.filter_by.assert_called_with(port=25) + self.db.Session.return_value.query.return_value.filter_by.return_value.join.return_value. \ + filter_by.assert_called_with(ip='1.2.3.4') + self.db.Session.return_value.query.return_value.filter_by.return_value.join.return_value. \ + filter_by.return_value.join.return_value.filter_by.assert_called_with(codename='SLEEPYPUPPY') + + self.db.Session.assert_called() + self.db.Session.return_value.expunge_all.assert_called() + self.db.Session.return_value.close.assert_called() + + def test_find_ports_filter_by_source(self): + """Should apply source filters to Ports search""" + self.db.Session = MagicMock() + + self.db.Session.return_value.query.return_value.join.return_value.all.return_value = [] + + self.db.find_ports(source='nmap') + self.db.Session.return_value.query.return_value.filter_by.assert_called_with(source='nmap') + self.db.Session.assert_called() + self.db.Session.return_value.expunge_all.assert_called() + self.db.Session.return_value.close.assert_called() + + def test_find_ports_filter_by_protocol(self): + """Should apply source filters to Ports search""" + self.db.Session = MagicMock() + + self.db.Session.return_value.query.return_value.join.return_value.all.return_value = [] + + self.db.find_ports(protocol='tcp') + self.db.Session.return_value.query.return_value.filter_by.assert_called_with(protocol='tcp') + self.db.Session.assert_called() + self.db.Session.return_value.expunge_all.assert_called() + self.db.Session.return_value.close.assert_called() + + def test_add_ips_existing_ips(self): + """Should not add IPs if already in db""" + self.db.Session = MagicMock() + results = [ + { + "ip": "1.1.1.1", + "target": "7gh33tjf72", + "source": "nmap", + "ports": [ + { + "port": "443", + "protocol": "tcp", + "service": "Super Apache NGINX Deluxe", + "screenshot_url": "http://127.0.0.1/h3298h23.png", + "url": "http://bubba.net" + }, + { + "port": "53", + "protocol": "udp", + "service": "DNS" + } + ] + } + ] + query = self.db.Session.return_value.query + with patch.object(sqlalchemy, 'and_') as mock_and: + mock_and.return_value = 'sqlalchemy.and_' + self.db.add_ips(results) + + mock_and.assert_called() + query.asset_called_with(synack.db.models.IP) + query.return_value.filter.assert_called_with('sqlalchemy.and_') + query.return_value.filter.return_value.first.assert_called_with() + self.db.Session.return_value.commit.assert_called_with() + self.db.Session.return_value.close.assert_called_with() + + def test_add_ips_new_ips(self): + """Should app IPs if new""" + self.db.Session = MagicMock() + results = [ + { + "ip": "1.1.1.1", + "target": "7gh33tjf72", + "source": "nmap", + "ports": [ + { + "port": "443", + "protocol": "tcp", + "service": "Super Apache NGINX Deluxe", + "screenshot_url": "http://127.0.0.1/h3298h23.png", + "url": "http://bubba.net" + }, + { + "port": "53", + "protocol": "udp", + "service": "DNS" + } + ] + } + ] + query = self.db.Session.return_value.query + self.db.Session.return_value.query.return_value.filter.return_value.first.return_value = None + with patch.object(sqlalchemy, 'and_') as mock_and: + mock_and.return_value = 'sqlalchemy.and_' + self.db.add_ips(results) + + mock_and.assert_called() + query.asset_called_with(synack.db.models.IP) + query.return_value.filter.assert_called_with('sqlalchemy.and_') + query.return_value.filter.return_value.first.assert_called_with() + self.db.Session.return_value.commit.assert_called_with() + self.db.Session.return_value.close.assert_called_with() + + def test_add_ports_update(self): + """Should update ports if existing""" + self.db.Session = MagicMock() + self.db.add_ips = MagicMock() + results = [ + { + "ip": "1.1.1.1", + "target": "7gh33tjf72", + "source": "nmap", + "ports": [ + { + "port": "443", + "protocol": "tcp", + "service": "Super Apache NGINX Deluxe", + "screenshot_url": "http://127.0.0.1/h3298h23.png", + "url": "http://bubba.net", + "open": True, + "updated": 1654969137 + + }, + { + "port": "53", + "protocol": "udp", + "service": "DNS" + } + ] + } + ] + query = self.db.Session.return_value.query + with patch.object(sqlalchemy, 'and_') as mock_and: + mock_and.return_value = 'sqlalchemy.and_' + self.db.add_ports(results) + + mock_and.assert_called() + query.asset_called_with(synack.db.models.Port) + query.return_value.filter_by.assert_has_calls([ + unittest.mock.call(ip='1.1.1.1'), + unittest.mock.call().__bool__(), + unittest.mock.call().first() + ]) + self.db.Session.return_value.commit.assert_called_with() + self.db.Session.return_value.close.assert_called_with() + self.db.add_ips.assert_called_with(results) + + def test_add_ports_new(self): + """Should add port if new""" + self.db.Session = MagicMock() + self.db.add_ips = MagicMock() + results = [ + { + "ip": "1.1.1.1", + "target": "7gh33tjf72", + "source": "nmap", + "ports": [ + { + "port": "443", + "protocol": "tcp", + "service": "Super Apache NGINX Deluxe", + "screenshot_url": "http://127.0.0.1/h3298h23.png", + "url": "http://bubba.net" + }, + { + "port": "53", + "protocol": "udp", + "service": "DNS plz AXFR me" + } + ] + } + ] + query = self.db.Session.return_value.query + query.return_value.filter.return_value = None + with patch.object(sqlalchemy, 'and_') as mock_and: + mock_and.return_value = 'sqlalchemy.and_' + self.db.add_ports(results) + + mock_and.assert_called() + query.asset_called_with(synack.db.models.Port) + query.return_value.filter.assert_called_with('sqlalchemy.and_') + self.db.Session.return_value.commit.assert_called_with() + self.db.Session.return_value.close.assert_called_with() + def test_add_categories(self): self.db.Session = MagicMock() cats = [{ @@ -215,6 +463,28 @@ def test_targets(self): query.return_value.all.assert_called_with() self.db.Session.return_value.close.assert_called_with() + def test_ports(self): + """Should get all ports from the database""" + self.db.Session = MagicMock() + query = self.db.Session.return_value.query + query.return_value.all.return_value = 'ports' + + self.assertEqual('ports', self.db.ports) + query.assert_called_with(synack.db.models.Port) + query.return_value.all.assert_called_with() + self.db.Session.return_value.close.assert_called_with() + + def test_ips(self): + """Should get all ips from the database""" + self.db.Session = MagicMock() + query = self.db.Session.return_value.query + query.return_value.all.return_value = 'ips' + + self.assertEqual('ips', self.db.ips) + query.assert_called_with(synack.db.models.IP) + query.return_value.all.assert_called_with() + self.db.Session.return_value.close.assert_called_with() + def test_http_proxy(self): """Should set and get the http_proxy from the database""" self.db.get_config = MagicMock() @@ -302,28 +572,36 @@ def test_add_organizations(self): targets = [{ "organization": {"slug": "qweqwe"} }] - mock.query.return_value.filter_by.return_value.\ - first.return_value = None + mock.query.return_value.filter_by.return_value.first.return_value = None self.db.add_organizations(targets, mock) mock.query.assert_called_with(synack.db.models.Organization) mock.query.return_value.filter_by.assert_called_with(slug='qweqwe') - mock.query.return_value.filter_by.return_value.first.\ - assert_called_with() + mock.query.return_value.filter_by.return_value.first.assert_called_with() mock.add.assert_called() + def test_add_organizations_no_session(self): + """Should create and destroy a db session if not provided""" + self.db.Session = MagicMock() + targets = [{ + "organization": {"slug": "qweqwe"} + }] + self.db.Session.return_value.query.return_value.filter_by.return_value.first.return_value = None + self.db.add_organizations(targets) + self.db.Session.assert_called() + self.db.Session.return_value.commit.assert_called() + self.db.Session.return_value.close.assert_called() + def test_add_organizations_organization_id(self): """Should update Organizations table if organization_id provided""" mock = MagicMock() targets = [{ "organization_id": "asdasd" }] - mock.query.return_value.filter_by.return_value.\ - first.return_value = None + mock.query.return_value.filter_by.return_value.first.return_value = None self.db.add_organizations(targets, mock) mock.query.assert_called_with(synack.db.models.Organization) mock.query.return_value.filter_by.assert_called_with(slug='asdasd') - mock.query.return_value.filter_by.return_value.first.\ - assert_called_with() + mock.query.return_value.filter_by.return_value.first.assert_called_with() mock.add.assert_called() def test_add_targets(self): diff --git a/test/test_hydra.py b/test/test_hydra.py new file mode 100644 index 0000000..e83c1b4 --- /dev/null +++ b/test/test_hydra.py @@ -0,0 +1,165 @@ +"""test_hydra.py + +Tests for the Hydra Plugin +""" + +import json +import os +import sys +import unittest + + +from unittest.mock import MagicMock + +sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../src'))) + +import synack # noqa: E402 + + +class HydraTestCase(unittest.TestCase): + def setUp(self): + self.state = synack._state.State() + self.hydra = synack.plugins.Hydra(self.state) + self.hydra.api = MagicMock() + self.hydra.db = MagicMock() + + def test_get_hydra(self): + """Should get information from Hydra""" + query = { + 'page': 1, + 'listing_uids': '87314gru', + 'q': '+port_is_open:true' + } + self.hydra.build_db_input = MagicMock() + self.hydra.build_db_input.return_value = 'BuildDbInputReturn' + self.hydra.db.find_targets.return_value = [ + synack.db.models.Target(codename='CRUSTYCRAB', slug='87314gru') + ] + self.hydra.api.request.return_value.status_code = 200 + content = '[{"somecontent": "content"}]' + self.hydra.api.request.return_value.content = content + returned = self.hydra.get_hydra(codename='CRUSTYCRAB') + self.assertTrue(returned == json.loads(content)) + self.hydra.api.request.assert_called_with('GET', + 'hydra_search/search', + query=query) + self.hydra.build_db_input.assert_called_with(json.loads(content)) + self.hydra.db.add_ports.assert_called_with('BuildDbInputReturn') + + def test_get_hydra_no_update_db(self): + """Should get information from Hydra without updating the DB""" + query = { + 'page': 1, + 'listing_uids': '87314gru', + 'q': '+port_is_open:true' + } + self.hydra.build_db_input = MagicMock() + self.hydra.build_db_input.return_value = 'BuildDbInputReturn' + self.hydra.db.find_targets.return_value = [ + synack.db.models.Target(codename='CRUSTYCRAB', slug='87314gru') + ] + self.hydra.api.request.return_value.status_code = 200 + content = '[{"somecontent": "content"}]' + self.hydra.api.request.return_value.content = content + returned = self.hydra.get_hydra(codename='CRUSTYCRAB', update_db=False) + self.assertTrue(returned == json.loads(content)) + self.hydra.api.request.assert_called_with('GET', + 'hydra_search/search', + query=query) + self.hydra.build_db_input.assert_not_called() + self.hydra.db.add_ports.assert_not_called() + + def test_get_hydra_multipage(self): + """Should get information from Hydra spanning multiple pages""" + query = { + 'page': 2, + 'listing_uids': '87314gru', + 'q': '+port_is_open:true' + } + self.hydra.build_db_input = MagicMock() + self.hydra.db.find_targets.return_value = [ + synack.db.models.Target(codename='CRUSTYCRAB', slug='87314gru') + ] + content = '[' + ','.join(['{"somecontent": "content"}' for i in range(0, 10)]) + ']' + self.hydra.api.request.return_value.status_code = 200 + self.hydra.api.request.return_value.content = content + returned = self.hydra.get_hydra(codename='CRUSTYCRAB', max_page=2) + self.assertTrue(len(returned) == 20) + self.hydra.api.request.assert_called_with('GET', + 'hydra_search/search', + query=query) + + def test_build_db_input(self): + """Should convert Hydra output into input for the DB""" + hydra_out = [ + { + 'host_plugins': {}, + 'ip': '1.1.1.1', + 'last_changed_dt': '2022-01-01T12:34:56Z', + 'listing_uid': 'owqeuhiqwe', + 'organization_profile_id': 0, + 'ports': { + '443': { + 'tcp': { + 'synack': { + 'cpe': { + 'last_changed_dt': '2021-01-01T01:01:01.123456Z', + 'parsed': '' + }, + 'open': { + 'last_changed_dt': '2022-01-01T12:34:56Z', + 'parsed': True + }, + 'product': { + 'last_changed_dt': '2021-10-01T12:43:06.654321Z', + 'parsed': '' + }, + 'verified_service': { + 'last_changed_dt': '2021-10-01T21:43:06.654321Z', + 'parsed': 'unknown' + } + } + }, + 'udp': {} + } + } + }, + { + 'host_plugins': {}, + 'ip': '1.1.1.2', + 'last_changed_dt': '2022-01-01T12:34:56.123456Z', + 'listing_uid': 'owqeuhiqwe', + 'organization_profile_id': 0, + 'ports': { + '443': { + 'tcp': { + 'synack': { + 'cpe': { + 'last_changed_dt': '2021-01-01T01:01:01.123456Z', + 'parsed': '' + }, + 'open': { + 'last_changed_dt': '2022-01-01T12:34:56Z', + 'parsed': True + }, + 'product': { + 'last_changed_dt': '2021-10-01T12:43:06.654321Z', + 'parsed': '' + }, + 'verified_service': { + 'last_changed_dt': '2021-10-01T21:43:06.654321Z', + 'parsed': 'unknown' + } + } + }, + 'udp': {} + } + } + } + ] + + returned = self.hydra.build_db_input(hydra_out) + expected = [{ + } + ] + self.assertTrue(returned, expected) diff --git a/test/test_notifications.py b/test/test_notifications.py index c2ca2cd..24d9be8 100644 --- a/test/test_notifications.py +++ b/test/test_notifications.py @@ -24,8 +24,7 @@ def setUp(self): def test_get(self): """Should get a list of notifications""" self.notifications.api.notifications.return_value.status_code = 200 - self.notifications.api.notifications.\ - return_value.json.return_value = {"one": "1"} + self.notifications.api.notifications.return_value.json.return_value = {"one": "1"} path = "notifications?meta=1" self.assertEqual({"one": "1"}, self.notifications.get()) self.notifications.api.notifications.assert_called_with("GET", path) @@ -33,8 +32,7 @@ def test_get(self): def test_get_unread_count(self): """Should get the number of unread notifications""" self.notifications.api.notifications.return_value.status_code = 200 - self.notifications.api.notifications.\ - return_value.json.return_value = {"one": "1"} + self.notifications.api.notifications.return_value.json.return_value = {"one": "1"} self.notifications.db.notifications_token = "good_token" query = { "authorization_token": "good_token" diff --git a/test/test_targets.py b/test/test_targets.py index 1278453..c969a2f 100644 --- a/test/test_targets.py +++ b/test/test_targets.py @@ -170,8 +170,7 @@ def test_get_credentials(self): self.targets.api.request.return_value.status_code = 200 self.targets.api.request.return_value.json.return_value = "json_return" - url = 'asset/v1/organizations/qwewqe/owners/listings/asdasd/' +\ - 'users/bobby/credentials' + url = 'asset/v1/organizations/qwewqe/owners/listings/asdasd/users/bobby/credentials' self.assertEqual("json_return", self.targets.get_credentials(codename='SLEEPYSLUG')) diff --git a/test/test_templates.py b/test/test_templates.py index a3f83c3..c4963f3 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -93,8 +93,7 @@ def test_get_file(self): mock_exists.return_value = True self.templates.get_file(mission) self.templates.build_filepath.assert_called_with(mission) - self.templates.build_sections.\ - assert_called_with('/tmp/mission.txt') + self.templates.build_sections.assert_called_with('/tmp/mission.txt') def test_set_file(self): self.templates.build_filepath = MagicMock()