Added Hydra Functionality (#4)

* Added ability to query Hydra
* Added new tables to the database to facilitate the new ability to pull Hydra information into the database. Can also be supplemented with wrappers you could write to ingest enumerated services from tools like nmap, masscan, etc.
* New functions added to Database to search for ports and neatly return them

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length=119

.github/workflows/mdbook.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - wip-hydra
   pull_request:
 
 jobs:

.gitignore

@@ -6,6 +6,10 @@ __pycache__/
 # C extensions
 *.so
 
+# VIM extensions
+*.swo
+*.swp
+
 # Distribution / packaging
 .Python
 build/

checks.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
 flake8 src test live-tests
-coverage run --source=src --omit=src/synack/db/alembic/env.py,src/synack/db/alembic/versions/649443e08834_initial.py -m unittest discover test
+coverage run --source=src --omit=src/synack/db/alembic/env.py,src/synack/db/alembic/versions/*.py -m unittest discover test
 coverage report | egrep -v "^[^T].*100%"
 coverage html

docs/src/usage/examples/mission-bot.md

@@ -21,9 +21,9 @@ known_missions = 0
 while True:
     time.sleep(30)
     curr_missions = h.missions.get_count()
-    if curr_missions > known_missions:
+    if curr_missions and curr_missions > known_missions:
         known_missions = curr_missions
-        msns = h.missions.get_available()
+        missions = h.missions.get_available()
         for m in missions:
             time.sleep(1)
             h.missions.set_claimed(m)

docs/src/usage/examples/mission-templates.md

@@ -23,7 +23,7 @@ def replace_placeholders(template, mission):
     """
     for k in template.keys():
         template[k] = template[k].replace('__CODENAME__',
-                                          mission['listingCodename']
+                                          mission['listingCodename'])
 
 h = synack.Handler()
 

docs/src/usage/plugins/db.md

@@ -14,9 +14,11 @@ Additionally, some properties can be overridden by the State, which allows you t
 | email | No | Yes | The email used to log into Synack
 | http_proxy | No | Yes | The http web proxy (Burp, etc.) to use for requests
 | https_proxy | No | Yes | The https web proxy (Burp, etc.) to use for requests
+| ips | Yes | No | All cached IPs
 | notifications_token | No | No | Synack Notifications Token used to authenticate requests
 | otp_secret | No | Yes | Synack OTP Secret
 | password | No | Yes | The password used to log into Synack
+| ports | Yes | No | All cached Ports
 | proxies | Yes | Yes | A dict built from http_proxy and https_proxy
 | targets | Yes | No | All cached Targets
 | template_dir | No | Yes | The path to a directory where your templates are stored
@@ -64,6 +66,48 @@ Additionally, some properties can be overridden by the State, which allows you t
 >> >>> h.db.add_targets([{...}, {...}, {...}])
 >> ```
 
+## db.find_ips(ip, **kwargs)
+
+> Filters through all the ips to return ones which match a given criteria
+>
+> | Argument | Type | Description
+> | --- | --- | ---
+> | `ip` | str | IP Address to search for
+> | `kwargs` | kwargs | Any attribute of the Target Database Model (codename, slug, is_active, etc.)
+>
+>> Examples
+>> ```python3
+>> >>> h.db.find_ips(codename="SLEEPYPUPPY")
+>> [{'ip': '1.1.1.1, 'target': '12398h21'}, ... ]
+>> ```
+
+## db.find_ports(port, protocol, source, ip, **kwargs)
+
+> Filters through all the ports to return ones which match a given criteria
+>
+> | Argument | Type | Description
+> | --- | --- | ---
+> | `port` | int | Port number to search for (443, 80, 25, etc.)
+> | `protocol` | str | Protocol to search for (tcp, udp, etc.)
+> | `source` | str | Source to search for (hydra, nmap, etc.)
+> | `ip` | str | IP Address to search for
+> | `kwargs` | kwargs | Any attribute of the Target Database Model (codename, slug, is_active, etc.)
+>
+>> Examples
+>> ```python3
+>> >>> h.db.find_ports(codename="SLEEPYPUPPY")
+>> [
+>>   {
+>>     'ip': '1.2.3.4', 'source': 'hydra', 'target': '123hg912',
+>>       'ports': [
+>>         { 'open': True, 'port': '443', 'protocol': 'tcp', 'screenshot_url': '', 'service': 'https - Wordpress', 'updated': 1654840021 },
+>>         ...
+>>       ]
+>>   },
+>>   ...
+>> ]
+>> ```
+
 ## db.find_targets(**kwargs)
 
 > Filters through all the targets to return ones which match a given criteria

docs/src/usage/plugins/hydra.md

@@ -0,0 +1,38 @@
+# Hydra
+
+## hydra.get_hydra(page, max_page, update_db, **kwargs)
+
+> Returns information from Synack Hydra Service
+>
+> | Arguments | Type | Description
+> | --- | --- | ---
+> | `page` | int | Page of the Hydra Service to start on (Default: 1)
+> | `max_page` | int | Highest page that should be queried (Default: 5)
+> | `update_db` | bool | Store the results in the database
+>
+>> Examples
+>> ```python3
+>> >>> h.hydra.get_hydra(codename='SLEEPYPUPPY')
+>> [{'host_plugins': {}, 'ip': '1.2.3.4', 'last_changed_dt': '2022-01-01T01:02:03Z', ... }, ... ]
+>> >>> h.hydra.get_hydra(codename='SLEEPYPUPPY', page=3, max_page=5, update_db=False)
+>> [{'host_plugins': {}, 'ip': '3.4.5.6', 'last_changed_dt': '2022-01-01T01:02:03Z', ... }, ... ]
+>> ```
+
+## hydra.build_db_input()
+
+> Builds a list of ports ready to be ingested by the Database from Hydra output
+>
+>> Examples
+>> ```python3
+>> >>> h.hydra.build_db_input(h.hydra.get_hydra(codename='SLEEPYPUPPY', update_db=False))
+>> [
+>>   {
+>>     'ip': '1.2.3.4', 'source': 'hydra', 'target': '123hg912',
+>>       'ports': [
+>>         { 'open': True, 'port': '443', 'protocol': 'tcp', 'screenshot_url': '', 'service': 'https - Wordpress', 'updated': 1654840021 },
+>>         ...
+>>       ]
+>>   },
+>>   ...
+>> ]
+>> ```

src/synack/_state.py

@@ -29,8 +29,7 @@ def __init__(self):
     @property
     def config_dir(self) -> pathlib.PosixPath:
         if self._config_dir is None:
-            self._config_dir = pathlib.Path('~/.config/synack').\
-                                       expanduser().resolve()
+            self._config_dir = pathlib.Path('~/.config/synack').expanduser().resolve()
         if self._config_dir:
             self._config_dir.mkdir(parents=True, exist_ok=True)
         return self._config_dir

src/synack/db/__init__.py

@@ -3,4 +3,6 @@
 from .models import Target
 from .models import Config
 from .models import Category
+from .models import IP
 from .models import Organization
+from .models import Port

src/synack/db/alembic/versions/649443e08834_initial.py

@@ -21,15 +21,11 @@ def upgrade():
                     sa.Column('api_token', sa.VARCHAR(200), server_default=""),
                     sa.Column('debug', sa.BOOLEAN, server_default='f'),
                     sa.Column('email', sa.VARCHAR(150), server_default=""),
-                    sa.Column('http_proxy', sa.VARCHAR(50),
-                              server_default='http://localhost:8080'),
-                    sa.Column('https_proxy', sa.VARCHAR(50),
-                              server_default='http://localhost:8080'),
+                    sa.Column('http_proxy', sa.VARCHAR(50), server_default='http://localhost:8080'),
+                    sa.Column('https_proxy', sa.VARCHAR(50), server_default='http://localhost:8080'),
                     sa.Column('login', sa.BOOLEAN, server_default='f'),
-                    sa.Column('template_dir', sa.VARCHAR(250),
-                              server_default='~/Templates'),
-                    sa.Column('notifications_token', sa.VARCHAR(1000),
-                              server_default=""),
+                    sa.Column('template_dir', sa.VARCHAR(250), server_default='~/Templates'),
+                    sa.Column('notifications_token', sa.VARCHAR(1000), server_default=""),
                     sa.Column('otp_secret', sa.VARCHAR(50), server_default=""),
                     sa.Column('password', sa.VARCHAR(150), server_default=""),
                     sa.Column('user_id', sa.VARCHAR(20), server_default=""),
@@ -38,20 +34,16 @@ def upgrade():
     op.create_table('categories',
                     sa.Column('id', sa.INTEGER, primary_key=True),
                     sa.Column('name', sa.VARCHAR(100)),
-                    sa.Column('passed_practical', sa.BOOLEAN,
-                              server_default='f'),
-                    sa.Column('passed_written', sa.BOOLEAN,
-                              server_default='f'))
+                    sa.Column('passed_practical', sa.BOOLEAN, server_default='f'),
+                    sa.Column('passed_written', sa.BOOLEAN, server_default='f'))
 
     op.create_table('organizations',
                     sa.Column('slug', sa.VARCHAR(20), primary_key=True))
 
     op.create_table('targets',
                     sa.Column('slug', sa.VARCHAR(20), primary_key=True),
-                    sa.Column('category', sa.INTEGER,
-                              sa.ForeignKey('categories.id')),
-                    sa.Column('organization', sa.VARCHAR(20),
-                              sa.ForeignKey('organizations.slug')),
+                    sa.Column('category', sa.INTEGER, sa.ForeignKey('categories.id')),
+                    sa.Column('organization', sa.VARCHAR(20), sa.ForeignKey('organizations.slug')),
                     sa.Column('average_payout', sa.REAL, server_default='0.0'),
                     sa.Column('codename', sa.VARCHAR(100)),
                     sa.Column('date_updated', sa.INTEGER, default=0),
@@ -62,10 +54,8 @@ def upgrade():
                     sa.Column('is_updated', sa.BOOLEAN, default='f'),
                     sa.Column('last_submitted', sa.INTEGER, default=0),
                     sa.Column('start_date', sa.INTEGER, default=0),
-                    sa.Column('vulnerability_discovery', sa.BOOLEAN,
-                              default='f'),
-                    sa.Column('workspace_access_missing', sa.BOOLEAN,
-                              default='f'))
+                    sa.Column('vulnerability_discovery', sa.BOOLEAN, default='f'),
+                    sa.Column('workspace_access_missing', sa.BOOLEAN, default='f'))
 
 
 def downgrade():

src/synack/db/alembic/versions/deb7dd07212c_added_ip_port_tables.py

@@ -0,0 +1,39 @@
+"""Added IP/Port tables
+
+Revision ID: deb7dd07212c
+Revises: 649443e08834
+Create Date: 2022-05-23 00:26:08.257745
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'deb7dd07212c'
+down_revision = '649443e08834'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('ips',
+                    sa.Column('id', sa.Integer, primary_key=True),
+                    sa.Column('ip', sa.VARCHAR(40)),
+                    sa.Column('target', sa.VARCHAR(20), sa.ForeignKey('targets.slug')))
+    op.create_table('ports',
+                    sa.Column('id', sa.INTEGER, autoincrement=True, primary_key=True),
+                    sa.Column('ip', sa.VARCHAR(40), sa.ForeignKey('ips.id')),
+                    sa.Column('port', sa.INTEGER),
+                    sa.Column('protocol', sa.VARCHAR(10)),
+                    sa.Column('source', sa.VARCHAR(50)),
+                    sa.Column('open', sa.BOOLEAN, server_default='f'),
+                    sa.Column('service', sa.VARCHAR(200), server_default=''),
+                    sa.Column('updated', sa.INTEGER, server_default='0'),
+                    sa.Column('url', sa.VARCHAR(200), server_default=''),
+                    sa.Column('screenshot_url', sa.VARCHAR(1000), server_default=''))
+
+
+def downgrade():
+    op.drop_table('ips')
+    op.drop_table('ports')

src/synack/db/models/__init__.py

@@ -3,4 +3,6 @@
 from .target import Target
 from .config import Config
 from .category import Category
+from .ip import IP
 from .organization import Organization
+from .port import Port

src/synack/db/models/ip.py

@@ -0,0 +1,17 @@
+"""db/models/ip.py
+
+Database Model for the IP item
+"""
+
+import sqlalchemy as sa
+from sqlalchemy.orm import declarative_base
+from .target import Target
+
+Base = declarative_base()
+
+
+class IP(Base):
+    __tablename__ = 'ips'
+    id = sa.Column(sa.Integer, autoincrement=True, primary_key=True)
+    ip = sa.Column(sa.VARCHAR(40))
+    target = sa.Column(sa.VARCHAR(20), sa.ForeignKey(Target.slug))

src/synack/db/models/port.py

@@ -0,0 +1,24 @@
+"""db/models/port.py
+
+Database Model for the Port item
+"""
+
+import sqlalchemy as sa
+from sqlalchemy.orm import declarative_base
+from .ip import IP
+
+Base = declarative_base()
+
+
+class Port(Base):
+    __tablename__ = 'ports'
+    id = sa.Column(sa.INTEGER, autoincrement=True, primary_key=True)
+    ip = sa.Column(sa.VARCHAR(40), sa.ForeignKey(IP.id))
+    port = sa.Column(sa.INTEGER)
+    protocol = sa.Column(sa.VARCHAR(10))
+    source = sa.Column(sa.VARCHAR(50))
+    open = sa.Column(sa.BOOLEAN, default=False)
+    service = sa.Column(sa.VARCHAR(200), default="")
+    updated = sa.Column(sa.INTEGER, default=0)
+    url = sa.Column(sa.VARCHAR(200), default="")
+    screenshot_url = sa.Column(sa.VARCHAR(1000), default="")

src/synack/plugins/__init__.py

@@ -4,6 +4,7 @@
 from .auth import Auth
 from .db import Db
 from .debug import Debug
+from .hydra import Hydra
 from .missions import Missions
 from .notifications import Notifications
 from .targets import Targets

src/synack/plugins/api.py

@@ -12,9 +12,7 @@ class Api(Plugin):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         for plugin in ['Debug', 'Db']:
-            setattr(self,
-                    plugin.lower(),
-                    self.registry.get(plugin)(self.state))
+            setattr(self, plugin.lower(), self.registry.get(plugin)(self.state))
 
     def login(self, method, path, **kwargs):
         """Modify API Request for Login