Added ability to use new assets endpoint and updated functions which …

…relied on depreated scope requests

docs/src/usage/plugins/scratchspace.md

@@ -16,21 +16,39 @@
 >> '/tmp/Scratchspace/ADAMANTANT/test.txt'
 >> ```
 
+## scratchspace.set_assets_file(content, target=None, codename=None)
+
+> This function will save a `assets.txt` scope file within a `codename` folder in within the `self.db.scratchspace_dir` folder
+> If `self.db.use_scratchspace` is `True`, this function is automatically run when you do `targets.get_scope()` or `targets.get_scope_web()`
+>
+> | Arguments | Type | Description
+> | --- | --- | ---
+> | `content` | str,list(str) | Either a preformatted string or (more likely) the return of `targets.get_scope()`
+> | `target` | db.models.Target | A Target Database Object
+> | `codename` | str | Codename of a Target
+>
+>> Examples
+>> ```python3
+>> >>> scope = h.targets.get_scope_web(codename='ADAMANTANT')
+>> >>> h.scratchspace.set_assets_file(scope, codename='ADAMANTANT')
+>> '/tmp/Scratchspace/ADAMANTANT/assets.txt'
+>> ```
+
 ## scratchspace.set_burp_file(content, target=None, codename=None)
 
 > This function will save a `burp.txt` scope file within a `codename` folder in within the `self.db.scratchspace_dir` folder
-> If `self.db.use_scratchspace` is `True`, this function is automatically run when you do `self.targets.get_scope()` or `self.targets.get_scope_web()`
+> If `self.db.use_scratchspace` is `True`, this function is automatically run when you do `targets.get_scope()` or `targets.get_scope_web()`
 >
 > | Arguments | Type | Description
 > | --- | --- | ---
-> | `content` | str,list(str) | Either a preformatted string or (more likely) the return of `self.targets.get_scope_host()`
+> | `content` | str,list(str) | Either a preformatted string or (more likely) the return of `targets.get_scope()`
 > | `target` | db.models.Target | A Target Database Object
 > | `codename` | str | Codename of a Target
 >
 >> Examples
 >> ```python3
 >> >>> scope = h.targets.get_scope_web(codename='ADAMANTANT')
->> >>> h.scratchspace.set_hosts_file(scope, codename='ADAMANTANT')
+>> >>> h.scratchspace.set_burp_file(scope, codename='ADAMANTANT')
 >> '/tmp/Scratchspace/ADAMANTANT/burp.txt'
 >> ```
 
@@ -67,11 +85,11 @@
 ## scratchspace.set_hosts_file(content, target=None, codename=None)
 
 > This function will save a `hosts.txt` scope file within a `codename` folder in within the `self.db.scratchspace_dir` folder.
-> If `self.db.use_scratchspace` is `True`, this function is automatically run when you do `self.targets.get_scope()` or `self.targets.get_scope_host()`
+> If `self.db.use_scratchspace` is `True`, this function is automatically run when you do `targets.get_scope()` or `targets.get_scope_host()`
 >
 > | Arguments | Type | Description
 > | --- | --- | ---
-> | `content` | str,list(str) | Either a preformatted string or (more likely) the return of `self.targets.get_scope_host()`
+> | `content` | str,list(str) | Either a preformatted string or (more likely) the return of `targets.get_scope_host()`
 > | `target` | db.models.Target | A Target Database Object
 > | `codename` | str | Codename of a Target
 >

docs/src/usage/plugins/targets.md

@@ -72,7 +72,7 @@
 >> [
 >>   {
 >>     'target': '94hw8ier',
->>     'urls': [{'url': https://good.monkey.com'}]
+>>     'urls': [{'url': 'https://good.monkey.com'}]
 >>   },
 >>   ...
 >> ]
@@ -120,6 +120,38 @@
 >> [{"id": 1, ...},...]
 >> ```
 
+## targets.get_assets(self, target=None, asset_type=None, host_type=None, active='true', scope=['in', 'discovered'], sort='location', sort_dir='asc', page=None, organization_uid=None, **kwargs)
+
+> Pull back a list of assets related to a target.
+>
+> If no arguments are provided, whatever target you are currently connected to will be queried with the default paramters.
+> You can use the following arguments to specify a target/organization or override default parameters.
+>
+> Note that `scopeRules` and `listings` both have a `scope` field, which is confusing.
+> From the best I can tell, the one in `listings` is the one that matters.
+> PLEASE double check my math before trusting it blindly and let me know if I'm wrong.
+>
+> | Arguments | Type | Description
+> | --- | --- | ---
+> | `target` | db.models.Target | A single Target returned from the database
+> | `asset_type` | str | Type of target ('host', 'webapp', etc.)
+> | `host_type` | str | Type of information to get back ('cidr')
+> | `active` | str | This field appears to specify whether the asset is an active item in the target's scope
+> | `scope` | str | I'm honestly not entirely sure what this field is, but the default is ['in', 'discovered'] when made officially.
+> | `sort_dir` | str | SQL-type sort direction (`asc`, `desc`)
+> | `page` | int | The page of assets to return
+> | `organization_uid` | str | slug of the organization that owns the target
+>
+>> Examples
+>> ```python3
+>> >>> h.targets.get_assets()
+>> [{
+>> 'listings': [{'listingUid': 'iuqwehuh4', 'scope': 'in'}, ...],
+>> 'location': 'https://www.something.com (https://www.something.com)',
+>> 'scopeRules': [{'appliesTo': 'both', 'rule': '*.www.something.com/*', 'uid': 'qiuwe'}, ...],
+>> ...
+>> }, ...]
+
 ## targets.get_attachments(target, **kwargs)
 
 > Gets the attachments of a specific target.

src/synack/plugins/alerts.py

@@ -39,11 +39,11 @@ def email(self, subject='Test Alert', message='This is a test'):
 
     def sanitize(self, message):
         message = re.sub(r'[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}.[0-9]{1,3}', '[IPv4]', message)
-        message = re.sub(r'(?:https?:\/\/)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}' +
+        message = re.sub(r'(?:h[tx]{1,2}ps?:\/\/)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{2,6}' +
                          r'\b(?:[-a-zA-Z0-9()@:%_\+.~#?&\\/=]*)', '[URL]', message)
-        message = re.sub(r'(?:https?:\/\/)?(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}' +
-                         r'\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*)', '[URL]', message)
-        message = re.sub(r'[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b' +
+        message = re.sub(r'(?:h[xt]{1,2}ps?:\/\/)?(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}' +
+                         r'\\.[a-zA-Z0-9()]{2,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*)', '[URL]', message)
+        message = re.sub(r'[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{2,6}\b' +
                          r'(?:[-a-zA-Z0-9()@:%_\+.~#?&\\/=]*)', '[URL]', message)
         message = re.sub(r'(?:^|(?<=\s))(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|' +
                          r'([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|' +

src/synack/plugins/scratchspace.py

@@ -27,6 +27,15 @@ def build_filepath(self, filename, target=None, codename=None):
             f = f / filename
             return f
 
+    def set_assets_file(self, content, target=None, codename=None):
+        if target or codename:
+            if type(content) in [list, set]:
+                content = '\n'.join(content)
+            dest_file = self.build_filepath('assets.txt', target=target, codename=codename)
+            with open(dest_file, 'w') as fp:
+                fp.write(content)
+                return dest_file
+
     def set_burp_file(self, content, target=None, codename=None):
         if target or codename:
             if type(content) == dict:
@@ -55,7 +64,7 @@ def set_download_attachments(self, attachments, target=None, codename=None, prom
 
     def set_hosts_file(self, content, target=None, codename=None):
         if target or codename:
-            if type(content) == list:
+            if type(content) in [list, set]:
                 content = '\n'.join(content)
             dest_file = self.build_filepath('hosts.txt', target=target, codename=codename)
             with open(dest_file, 'w') as fp:

src/synack/plugins/targets.py

@@ -4,6 +4,7 @@
 """
 
 import ipaddress
+import re
 
 from urllib.parse import urlparse
 from .base import Plugin
@@ -45,45 +46,41 @@ def build_scope_host_db(self, slug, scope):
 
     def build_scope_web_burp(self, scope):
         """Return a Burp Suite scope given retrieved web scope"""
-        ret = {'target': {'scope': {'advanced_mode': 'true', 'exclude': [], 'include': []}}}
+        ret = {'target': {'scope': {'advanced_mode': 'true', 'exclude': list(), 'include': list()}}}
+
         for asset in scope:
-            state = 'include' if asset['status'] == 'in' else 'exclude'
-            raw = urlparse(asset['raw_url'])
-            for item in asset['rules']:
-                item = item['rule'].strip('.*')
-                url = urlparse(item)
-                if len(url.netloc) == 0:
-                    url = urlparse(raw.scheme + '://' + item)
-                ret['target']['scope'][state].append({
-                    'enabled': True if url.hostname else False,
-                    'scheme': url.scheme if url.scheme else 'any',
-                    'host': url.hostname,
-                    'file': url.path
-                })
+            state = 'include' if asset.get('status') == 'in' else 'exclude'
+            raw = urlparse(asset.get('location', ''))
+            item = asset.get('rule', '').strip('.*')
+            url = urlparse(item)
+            if len(url.netloc) == 0:
+                url = urlparse(raw.scheme + '://' + item)
+            ret['target']['scope'][state].append({
+                'enabled': True if url.hostname else False,
+                'scheme': url.scheme if url.scheme else 'any',
+                'host': url.hostname,
+                'file': url.path
+            })
         return ret
 
     def build_scope_web_db(self, scope):
         """Return a Web Scope that can be ingested into the Database"""
+        sorting = dict()
+
+        for item in scope:
+            if item.get('status') == 'in':
+                slug = item.get('listing')
+                sorting[slug] = sorting.get(slug, list())
+                sorting[slug].append({'url': item.get('location')})
+
         ret = list()
-        for asset in scope:
-            if asset.get('status') == 'in':
-                for owner in asset.get('owners'):
-                    ret.append({
-                        "target": owner.get('owner_uid'),
-                        "urls": [{
-                            "url": asset.get('raw_url')
-                        }]
-                    })
-        return ret
 
-    def build_scope_web_urls(self, scope):
-        """Return a list of the raw urls gived a retrieved web scope"""
-        ret = {"in": list(), "out": list()}
-        for asset in scope:
-            if asset.get('status') == 'in':
-                ret["in"].append(asset["raw_url"])
-            else:
-                ret["out"].append(asset["raw_url"])
+        for slug, urls in sorting.items():
+            ret.append({
+                'target': slug,
+                'urls': urls
+            })
+
         return ret
 
     def build_slug_from_codename(self, codename):
@@ -104,6 +101,49 @@ def get_assessments(self):
             self.db.add_categories(res.json())
             return self.db.categories
 
+    def get_assets(self, target=None, asset_type=None, host_type=None, active='true',
+                   scope=['in', 'discovered'], sort='location', sort_dir='asc',
+                   page=None, organization_uid=None, **kwargs):
+        """Get the assets (scope) of a target"""
+        if target is None:
+            if len(kwargs) > 0:
+                target = self.db.find_targets(**kwargs)
+            else:
+                curr = self.get_connected()
+                target = self.db.find_targets(slug=curr.get('slug'))
+
+        if type(scope) == str:
+            scope = [scope]
+
+        if target:
+            if type(target) is list and len(target) > 0:
+                target = target[0]
+            queries = list()
+
+            queries.append(f'listingUid%5B%5D={target.slug}')
+            if organization_uid is not None:
+                queries.append(f'organizationUid%5B%5D={organization_uid}')
+            if asset_type is not None:
+                queries.append(f'assetType%5B%5D={asset_type}')
+            if host_type is not None:
+                queries.append(f'hostType%5B%5D={host_type}')
+            for item in scope:
+                queries.append(f'scope%5B%5D={item}')
+            if sort is not None:
+                queries.append(f'sort%5B%5D={sort}')
+            if active is not None:
+                queries.append(f'active={active}')
+            if sort_dir is not None:
+                queries.append(f'sortDir={sort_dir}')
+            if page is not None:
+                queries.append(f'page={page}')
+
+            res = self.api.request('GET', f'asset/v2/assets?{"&".join(queries)}')
+            if res.status_code == 200:
+                if self.db.use_scratchspace:
+                    self.scratchspace.set_assets_file(res.text, target=target)
+                return res.json()
+
     def get_attachments(self, target=None, **kwargs):
         """Get the attachments of a target."""
         if target is None:
@@ -211,37 +251,65 @@ def get_scope(self, add_to_db=False, **kwargs):
     def get_scope_host(self, target=None, add_to_db=False, **kwargs):
         """Get the scope of a Host target"""
         if target is None:
-            target = self.db.find_targets(**kwargs)
-            if target:
-                target = target[0]
+            targets = self.db.find_targets(**kwargs)
+            if targets:
+                target = next(iter(targets), None)
+
+        scope = set()
+
         if target:
-            res = self.api.request('GET', f'targets/{target.slug}/cidrs?page=all')
-            if res.status_code == 200:
-                scope = res.json()['cidrs']
+            assets = self.get_assets(target=target, active='true', asset_type='host', host_type='cidr')
+            for asset in assets:
+                if asset.get('active'):
+                    try:
+                        ipaddress.IPv4Network(asset.get('location'))
+                        scope.add(asset.get('location'))
+                    except ipaddress.AddressValueError:
+                        # Not actually an IP
+                        pass
+
+            scope.discard(None)
+
+            if len(scope) > 0:
                 if add_to_db:
                     self.db.add_ips(self.build_scope_host_db(target.slug, scope))
                 if self.db.use_scratchspace:
                     self.scratchspace.set_hosts_file(scope, target=target)
-                return scope
+
+        return scope
 
     def get_scope_web(self, target=None, add_to_db=False, **kwargs):
-        """Get the web scpope of a Web or Mobile target"""
+        """Get the scope of a Web target"""
         if target is None:
-            target = self.db.find_targets(**kwargs)
-            if target:
-                target = target[0]
-        res = self.api.request('GET', f'asset/v1/organizations/{target.organization}' +
-                                      f'/owners/listings/{target.slug}/webapps')
-        if res.status_code == 200:
-            scope = list()
-            for asset in res.json():
-                if target.slug in [o['owner_uid'] for o in asset['owners']]:
-                    scope.append(asset)
-            if add_to_db:
-                self.db.add_urls(self.build_scope_web_db(scope))
-            if self.db.use_scratchspace:
-                self.scratchspace.set_burp_file(self.build_scope_web_burp(scope), target=target)
-            return scope
+            targets = self.db.find_targets(**kwargs)
+            if targets:
+                target = next(iter(targets), None)
+
+        scope = list()
+
+        if target:
+            assets = self.get_assets(target=target, active='true', asset_type='webapp')
+            for asset in assets:
+                if asset.get('active'):
+                    location = next(iter(re.split(r' \(', asset.get('location', ''))))
+                    for listing in asset.get('listings', []):
+                        status = listing.get('scope')
+                        listing = listing.get('listingUid')
+                        for rule in asset.get('scopeRules', []):
+                            scope.append({
+                                'status': status,
+                                'listing': listing,
+                                'location': location,
+                                'rule': rule.get('rule')
+                            })
+
+            if len(scope) > 0:
+                if add_to_db:
+                    self.db.add_urls(self.build_scope_web_db(scope))
+                if self.db.use_scratchspace:
+                    self.scratchspace.set_web_file(self.build_scope_web_burp(scope), target=target)
+
+        return scope
 
     def get_submissions(self, target=None, status="accepted", **kwargs):
         """Get the details of previously submitted vulnerabilities from the analytics of a target."""

test/test_alerts.py

@@ -70,11 +70,13 @@ def test_sanitize_overzealous(self):
         self.assertEqual(self.alerts.sanitize('This is fine!'), 'This is fine!')
         self.assertEqual(self.alerts.sanitize('This is: fine'), 'This is: fine')
         self.assertEqual(self.alerts.sanitize('This is fine?'), 'This is fine?')
+        self.assertEqual(self.alerts.sanitize('24.0'), '24.0')
 
     def test_sanitize_urls(self):
         """Should sanitize URLs"""
         self.assertEqual(self.alerts.sanitize('test.cc'), '[URL]')
         self.assertEqual(self.alerts.sanitize('http://1.2.ewufg.4.test.cc'), '[URL]')
+        self.assertEqual(self.alerts.sanitize('hxxp://1.2.ewufg.4.test.cc'), '[URL]')
         self.assertEqual(self.alerts.sanitize('http://1.2.ewufg.4.test.cc:8081'), '[URL]')
         self.assertEqual(self.alerts.sanitize('http://1.2.ewufg.4.test.cc:8081/tacos/are/number/1'), '[URL]')
         self.assertEqual(self.alerts.sanitize('URL: https://www.test.com/1/2/3/air'), 'URL: [URL]')