From 82c2c977ee4490032267106bf045526a4af9656d Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Tue, 25 Aug 2020 12:06:33 +1000 Subject: [PATCH 01/13] moving brute parsing to utils --- enum_tools/azure_checks.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/enum_tools/azure_checks.py b/enum_tools/azure_checks.py index abe06cc..5ac65be 100644 --- a/enum_tools/azure_checks.py +++ b/enum_tools/azure_checks.py @@ -151,18 +151,7 @@ def brute_force_containers(storage_accounts, brute_list, threads): valid_accounts.append(account) # Read the brute force file into memory - with open(brute_list, encoding="utf8", errors="ignore") as infile: - names = infile.read().splitlines() - - # Clean up the names to usable for containers - banned_chars = re.compile('[^a-z0-9-]') - clean_names = [] - for name in names: - name = name.lower() - name = banned_chars.sub('', name) - if 63 >= len(name) >= 3: - if name not in clean_names: - clean_names.append(name) + clean_names = utils.get_brute(brute_list, mini=3) # Start a counter to report on elapsed time start_time = utils.start_timer() From dd631b7c1fe837888beebac1c4dfd4739fa2bbcf Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Tue, 25 Aug 2020 12:07:04 +1000 Subject: [PATCH 02/13] small util updates --- enum_tools/utils.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/enum_tools/utils.py b/enum_tools/utils.py index f6ef36d..4ebae98 100644 --- a/enum_tools/utils.py +++ b/enum_tools/utils.py @@ -34,7 +34,7 @@ def init_logfile(logfile): log_writer.write("\n\n#### CLOUD_ENUM {} ####\n" .format(now)) -def get_url_batch(url_list, use_ssl=False, callback='', threads=5): +def get_url_batch(url_list, use_ssl=False, callback='', threads=5, redir=True): """ Processes a list of URLs, sending the results back to the calling function in real-time via the `callback` parameter @@ -66,7 +66,7 @@ def get_url_batch(url_list, use_ssl=False, callback='', threads=5): # First, grab the pending async request and store it in a dict for url in batch: - batch_pending[url] = session.get(proto + url) + batch_pending[url] = session.get(proto + url, allow_redirects=redir) # Then, grab all the results from the queue. # This is where we need to catch exceptions that occur with large @@ -212,6 +212,26 @@ def printc(text, color): with open(LOGFILE, 'a') as log_writer: log_writer.write(text.lstrip()) +def get_brute(brute_file, mini=1, maxi=63, banned='[^a-z0-9_-]'): + """ + Generates a list of brute-force words based on length and allowed chars + """ + # Read the brute force file into memory + with open(brute_file, encoding="utf8", errors="ignore") as infile: + names = infile.read().splitlines() + + # Clean up the names to usable for containers + banned_chars = re.compile(banned) + clean_names = [] + for name in names: + name = name.lower() + name = banned_chars.sub('', name) + if maxi >= len(name) >= mini: + if name not in clean_names: + clean_names.append(name) + + return clean_names + def start_timer(): """ Starts a timer for functions in main module From 565d9a74116c40f3d328ce145772a2b4e602d313 Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Tue, 25 Aug 2020 12:07:22 +1000 Subject: [PATCH 03/13] added checks for GCP cloud functions --- enum_tools/gcp_checks.py | 116 +++++++++++++++++++++++++++++++++++++- enum_tools/gcp_regions.py | 23 ++++++++ 2 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 enum_tools/gcp_regions.py diff --git a/enum_tools/gcp_checks.py b/enum_tools/gcp_checks.py index 511a648..f4f0170 100644 --- a/enum_tools/gcp_checks.py +++ b/enum_tools/gcp_checks.py @@ -4,6 +4,7 @@ """ from enum_tools import utils +from enum_tools import gcp_regions BANNER = ''' ++++++++++++++++++++++++++ @@ -11,9 +12,14 @@ ++++++++++++++++++++++++++ ''' -# Known S3 domain names +# Known GCP domain names GCP_URL = 'storage.googleapis.com' APPSPOT_URL = 'appspot.com' +FUNC_URL = 'cloudfunctions.net' + +# Hacky, I know. Used to store project/region combos that report at least +# one cloud function, to brute force later on +HAS_FUNCS = [] def print_bucket_response(reply): """ @@ -107,6 +113,113 @@ def check_appspot(names, threads): # Stop the time utils.stop_timer(start_time) +def print_functions_response1(reply): + """ + Parses the HTTP reply the initial Cloud Functions check + + This function is passed into the class object so we can view results + in real-time. + """ + if reply.status_code == 404: + pass + elif reply.status_code == 302: + utils.printc(" Contains at least 1 Cloud Function: {}\n" + .format(reply.url), 'green') + HAS_FUNCS.append(reply.url) + else: + print(" Unknown status codes being received from {}:\n" + " {}: {}" + .format(reply.url, reply.status_code, reply.reason)) + +def print_functions_response2(reply): + """ + Parses the HTTP reply from the secondary, brute-force Cloud Functions check + + This function is passed into the class object so we can view results + in real-time. + """ + if reply.status_code == 302: + pass + elif reply.status_code == 403: + utils.printc(" AUTH REQUIRED Cloud Function: {}\n" + .format(reply.url), 'red') + elif reply.status_code == 405: + utils.printc(" UNAUTH Cloud Function (POST-Only): {}\n" + .format(reply.url), 'green') + elif reply.status_code == 200: + utils.printc(" UNAUTH Cloud Function (GET-OK): {}\n" + .format(reply.url), 'green') + else: + print(" Unknown status codes being received from {}:\n" + " {}: {}" + .format(reply.url, reply.status_code, reply.reason)) + +def check_functions(names, brute_list, threads): + """ + Checks for Google Cloud Functions running on cloudfunctions.net + + This is a two-part process. First, we want to find region/project combos + that have existing Cloud Functions. The URL for a function looks like this: + https://[ZONE]-[PROJECT-ID].cloudfunctions.net/[FUNCTION-NAME] + + We look for a 302 in [ZONE]-[PROJECT-ID].cloudfunctions.net. That means + there are some functions defined in that region. Then, we brute force a list + of possible function names there. + + See gcp_regions.py to define which regions to check. The tool currently + defaults to only 1 region, so you should really modify it for best results. + """ + print("[+] Checking for project/zones with Google Cloud Functions.") + + # Start a counter to report on elapsed time + start_time = utils.start_timer() + + # Pull the regions from a config file + regions = gcp_regions.REGIONS + + print("[*] Testing across {} regions defined in the config file" + .format(len(regions))) + + for region in regions: + # Initialize the list of initial URLs to check + candidates = [region + '-' + name + '.' + FUNC_URL for name in names] + + # Send the valid names to the batch HTTP processor + utils.get_url_batch(candidates, use_ssl=False, + callback=print_functions_response1, + threads=threads, + redir=False) + + # Retun from function if we have not found any valid combos + if not HAS_FUNCS: + utils.stop_timer(start_time) + return + + # If we did find something, we'll use the brute list. This will allow people + # to provide a separate fuzzing list if they choose. + print("[*] Now, brute forcing the valid combos for actual function names") + + # Load brute list in memory, based on allowed chars/etc + brute_strings = utils.get_brute(brute_list) + + # The global was built in a previous function. We only want to brute force + # project/region combos that we know have existing functions defined + for func in HAS_FUNCS: + # Initialize the list of initial URLs to check. Strip out the HTTP + # protocol first, as that is handled in the utility + func = func.replace("http://", "") + + candidates = [func + brute for brute in brute_strings] + + # Send the valid names to the batch HTTP processor + utils.get_url_batch(candidates, use_ssl=False, + callback=print_functions_response2, + threads=threads, + redir=False) + + # Stop the time + utils.stop_timer(start_time) + def run_all(names, args): """ Function is called by main program @@ -115,3 +228,4 @@ def run_all(names, args): check_gcp_buckets(names, args.threads) check_appspot(names, args.threads) + check_functions(names, args.brute, args.threads) diff --git a/enum_tools/gcp_regions.py b/enum_tools/gcp_regions.py new file mode 100644 index 0000000..fea6465 --- /dev/null +++ b/enum_tools/gcp_regions.py @@ -0,0 +1,23 @@ +""" +File used to track the DNS regions for GCP resources. +""" + +# Some enumeration tasks will need to go through the complete list of +# possible DNS names for each region. You may want to modify this file to +# use the regions meaningful to you. +# +# Whatever is listed in the last instance of 'REGIONS' below is what the tool +# will use. + + +# Here is the list I get when running `gcloud functions regions list` +REGIONS = ['us-central1', 'us-east1', 'us-east4', 'us-west2', 'us-west3', + 'us-west4', 'europe-west1', 'europe-west2', 'europe-west3', + 'europe-west6', 'asia-east2', 'asia-northeast1', 'asia-northeast2', + 'asia-northeast3', 'asia-south1', 'asia-southeast2', + 'northamerica-northeast1', 'southamerica-east1', + 'australia-southeast1'] + + +# And here I am limiting the search by overwriting this variable: +REGIONS = ['us-central1',] From ed58a08c8ed36e6897795d2942c6f77120d0cec8 Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Tue, 25 Aug 2020 12:19:27 +1000 Subject: [PATCH 04/13] added response code --- enum_tools/gcp_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enum_tools/gcp_checks.py b/enum_tools/gcp_checks.py index f4f0170..ecd2249 100644 --- a/enum_tools/gcp_checks.py +++ b/enum_tools/gcp_checks.py @@ -140,7 +140,7 @@ def print_functions_response2(reply): """ if reply.status_code == 302: pass - elif reply.status_code == 403: + elif reply.status_code == 403 or reply.status_code == 401: utils.printc(" AUTH REQUIRED Cloud Function: {}\n" .format(reply.url), 'red') elif reply.status_code == 405: From c55d4aeab9e971c6ea1b9d8d34556fbbd5327315 Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Wed, 26 Aug 2020 08:23:17 +1000 Subject: [PATCH 05/13] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9a772d4..116d440 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Currently enumerates the following: - Open GCP Buckets - Protected GCP Buckets - Google App Engine sites +- Cloud Functions (enumerates project/regions with existing functions, then brute forces actual function names) By "open" buckets/containers, I mean those that allow anonymous users to list contents. if you discover a protected bucket/container, it is still worth trying to brute force the contents with another tool. From 17f3d5724107e068a13ec46d8846263426e57c3f Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Wed, 26 Aug 2020 09:54:56 +1000 Subject: [PATCH 06/13] added a few keywords --- enum_tools/fuzz.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/enum_tools/fuzz.txt b/enum_tools/fuzz.txt index 18df3fb..c20d1ef 100644 --- a/enum_tools/fuzz.txt +++ b/enum_tools/fuzz.txt @@ -120,6 +120,7 @@ graphql gs gw help +iaas hub iam images @@ -131,6 +132,7 @@ ios iot jira js +k8s kube kubeengine kubernetes @@ -156,6 +158,7 @@ ops oracle org packages +paas passwords photos pics @@ -180,6 +183,7 @@ repo reports resources s3 +saas screenshots scripts sec @@ -210,6 +214,7 @@ store subversion support svn +svc syslog tasks teamcity @@ -228,6 +233,7 @@ userpictures users ux videos +vm web website wp From a07207b72752e45525db57b3ad451d69b774080e Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Wed, 26 Aug 2020 09:55:30 +1000 Subject: [PATCH 07/13] working on functions --- enum_tools/gcp_checks.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/enum_tools/gcp_checks.py b/enum_tools/gcp_checks.py index ecd2249..3c41ecb 100644 --- a/enum_tools/gcp_checks.py +++ b/enum_tools/gcp_checks.py @@ -138,16 +138,16 @@ def print_functions_response2(reply): This function is passed into the class object so we can view results in real-time. """ - if reply.status_code == 302: + if 'accounts.google.com/ServiceLogin' in reply.url: pass elif reply.status_code == 403 or reply.status_code == 401: - utils.printc(" AUTH REQUIRED Cloud Function: {}\n" - .format(reply.url), 'red') + utils.printc(" Auth required Cloud Function: {}\n" + .format(reply.url), 'orange') elif reply.status_code == 405: - utils.printc(" UNAUTH Cloud Function (POST-Only): {}\n" + utils.printc(" UNAUTHENTICATED Cloud Function (POST-Only): {}\n" .format(reply.url), 'green') elif reply.status_code == 200: - utils.printc(" UNAUTH Cloud Function (GET-OK): {}\n" + utils.printc(" UNAUTHENTICATED Cloud Function (GET-OK): {}\n" .format(reply.url), 'green') else: print(" Unknown status codes being received from {}:\n" @@ -209,13 +209,15 @@ def check_functions(names, brute_list, threads): # protocol first, as that is handled in the utility func = func.replace("http://", "") - candidates = [func + brute for brute in brute_strings] + # Noticed weird behaviour with functions when a slash is not appended. + # Works for some, but not others. However, appending a slash seems to + # get consistent results. Might need further validation. + candidates = [func + brute + '/' for brute in brute_strings] # Send the valid names to the batch HTTP processor utils.get_url_batch(candidates, use_ssl=False, callback=print_functions_response2, - threads=threads, - redir=False) + threads=threads) # Stop the time utils.stop_timer(start_time) From dfa04787c4e996476dc715c09753100e09097904 Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Thu, 27 Aug 2020 10:42:54 +1000 Subject: [PATCH 08/13] additional error checks --- enum_tools/azure_checks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/enum_tools/azure_checks.py b/enum_tools/azure_checks.py index 5ac65be..6ec8128 100644 --- a/enum_tools/azure_checks.py +++ b/enum_tools/azure_checks.py @@ -102,8 +102,10 @@ def print_container_response(reply): # Stop brute forcing accounts without permission if ('not authorized to perform this operation' in reply.reason or - 'not have sufficient permissions' in reply.reason): - print(" [!] Breaking out early, auth errors.") + 'not have sufficient permissions' in reply.reason or + 'Public access is not permitted' in reply.reason or + 'Server failed to authenticate the request' in reply.reason): + print(" [!] Breaking out early, auth required.") return 'breakout' # Stop brute forcing unsupported accounts From 0838ea10f0f183c13e46aa4fbf6181200b73b54e Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Thu, 27 Aug 2020 10:47:37 +1000 Subject: [PATCH 09/13] additional status code checks --- enum_tools/gcp_checks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/enum_tools/gcp_checks.py b/enum_tools/gcp_checks.py index 3c41ecb..7a60f2a 100644 --- a/enum_tools/gcp_checks.py +++ b/enum_tools/gcp_checks.py @@ -75,10 +75,12 @@ def print_appspot_response(reply): """ if reply.status_code == 404: pass - elif reply.status_code == 500 or reply.status_code == 503: + elif (str(reply.status_code)[0] == 5): utils.printc(" Google App Engine app with a 50x error: {}\n" .format(reply.url), 'orange') - elif reply.status_code == 200 or reply.status_code == 302: + elif (reply.status_code == 200 + or reply.status_code == 302 + or reply.status_code == 404): utils.printc(" Google App Engine app: {}\n" .format(reply.url), 'green') else: From c7aa8c9102a30603d967929817001ce925a52745 Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Thu, 27 Aug 2020 10:56:05 +1000 Subject: [PATCH 10/13] adding logging on function progress --- enum_tools/gcp_checks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/enum_tools/gcp_checks.py b/enum_tools/gcp_checks.py index 7a60f2a..f377d9b 100644 --- a/enum_tools/gcp_checks.py +++ b/enum_tools/gcp_checks.py @@ -199,7 +199,8 @@ def check_functions(names, brute_list, threads): # If we did find something, we'll use the brute list. This will allow people # to provide a separate fuzzing list if they choose. - print("[*] Now, brute forcing the valid combos for actual function names") + print("[*] Brute-forcing function names in {} project/region combos" + .format(len(HAS_FUNCS))) # Load brute list in memory, based on allowed chars/etc brute_strings = utils.get_brute(brute_list) @@ -207,6 +208,8 @@ def check_functions(names, brute_list, threads): # The global was built in a previous function. We only want to brute force # project/region combos that we know have existing functions defined for func in HAS_FUNCS: + print("[*] Brute-forcing {} function names in {}" + .format(len(brute_strings), func)) # Initialize the list of initial URLs to check. Strip out the HTTP # protocol first, as that is handled in the utility func = func.replace("http://", "") From 7c5be663d284666f6caa2dcef6f67948b1961f11 Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Thu, 27 Aug 2020 11:06:45 +1000 Subject: [PATCH 11/13] additional status code checks --- enum_tools/gcp_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enum_tools/gcp_checks.py b/enum_tools/gcp_checks.py index f377d9b..69efe03 100644 --- a/enum_tools/gcp_checks.py +++ b/enum_tools/gcp_checks.py @@ -148,7 +148,7 @@ def print_functions_response2(reply): elif reply.status_code == 405: utils.printc(" UNAUTHENTICATED Cloud Function (POST-Only): {}\n" .format(reply.url), 'green') - elif reply.status_code == 200: + elif reply.status_code == 200 or reply.status_code == 404: utils.printc(" UNAUTHENTICATED Cloud Function (GET-OK): {}\n" .format(reply.url), 'green') else: From 4080811195252ddbb11270b82d6888486e954a7a Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Fri, 28 Aug 2020 13:43:37 +1000 Subject: [PATCH 12/13] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 116d440..a0145cd 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Currently enumerates the following: By "open" buckets/containers, I mean those that allow anonymous users to list contents. if you discover a protected bucket/container, it is still worth trying to brute force the contents with another tool. -**IMPORTANT**: Azure Virtual Machine DNS records can span a lot of geo regions. To save time scanning, there is a "REGIONS" variable defined in cloudenum/azure_regions.py. You'll want to look at this file and edit it to be relevant to your own work. +**IMPORTANT**: Some resources (Azure Containers, GCP Functions) are discovered per-region. To save time scanning, there is a "REGIONS" variable defined in `cloudenum/azure_regions.py and cloudenum/gcp_regions.py` that is set by default to use only 1 region. You may want to look at these files and edit them to be relevant to your own work. See it in action in [Codingo](https://github.com/codingo)'s video demo [here](https://www.youtube.com/embed/pTUDJhWJ1m0). @@ -44,7 +44,7 @@ The only required argument is at least one keyword. You can use the built-in fuz You can provide multiple keywords by specifying the `-k` argument multiple times. -Azure Containers required two levels of brute-forcing, both handled automatically by this tool. First, by finding valid accounts (DNS). Then, by brute-forcing container names inside that account (HTTP scraping). The tool uses the same fuzzing file for both by default, but you can specificy individual files separately if you'd like. +Keywords are mutated automatically using strings from `enum_tools/fuzz.txt` or a file you provide with the `-m` flag. Services that require a second-level of brute forcing (Azure Containers and GCP Functions) will also use `fuzz.txt` by default or a file you provide with the `-b` flag. Let's say you were researching "somecompany" whose website is "somecompany.io" that makes a product called "blockchaindoohickey". You could run the tool like this: From a62efe2033bce332fee335904b6a1530014b4994 Mon Sep 17 00:00:00 2001 From: initstring <26131150+initstring@users.noreply.github.com> Date: Fri, 28 Aug 2020 13:44:52 +1000 Subject: [PATCH 13/13] Update README.md --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a0145cd..e51d30d 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,6 @@ Currently enumerates the following: - Google App Engine sites - Cloud Functions (enumerates project/regions with existing functions, then brute forces actual function names) -By "open" buckets/containers, I mean those that allow anonymous users to list contents. if you discover a protected bucket/container, it is still worth trying to brute force the contents with another tool. - -**IMPORTANT**: Some resources (Azure Containers, GCP Functions) are discovered per-region. To save time scanning, there is a "REGIONS" variable defined in `cloudenum/azure_regions.py and cloudenum/gcp_regions.py` that is set by default to use only 1 region. You may want to look at these files and edit them to be relevant to your own work. - See it in action in [Codingo](https://github.com/codingo)'s video demo [here](https://www.youtube.com/embed/pTUDJhWJ1m0). @@ -58,6 +54,8 @@ HTTP scraping and DNS lookups use 5 threads each by default. You can try increas cloudenum.py -k keyword -t 10 ``` +**IMPORTANT**: Some resources (Azure Containers, GCP Functions) are discovered per-region. To save time scanning, there is a "REGIONS" variable defined in `cloudenum/azure_regions.py and cloudenum/gcp_regions.py` that is set by default to use only 1 region. You may want to look at these files and edit them to be relevant to your own work. + **Complete Usage Details** ``` usage: cloud_enum.py [-h] -k KEYWORD [-m MUTATIONS] [-b BRUTE]