From eaa83939a72e899f9a2a07d4a232beb70dab2d95 Mon Sep 17 00:00:00 2001 From: Jonathan Cran Date: Wed, 17 Jun 2020 17:58:34 -0500 Subject: [PATCH] adjust scoping to use deny/accept lists, and push the individual decisions further down to enities --- Rakefile | 6 +- app/all.rb | 3 + app/models/global_entity.rb | 150 ---------------------- app/models/project.rb | 200 +++++++++++++++++++++++++++--- app/views/entities/detail.erb | 3 +- core.rb | 3 + db/060_entity_allow_deny_list.rb | 10 ++ lib/all.rb | 5 + lib/entities/analytics_id.rb | 4 +- lib/entities/autonomous_system.rb | 6 +- lib/entities/aws_credential.rb | 4 +- lib/entities/aws_iam_account.rb | 4 +- lib/entities/aws_region.rb | 4 +- lib/entities/aws_s3_bucket.rb | 4 +- lib/entities/credential.rb | 4 +- lib/entities/dns_record.rb | 6 +- lib/entities/domain.rb | 7 +- lib/entities/email_address.rb | 7 +- lib/entities/file_hash.rb | 4 +- lib/entities/github_account.rb | 4 +- lib/entities/github_repository.rb | 4 +- lib/entities/ip_address.rb | 4 +- lib/entities/mailserver.rb | 7 +- lib/entities/nameserver.rb | 7 +- lib/entities/net_block.rb | 4 +- lib/entities/network_service.rb | 4 +- lib/entities/organization.rb | 6 +- lib/entities/person.rb | 4 +- lib/entities/phone_number.rb | 4 +- lib/entities/physical_location.rb | 4 +- lib/entities/software_package.rb | 4 +- lib/entities/ssl_certificate.rb | 7 +- lib/entities/string.rb | 4 +- lib/entities/uri.rb | 12 +- lib/entities/web_account.rb | 4 +- lib/system/dns_helpers.rb | 66 ++++++++++ lib/tasks/base.rb | 27 ++-- lib/tasks/dns_transfer_zone.rb | 13 +- lib/tasks/gitrob.rb | 18 +-- lib/tasks/helpers/dns.rb | 49 +------- lib/tasks/helpers/issue.rb | 3 + lib/tasks/helpers/services.rb | 3 - lib/tasks/helpers/web_content.rb | 11 +- 43 files changed, 356 insertions(+), 351 deletions(-) create mode 100644 db/060_entity_allow_deny_list.rb create mode 100644 lib/system/dns_helpers.rb diff --git a/Rakefile b/Rakefile index f5afd563f..b174c18df 100644 --- a/Rakefile +++ b/Rakefile @@ -60,9 +60,9 @@ task :update do end def _get_global_entities - uri = "https://app.intrigue.io/api/global/entities?key=#{$intrigueio_api_key}" + uri = "https://app.intrigue.io/api/system/entities/global/entities/?key=#{$intrigueio_api_key}" begin - puts "[+] Making request for global entities" + puts "[+] Making request for global entities!" response = RestClient.get(uri) # handle missing data @@ -70,7 +70,7 @@ def _get_global_entities j = JSON.parse(response.body) rescue JSON::ParserError => e - puts "[+] Unable to parse bootstrap json" + puts "[+] Unable to parse json: #{e}" return -1 end j diff --git a/app/all.rb b/app/all.rb index 60ec4b785..78829716d 100644 --- a/app/all.rb +++ b/app/all.rb @@ -3,6 +3,9 @@ # must be brought in first, system should be skipped as a directive require_relative "routes/system" +# useful to bring in generic helper functions +require_relative '../lib/system/dns_helpers' + require_relative "routes/analysis" require_relative "routes/entities" require_relative "routes/issues" diff --git a/app/models/global_entity.rb b/app/models/global_entity.rb index b2b9fd78b..8e7ccf9ba 100644 --- a/app/models/global_entity.rb +++ b/app/models/global_entity.rb @@ -11,156 +11,6 @@ def validate validates_unique([:namespace, :type, :name]) end - def self.parse_domain_name(record) - return nil unless record - split_tld = parse_tld(record).split(".") - if (split_tld.last == "com" || split_tld.last == "net") && split_tld.count > 1 # handle cases like amazonaws.com, netlify.com - length = split_tld.count - else - length = split_tld.count + 1 - end - - record.split(".").last(length).join(".") - end - - # assumes we get a dns name of arbitrary length - def self.parse_tld(record) - return nil unless record - - # first check if we're not long enough to split, just returning the domain - return record if record && record.split(".").length < 2 - - # Make sure we're comparing bananas to bananas - record = "#{record}".downcase - - # now one at a time, check all known TLDs and match - begin - raw_suffix_list = File.open("#{$intrigue_basedir}/data/public_suffix_list.clean.txt").read.split("\n") - suffix_list = raw_suffix_list.map{|l| "#{l.downcase}".strip } - - # first find all matches - matches = [] - suffix_list.each do |s| - if record =~ /.*#{Regexp.escape(s.strip)}$/i # we have a match .. - matches << s.strip - end - end - - # then find the longest match - if matches.count > 0 - longest_match = matches.sort_by{|x| x.split(".").length }.last - return longest_match - end - - rescue Errno::ENOENT => e - _log_error "Unable to locate public suffix list, failing to check / create domain for #{lookup_name}" - return nil - end - - # unknown tld - record - end - - - #TODO .. this method should only be called if we don't already have the entity in our project - def self.traversable?(entity_type, entity_name, project) - - # by default things are not traversable - out = false - - # first check to see if we know about this exact entity (type matters too) - puts "Looking for global entity: #{entity_type} #{entity_name}" - global_entity = Intrigue::Model::GlobalEntity.first(:name => entity_name, :type => entity_type) - - # If we know it exists, is it in our project (cool) or someone else (no traverse!) - if global_entity - puts "Global entity found: #{entity_type} #{entity_name}!" - - # we need to have a namespace to validate against - if project.allowed_namespaces - project.allowed_namespaces.each do |namespace| - # if the entity's' namespace matches one of ours, we're good! - if global_entity.namespace.downcase == namespace.downcase - puts "Matches our namespace!" - return true # we can immediately return - end - end - else - puts "No Allowed Namespaces, but this is a claimed entity!" - return false - end - - else - puts "No Global entity found!" - end - - # okay so if we made it this far, we may or may not have a matching entiy, so now - # we need to find if it matches based on regex... since entities can have a couple - # different forms (uri, dns_record, domain, etc) - - # TODAY this only works on domains... and things that have a domain (like a uri) - - # then check each for a match - found_entity = nil - - ## Okay let's get smart by getting down to the smallest searchable unit first - searchable_name = nil - - #include Intrigue::Task::Dns # useful for parsing domain names - - if entity_type == "Domain" - # this should have gotten caught above... - searchable_name = parse_domain_name(entity_name) - elsif entity_type == "DnsRecord" - searchable_name = parse_domain_name(entity_name) - elsif entity_type == "EmailAddress" - searchable_name = parse_domain_name(entity_name.split("@").last) - elsif entity_type == "Nameserver" - searchable_name = parse_domain_name(entity_name) - elsif entity_type == "Uri" - searchable_name = parse_domain_name(URI.parse(entity_name).host) - end - - # now form the query, taking into acount the filter if we can - if searchable_name - found_entity = Intrigue::Model::GlobalEntity.first(:type => "Domain", :name => searchable_name) - else - global_entities = Intrigue::Model::GlobalEntity.all - - global_entities.each do |ge| - # this needs a couple (3) cases: - # 1) case where we're an EXACT match (ey.com) - # 2) case where we're a subdomain of an exception domain (x.ey.com) - # 3) case where we're a uri and should match an exception domain (https://ey.com) - # none of these cases should match the case: jcpenney.com - if (entity_name.downcase =~ /^#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/ || - entity_name.downcase =~ /^.*\.#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/ || - entity_name.downcase =~ /^https?:\/\/#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/) - - #okay we found it... now we need to check if it's an allowed project - found_entity = ge - end - end - end - - if found_entity && !project.allowed_namespaces.empty? # now lets check if we have an allowance for it - - (project.allowed_namespaces || []).each do |namespace| - if found_entity.namespace.downcase == namespace.downcase # Good! - return true - end - end - - out = false - else # we never found it or we don't care (no namespaces)! - out = true - end - - #puts "Result for: #{entity_type} #{entity_name} in project #{project.name}: #{out}" - - out - end - def self.load_global_namespace(data) (data["entities"] || []).each do |x| Intrigue::Model::GlobalEntity.update_or_create(:name => x["name"], :type => x["type"], :namespace => x["namespace"]) diff --git a/app/models/project.rb b/app/models/project.rb index fe62555eb..a9131cd95 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -11,6 +11,7 @@ class Project < Sequel::Model one_to_many :issues include Intrigue::Model::Mixins::Handleable + include Intrigue::System::DnsHelpers def validate super @@ -157,13 +158,107 @@ def export_applications_csv out end - # Method gives us a true/false, depending on whether the entity is in an - # exception list. Currently only used on project, but could be included - # in task_result or scan_result. Note that they'd need the "additional_exception_list" - # to be populated (automated by bootstrap) - def traversable_entity?(entity_name, type_string) + def globally_traversable_entity?(entity_type, entity_name) + + # by default things are not traversable + out = false + + # first check to see if we know about this exact entity (type matters too) + puts "Looking for global entity: #{entity_type} #{entity_name}" + global_entity = Intrigue::Model::GlobalEntity.first(:name => entity_name, :type => entity_type) + + # If we know it exists, is it in our project (cool) or someone else (no traverse!) + if global_entity + puts "Global entity found: #{entity_type} #{entity_name}!" + + # we need to have a namespace to validate against + if self.allowed_namespaces + self.allowed_namespaces.each do |namespace| + # if the entity's' namespace matches one of ours, we're good! + if global_entity.namespace.downcase == namespace.downcase + puts "Matches our namespace!" + return true # we can immediately return + end + end + else + puts "No allowed namespaces, and this is a claimed entity but not a seed!" + return false + end + + else + puts "No Global entity found, trying harder!" + end + + # okay so if we made it this far, we may or may not have a matching entiy, so now + # we need to find if it matches based on regex... since entities can have a couple + # different forms (uri, dns_record, domain, etc) + + # then check each for a match + found_entity = nil + + ## Okay let's get smart by getting down to the smallest searchable unit first + searchable_name = nil + + if entity_type == "Domain" + # this should have gotten caught above... + searchable_name = parse_domain_name(entity_name) + elsif entity_type == "DnsRecord" + searchable_name = parse_domain_name(entity_name) + elsif entity_type == "EmailAddress" + searchable_name = parse_domain_name(entity_name.split("@").last) + elsif entity_type == "Nameserver" + searchable_name = parse_domain_name(entity_name) + elsif entity_type == "Uri" + searchable_name = parse_domain_name(URI.parse(entity_name).host) + end + + # now form the query, taking into acount the filter if we can + if searchable_name + found_entity = Intrigue::Model::GlobalEntity.first(:type => "Domain", :name => searchable_name) + else + global_entities = Intrigue::Model::GlobalEntity.all + + global_entities.each do |ge| + # this needs a couple (3) cases: + # 1) case where we're an EXACT match (ey.com) + # 2) case where we're a subdomain of an exception domain (x.ey.com) + # 3) case where we're a uri and should match an exception domain (https://ey.com) + # none of these cases should match the case: jcpenney.com + if (entity_name.downcase =~ /^#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/ || + entity_name.downcase =~ /^.*\.#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/ || + entity_name.downcase =~ /^https?:\/\/#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/) + + #okay we found it... now we need to check if it's an allowed project + found_entity = ge + end + end + end + + if found_entity && !self.allowed_namespaces.empty? # now lets check if we have an allowance for it - # if it's an explicit seed, it's always traversable + (self.allowed_namespaces || []).each do |namespace| + if found_entity.namespace.downcase == namespace.downcase # Good! + return true + end + end + + out = false + else # we never found it or we don't care (no namespaces)! + out = true + end + + #puts "Result for: #{entity_type} #{entity_name} in project #{project.name}: #{out}" + + out + end + + + ### + ### Use this when you wan to scope in stuff + ### + def allow_list_entity?(type_string, entity_name) + + # if it's an explicit seed, it's whitelisted return true if seed_entity?(type_string,entity_name) ### CHECK OUR SEED ENTITIES TO SEE IF THE TEXT MATCHES A DOMAIN @@ -175,44 +270,113 @@ def traversable_entity?(entity_name, type_string) "Intrigue::Entity::EmailAddress", "Intrigue::Entity::Organization", "Intrigue::Entity::Nameserver", - "Intrigue::Entity::Uri" + "Intrigue::Entity::Uri" ] - # skip anything else!! - return true unless scope_check_entity_types.include? "Intrigue::Entity::#{type_string}" + # skip anything else thats not verifiable!! + return false unless scope_check_entity_types.include? "Intrigue::Entity::#{type_string}" - seeds.each do |s| + seeds.each do |s| if entity_name =~ /[\.\s\@]#{Regexp.escape(s.name)}/i - #puts "matched a seed, returning true" - return true + return true # matches a seed pattern, it's whitelisted end end - # Check standard exceptions (hardcoded list) first if we show up here (and we werent' a seed), we should skip + # Check standard exceptions (hardcoded list) first if we + # show up here (and we werent' a seed), we should skip if use_standard_exceptions if standard_no_traverse?(entity_name, type_string) - #puts 'Matched a standard exception, returning false' + #puts 'Matched a standard exception, not whitelisted' return false end end - # unless we can verify it against a domain, it's probably not that helpful to do this - # just assume we can't go any further + # now check the global intel verifiable_entity_types = ["DnsRecord", "Domain", "EmailAddress", "NameServer" "Uri"] if verifiable_entity_types.include? type_string # if we don't have a list, safe to return false now, otherwise proceed to # additional exceptions which are provided as an attribute on the object - unless Intrigue::Model::GlobalEntity.traversable?(type_string, entity_name, self) + unless globally_traversable_entity?(type_string, entity_name) puts 'Global intel says not traversable, returning false' return false end end - #puts "Defaulting to not traversable (Whitelist approach!)" + ### + # Defaulting to not traversable (Whitelist approach!) + ### + false + end + + ### + ### Use this when you wan to scope out stuff based on rules or global intel + ### + def deny_list_entity?(type_string, entity_name) + + # if it's an explicit seed, it's not blacklisted + return false if seed_entity?(type_string,entity_name) + + ### CHECK OUR SEED ENTITIES TO SEE IF THE TEXT MATCHES A DOMAIN + ###################################################### + # if it matches an explicit seed pattern + scope_check_entity_types = [ + "Intrigue::Entity::DnsRecord", + "Intrigue::Entity::Domain", + "Intrigue::Entity::EmailAddress", + "Intrigue::Entity::Organization", + "Intrigue::Entity::Nameserver", + "Intrigue::Entity::Uri" + ] + + # not blacklisted if we're not one of the check types + return false unless scope_check_entity_types.include? "Intrigue::Entity::#{type_string}" + + seeds.each do |s| + if entity_name =~ /[\.\s\@]#{Regexp.escape(s.name)}/i + return false # not blacklisted if we're matching a seed derivative + end + end + + # Check standard exceptions (hardcoded list) first if we + # show up here (and we werent' a seed), we should skip + if use_standard_exceptions + if standard_no_traverse?(entity_name, type_string) + return true # matched a blacklist + end + end + + # now check the global intel + verifiable_entity_types = ["DnsRecord", "Domain", "EmailAddress", "NameServer" "Uri"] + if verifiable_entity_types.include? type_string + # if we don't have a list, safe to return false now, otherwise proceed to + # additional exceptions which are provided as an attribute on the object + if !globally_traversable_entity?(type_string, entity_name) + puts 'Global intel says not traversable so we are blacklisted, returning true' + return true + end + end + + ### + # we made it this far, not blacklisted! + ### false end + # Method gives us a true/false, depending on whether the entity is in an + # exception list. Currently only used on project, but could be included + # in task_result or scan_result. Note that they'd need the "additional_exception_list" + # to be populated (automated by bootstrap) + ### + ### DEFAULTS TO FALSE!!! WHITELIST APPROACH + ### + def traversable_entity?(type_string, entity_name) + return true if allow_list_entity?(type_string, entity_name) + return false if deny_list_entity?(type_string, entity_name) + # otherwise, perimissive + true + end + # TODO - there must be a cleaner way? def get_option(option_name) opt = options.detect{|h| h[option_name] } if options diff --git a/app/views/entities/detail.erb b/app/views/entities/detail.erb index d19a38d52..bcb080eeb 100644 --- a/app/views/entities/detail.erb +++ b/app/views/entities/detail.erb @@ -10,7 +10,8 @@ Scoped: <%= @entity.scoped %>
Enriched: <%= @entity.enriched %>
-Hidden: <%= @entity.hidden %>
+Deny List Status: <%= @entity.deny_list %>
+Allow List Status: <%= @entity.allow_list %>

diff --git a/core.rb b/core.rb index 4c566b38b..ec491f3da 100644 --- a/core.rb +++ b/core.rb @@ -53,6 +53,9 @@ #require 'pry-byebug' require 'logger' +# disable annoying redis messages +Redis.exists_returns_integer = false + # # Simple configuration check to ensure we have configs in place def sanity_check_system diff --git a/db/060_entity_allow_deny_list.rb b/db/060_entity_allow_deny_list.rb new file mode 100644 index 000000000..574a24bc8 --- /dev/null +++ b/db/060_entity_allow_deny_list.rb @@ -0,0 +1,10 @@ +Sequel.migration do + change do + + alter_table(:entities) do + add_column :allow_list, TrueClass + add_column :deny_list, TrueClass + end + + end +end \ No newline at end of file diff --git a/lib/all.rb b/lib/all.rb index 289fde48f..58194f4f0 100644 --- a/lib/all.rb +++ b/lib/all.rb @@ -19,6 +19,11 @@ require_relative 'system/helpers' include Intrigue::System::Helpers +# Intrigue System-wide Helpers (both app and backend) +require_relative 'system/dns_helpers' +#include Intrigue::System::DnsHelpers + + # Intrigue Export Format require_relative 'system/json_data_export_file' diff --git a/lib/entities/analytics_id.rb b/lib/entities/analytics_id.rb index 47a1bfede..0f1831863 100644 --- a/lib/entities/analytics_id.rb +++ b/lib/entities/analytics_id.rb @@ -34,8 +34,8 @@ def supported_hash_types end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true # otherwise just default to true end diff --git a/lib/entities/autonomous_system.rb b/lib/entities/autonomous_system.rb index aed5bba80..83f0f7d3e 100644 --- a/lib/entities/autonomous_system.rb +++ b/lib/entities/autonomous_system.rb @@ -16,9 +16,9 @@ def validate_entity end def scoped? - return true if self.seed - return false if self.hidden - true # otherwise just default to true + return true if self.allow_list + return false if self.deny_list + false # otherwise false end end diff --git a/lib/entities/aws_credential.rb b/lib/entities/aws_credential.rb index 437ec239a..34f756199 100644 --- a/lib/entities/aws_credential.rb +++ b/lib/entities/aws_credential.rb @@ -33,8 +33,8 @@ def validate_entity end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true # otherwise just default to true end diff --git a/lib/entities/aws_iam_account.rb b/lib/entities/aws_iam_account.rb index cf3bd32b4..213e5065d 100644 --- a/lib/entities/aws_iam_account.rb +++ b/lib/entities/aws_iam_account.rb @@ -19,8 +19,8 @@ def detail_string end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true # otherwise just default to true end diff --git a/lib/entities/aws_region.rb b/lib/entities/aws_region.rb index 87412c158..aa71546c9 100644 --- a/lib/entities/aws_region.rb +++ b/lib/entities/aws_region.rb @@ -40,8 +40,8 @@ def enrichment_tasks end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true # otherwise just default to true end diff --git a/lib/entities/aws_s3_bucket.rb b/lib/entities/aws_s3_bucket.rb index 10d85507e..b502c88e6 100644 --- a/lib/entities/aws_s3_bucket.rb +++ b/lib/entities/aws_s3_bucket.rb @@ -24,8 +24,8 @@ def enrichment_tasks end def scoped?(conditions={}) - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true # otherwise just default to true end diff --git a/lib/entities/credential.rb b/lib/entities/credential.rb index d31d02aa9..376955adf 100644 --- a/lib/entities/credential.rb +++ b/lib/entities/credential.rb @@ -18,8 +18,8 @@ def validate_entity end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true # otherwise just default to true end diff --git a/lib/entities/dns_record.rb b/lib/entities/dns_record.rb index 4c808b3fe..f3a87b3ca 100644 --- a/lib/entities/dns_record.rb +++ b/lib/entities/dns_record.rb @@ -30,15 +30,15 @@ def enrichment_tasks ### SCOPING ### def scoped?(conditions={}) - return true if self.seed - return false if self.hidden # hit our blacklist so definitely false + return true if self.allow_list + return false if self.deny_list #self.project.seeds.each do |s| # return true if self.name =~ /[\.\s\@]#{Regexp.escape(s.name)}/i #end # check hidden on-demand - return false unless self.project.traversable_entity?(parse_domain_name(self.name), "Domain") + return false unless self.project.traversable_entity?("Domain", parse_domain_name(self.name)) # if we didnt match the above and we were asked, default to true true diff --git a/lib/entities/domain.rb b/lib/entities/domain.rb index cfca7a025..4c26f8f52 100644 --- a/lib/entities/domain.rb +++ b/lib/entities/domain.rb @@ -29,11 +29,8 @@ def enrichment_tasks ### SCOPING ### def scoped?(conditions={}) - return true if self.seed - return false if self.hidden - - # check hidden on-demand - return true if self.project.traversable_entity?(self.name, "Domain") + return true if self.allow_list + return false if self.deny_list # if we didnt match the above and we were asked, let's not allow it false diff --git a/lib/entities/email_address.rb b/lib/entities/email_address.rb index c5ed78df0..59c07b572 100644 --- a/lib/entities/email_address.rb +++ b/lib/entities/email_address.rb @@ -29,11 +29,8 @@ def enrichment_tasks ### SCOPING ### def scoped?(conditions={}) - return true if self.seed - return false if self.hidden - - # check hidden on-demand - return false unless self.project.traversable_entity?(self.name.split("@").last, "Domain") + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/entities/file_hash.rb b/lib/entities/file_hash.rb index 3a014f97b..ade91d775 100644 --- a/lib/entities/file_hash.rb +++ b/lib/entities/file_hash.rb @@ -25,8 +25,8 @@ def supported_hash_types end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/entities/github_account.rb b/lib/entities/github_account.rb index d199187b8..298aad366 100644 --- a/lib/entities/github_account.rb +++ b/lib/entities/github_account.rb @@ -20,8 +20,8 @@ def enrichment_tasks end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/entities/github_repository.rb b/lib/entities/github_repository.rb index fa54211c4..3e7765efd 100644 --- a/lib/entities/github_repository.rb +++ b/lib/entities/github_repository.rb @@ -16,8 +16,8 @@ def validate_entity end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/entities/ip_address.rb b/lib/entities/ip_address.rb index 30f00ff35..b62da8a66 100644 --- a/lib/entities/ip_address.rb +++ b/lib/entities/ip_address.rb @@ -31,8 +31,8 @@ def enrichment_tasks ### SCOPING ### def scoped?(conditions={}) - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list # if we have aliases and they're all false, we don't really want this thing if self.aliases.count > 0 diff --git a/lib/entities/mailserver.rb b/lib/entities/mailserver.rb index c5f73e389..5a26ec716 100644 --- a/lib/entities/mailserver.rb +++ b/lib/entities/mailserver.rb @@ -25,11 +25,8 @@ def enrichment_tasks ### SCOPING ### def scoped?(conditions={}) - return true if self.seed - return false if self.hidden # hit our blacklist so definitely false - - # check hidden on-demand - return true if self.project.traversable_entity?(parse_domain_name(self.name), "Domain") + return true if self.allow_list + return false if self.deny_list # if we didnt match the above and we were asked, it's false false diff --git a/lib/entities/nameserver.rb b/lib/entities/nameserver.rb index 2f9e6f5cf..10ed36657 100644 --- a/lib/entities/nameserver.rb +++ b/lib/entities/nameserver.rb @@ -25,11 +25,8 @@ def enrichment_tasks ### SCOPING ### def scoped?(conditions={}) - return true if self.seed - return false if self.hidden # hit our blacklist so definitely false - - # check hidden on-demand - return true if self.project.traversable_entity?(parse_domain_name(self.name), "Domain") + return true if self.allow_list + return false if self.deny_list # if we didnt match the above and we were asked, it's false false diff --git a/lib/entities/net_block.rb b/lib/entities/net_block.rb index ba20e50ae..2743d1dbb 100644 --- a/lib/entities/net_block.rb +++ b/lib/entities/net_block.rb @@ -27,8 +27,8 @@ def enrichment_tasks ### SCOPING ### def scoped?(conditions={}) - return true if self.seed - return false if self.hidden # hit our blacklist so definitely false + return true if self.allow_list + return false if self.deny_list our_ip = self.name.split("/").first our_route = self.name.split("/").last.to_i diff --git a/lib/entities/network_service.rb b/lib/entities/network_service.rb index 74608fe46..aa2e93b75 100644 --- a/lib/entities/network_service.rb +++ b/lib/entities/network_service.rb @@ -23,8 +23,8 @@ def enrichment_tasks end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/entities/organization.rb b/lib/entities/organization.rb index 0785abed0..c698caee1 100644 --- a/lib/entities/organization.rb +++ b/lib/entities/organization.rb @@ -20,10 +20,10 @@ def enrichment_tasks end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list - true + false end end diff --git a/lib/entities/person.rb b/lib/entities/person.rb index 43bd8dd7f..3cb16f47d 100644 --- a/lib/entities/person.rb +++ b/lib/entities/person.rb @@ -20,8 +20,8 @@ def detail_string end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/entities/phone_number.rb b/lib/entities/phone_number.rb index 1624a7680..047a5a961 100644 --- a/lib/entities/phone_number.rb +++ b/lib/entities/phone_number.rb @@ -20,8 +20,8 @@ def detail_string end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/entities/physical_location.rb b/lib/entities/physical_location.rb index 3b2049211..ca25bbc9e 100644 --- a/lib/entities/physical_location.rb +++ b/lib/entities/physical_location.rb @@ -17,8 +17,8 @@ def validate_entity end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/entities/software_package.rb b/lib/entities/software_package.rb index bf0b85796..87f632625 100644 --- a/lib/entities/software_package.rb +++ b/lib/entities/software_package.rb @@ -20,8 +20,8 @@ def detail_string end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/entities/ssl_certificate.rb b/lib/entities/ssl_certificate.rb index d4fdade71..9a898a985 100644 --- a/lib/entities/ssl_certificate.rb +++ b/lib/entities/ssl_certificate.rb @@ -31,10 +31,9 @@ def detail_string ### SCOPING ### def scoped?(conditions={}) - return true if self.seed - return false if self.hidden # hit our blacklist so definitely false - - # if we didnt match the above and we were asked, it's still true + return true if self.allow_list + return false if self.deny_list + true end diff --git a/lib/entities/string.rb b/lib/entities/string.rb index 896834a10..9cebd9559 100644 --- a/lib/entities/string.rb +++ b/lib/entities/string.rb @@ -20,8 +20,8 @@ def enrichment_tasks end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/entities/uri.rb b/lib/entities/uri.rb index 2f45638b5..56524797e 100644 --- a/lib/entities/uri.rb +++ b/lib/entities/uri.rb @@ -42,17 +42,9 @@ def detail_string ### SCOPING ### def scoped?(conditions={}) - return true if self.seed - return true if self.hidden + return true if self.allow_list + return false if self.deny_list - ## CHECK IF DOMAIN NAME IS KNOWN - # ================================= - hostname = URI.parse(self.name).host.to_s - if !hostname.is_ip_address? - domain_name = parse_domain_name(hostname) - return false unless self.project.traversable_entity?(domain_name, "Domain") - end - # if we didnt match the above and we were asked, it's still true true end diff --git a/lib/entities/web_account.rb b/lib/entities/web_account.rb index 298544d54..fb9360a6e 100644 --- a/lib/entities/web_account.rb +++ b/lib/entities/web_account.rb @@ -18,8 +18,8 @@ def validate_entity end def scoped? - return true if self.seed - return false if self.hidden + return true if self.allow_list + return false if self.deny_list true end diff --git a/lib/system/dns_helpers.rb b/lib/system/dns_helpers.rb new file mode 100644 index 000000000..e4f47a1da --- /dev/null +++ b/lib/system/dns_helpers.rb @@ -0,0 +1,66 @@ +module Intrigue +module System +module DnsHelpers + +### + ### TODO ... system helper + ### + def parse_domain_name(record) + return nil unless record + split_tld = parse_tld(record).split(".") + if (split_tld.last == "com" || split_tld.last == "net") && split_tld.count > 1 # handle cases like amazonaws.com, netlify.com + length = split_tld.count + else + length = split_tld.count + 1 + end + + record.split(".").last(length).join(".") + end + + + ### + ### TODO ... system helper + ### + # assumes we get a dns name of arbitrary length + def parse_tld(record) + return nil unless record + + # first check if we're not long enough to split, just returning the domain + return record if record && record.split(".").length < 2 + + # Make sure we're comparing bananas to bananas + record = "#{record}".downcase + + # now one at a time, check all known TLDs and match + begin + raw_suffix_list = File.open("#{$intrigue_basedir}/data/public_suffix_list.clean.txt").read.split("\n") + suffix_list = raw_suffix_list.map{|l| "#{l.downcase}".strip } + + # first find all matches + matches = [] + suffix_list.each do |s| + if record =~ /.*#{Regexp.escape(s.strip)}$/i # we have a match .. + matches << s.strip + end + end + + # then find the longest match + if matches.count > 0 + longest_match = matches.sort_by{|x| x.split(".").length + x.split(".").last.length }.last + puts "Returning: #{longest_match} from #{matches}" + return longest_match + end + + rescue Errno::ENOENT => e + _log_error "Unable to locate public suffix list, failing to check / create domain for #{lookup_name}" + return nil + end + + # unknown tld + record + end + + +end +end +end \ No newline at end of file diff --git a/lib/tasks/base.rb b/lib/tasks/base.rb index e1aaa6e9f..e1613e23c 100644 --- a/lib/tasks/base.rb +++ b/lib/tasks/base.rb @@ -98,27 +98,15 @@ def perform(task_result_id) end end - ### ENRICHMENT TASK SPECIFICS + ### POST ENRICHMENT + # # Now, if this is an enrichment task, we want to do some things if @task_result.task_name =~ /^enrich\/.*/ - - # MARK ENTITY AS ENRICHED - _log "Marking entity as enriched!" - @entity.enriched = true - - ### HANDLE SCOPING - We should have enough info now that enrichment is complete - # check the scoped? method on the entity and set the attribute - if @entity.scoped? - _log "Marking entity as scoped!" - @entity.scoped = true - else - _log "Marking entity as unscoped!" - @entity.scoped = false - end - - # Save it! - _log "Saving entity!" - @entity.save_changes + + # entity should now be enriched! + # + @entity.enriched = true + @entity.save_changes # MACHINE LAUNCH (ONLY IF WE ARE ATTACHED TO A MACHINE) # if this is part of a scan and we're in depth @@ -132,6 +120,7 @@ def perform(task_result_id) end machine.start(@entity, @task_result) + else @task_result.log "No machine configured for #{@entity.name}!" end diff --git a/lib/tasks/dns_transfer_zone.rb b/lib/tasks/dns_transfer_zone.rb index e1e28697d..ff8bee23f 100644 --- a/lib/tasks/dns_transfer_zone.rb +++ b/lib/tasks/dns_transfer_zone.rb @@ -46,18 +46,7 @@ def run zt.server = nameserver zone = zt.transfer(domain_name) - ############################################ - ### Old Issue ### - ########################################### - # create an issue to track this - #_create_issue({ - # name: "DNS Zone (AXFR) Transfer Enabled", - # type: "dns_zone_transfer", - # severity: 4, - # status: "confirmed", - # description: "Zone transfer on #{domain_name} using #{nameserver} resulted in leak of #{zone.count} records.", - # details: { records: zone.map{|r| r.name.to_s } } - # }) + description = "Zone transfer on #{domain_name} using #{nameserver} resulted in leak of #{zone.count} records.", ############################################ ### New Issue ### diff --git a/lib/tasks/gitrob.rb b/lib/tasks/gitrob.rb index 3062dd194..2e1547549 100644 --- a/lib/tasks/gitrob.rb +++ b/lib/tasks/gitrob.rb @@ -88,26 +88,10 @@ def run next if (f["Description"] == "Contains word: credential" && f["FilePath"] =~ /credential.html/i ) next if (f["Description"] == "Contains word: password" && f["FilePath"] =~ /password.html/i ) - ############################################ - ### Old Issue ### - ########################################### - # _create_issue({ - # name: "Suspicious #{f["Action"]} Commit Found In Github Repository", - # type: "suspicious_commit", - # severity: 4, - # status: "potential", - # description: "A suspicious commit was found in a public Github repository.\n" + - # "Repository URL: #{f['RepositoryUrl']}\n" + - # "Commit Author: #{f["CommitAuthor"]}\n" + - # "Commit Message #{f["CommitMessage"]}\n" + - # "Details: #{f["Action"]} #{f["Description"]} at #{f["FileUrl"]}\n\n#{f["Comment"]}", - # details: f.merge({uri: "#{f["CommitUrl"]}"}) - # }) - ############################################ ### New Issue ### ########################################### - _create_linked_issue({ + _create_linked_issue("suspicious_commit", { name: "Suspicious #{f["Action"]} Commit Found In Github Repository", detailed_description: "A suspicious commit was found in a public Github repository.\n" + "Repository URL: #{f['RepositoryUrl']}\n" + diff --git a/lib/tasks/helpers/dns.rb b/lib/tasks/helpers/dns.rb index 799389e75..19b9ef7df 100644 --- a/lib/tasks/helpers/dns.rb +++ b/lib/tasks/helpers/dns.rb @@ -3,6 +3,7 @@ module Task module Dns include Intrigue::Task::Generic + include Intrigue::System::DnsHelpers # parse_tld, parse_domain_name def create_unscoped_dns_entity_from_string(s) create_dns_entity_from_string(s, nil, true) @@ -27,54 +28,6 @@ def create_dns_entity_from_string(s, alias_entity=nil, unscoped=false) end end - def parse_domain_name(record) - split_tld = parse_tld(record).split(".") - if (split_tld.last == "com" || split_tld.last == "net") && split_tld.count > 1 # handle cases like amazonaws.com, netlify.com - length = split_tld.count - else - length = split_tld.count + 1 - end - - record.split(".").last(length).join(".") - end - - # assumes we get a dns name of arbitrary length - def parse_tld(record) - - # first check if we're not long enough to split, just returning the domain - return record if record.split(".").length < 2 - - # Make sure we're comparing bananas to bananas - record = record.downcase - - # now one at a time, check all known TLDs and match - begin - raw_suffix_list = File.open("#{$intrigue_basedir}/data/public_suffix_list.clean.txt").read.split("\n") - suffix_list = raw_suffix_list.map{|l| "#{l.downcase}".strip } - - # first find all matches - matches = [] - suffix_list.each do |s| - if record =~ /.*#{Regexp.escape(s.strip)}$/i # we have a match .. - matches << s.strip - end - end - - # then find the longest match - if matches.count > 0 - longest_match = matches.sort_by{|x| x.split(".").length }.sort_by{|x| x.length }.last - return longest_match - end - - rescue Errno::ENOENT => e - _log_error "Unable to locate public suffix list, failing to check / create domain" - return nil - end - - # unknown tld - record - end - # Check for wildcard DNS def check_wildcard(suffix) _log "Checking for wildcards on #{suffix}." diff --git a/lib/tasks/helpers/issue.rb b/lib/tasks/helpers/issue.rb index 95d1d51bf..c4ebc4f3c 100644 --- a/lib/tasks/helpers/issue.rb +++ b/lib/tasks/helpers/issue.rb @@ -143,6 +143,9 @@ def _check_requests_for_mixed_content(uri, requests) resource_url = req["url"] + # skip data + return if uri =~ /^data:.*$/ + # skip this for anything other than hostnames hostname = URI(resource_url).hostname return if hostname =~ ipv4_regex || hostname =~ /ipv6_regex/ diff --git a/lib/tasks/helpers/services.rb b/lib/tasks/helpers/services.rb index 346d35d74..aa309a44f 100644 --- a/lib/tasks/helpers/services.rb +++ b/lib/tasks/helpers/services.rb @@ -123,7 +123,6 @@ def _create_network_service_entity(ip_entity,port_num,protocol="tcp",generic_det end entity_details = { - "scoped" => true, # always scope in "name" => uri, "uri" => uri, "service" => prefix @@ -141,7 +140,6 @@ def _create_network_service_entity(ip_entity,port_num,protocol="tcp",generic_det name = "#{h.name.strip}:#{port_num}" entity_details = { - "scoped" => true, # always scope in "name" => name, "service" => service } @@ -168,7 +166,6 @@ def _create_network_service_entity(ip_entity,port_num,protocol="tcp",generic_det name = "#{h.name.strip}:#{port_num}" entity_details = { - "scoped" => true, # always scope in "name" => name, "service" => service } diff --git a/lib/tasks/helpers/web_content.rb b/lib/tasks/helpers/web_content.rb index 60c2c1312..2adacfb56 100644 --- a/lib/tasks/helpers/web_content.rb +++ b/lib/tasks/helpers/web_content.rb @@ -7,7 +7,16 @@ def extract_and_fingerprint_scripts(script_list, host) components = [] script_list.each do |s| - uri = URI.parse(s) + # skip anything that's not http + next unless s =~ /^http/ + + begin + uri = URI.parse(s) + rescue URI::InvalidURIError => e + @task_result.logger.log "Unable to parse improperly formatted URI: #{s}" + next # unable to parse + end + next unless uri.host && uri.port && uri.scheme =~ /^http/ ### ### Determine who's hosting