diff --git a/Rakefile b/Rakefile
index f5afd563f..b174c18df 100644
--- a/Rakefile
+++ b/Rakefile
@@ -60,9 +60,9 @@ task :update do
end
def _get_global_entities
- uri = "https://app.intrigue.io/api/global/entities?key=#{$intrigueio_api_key}"
+ uri = "https://app.intrigue.io/api/system/entities/global/entities/?key=#{$intrigueio_api_key}"
begin
- puts "[+] Making request for global entities"
+ puts "[+] Making request for global entities!"
response = RestClient.get(uri)
# handle missing data
@@ -70,7 +70,7 @@ def _get_global_entities
j = JSON.parse(response.body)
rescue JSON::ParserError => e
- puts "[+] Unable to parse bootstrap json"
+ puts "[+] Unable to parse json: #{e}"
return -1
end
j
diff --git a/app/all.rb b/app/all.rb
index 60ec4b785..78829716d 100644
--- a/app/all.rb
+++ b/app/all.rb
@@ -3,6 +3,9 @@
# must be brought in first, system should be skipped as a directive
require_relative "routes/system"
+# useful to bring in generic helper functions
+require_relative '../lib/system/dns_helpers'
+
require_relative "routes/analysis"
require_relative "routes/entities"
require_relative "routes/issues"
diff --git a/app/models/global_entity.rb b/app/models/global_entity.rb
index b2b9fd78b..8e7ccf9ba 100644
--- a/app/models/global_entity.rb
+++ b/app/models/global_entity.rb
@@ -11,156 +11,6 @@ def validate
validates_unique([:namespace, :type, :name])
end
- def self.parse_domain_name(record)
- return nil unless record
- split_tld = parse_tld(record).split(".")
- if (split_tld.last == "com" || split_tld.last == "net") && split_tld.count > 1 # handle cases like amazonaws.com, netlify.com
- length = split_tld.count
- else
- length = split_tld.count + 1
- end
-
- record.split(".").last(length).join(".")
- end
-
- # assumes we get a dns name of arbitrary length
- def self.parse_tld(record)
- return nil unless record
-
- # first check if we're not long enough to split, just returning the domain
- return record if record && record.split(".").length < 2
-
- # Make sure we're comparing bananas to bananas
- record = "#{record}".downcase
-
- # now one at a time, check all known TLDs and match
- begin
- raw_suffix_list = File.open("#{$intrigue_basedir}/data/public_suffix_list.clean.txt").read.split("\n")
- suffix_list = raw_suffix_list.map{|l| "#{l.downcase}".strip }
-
- # first find all matches
- matches = []
- suffix_list.each do |s|
- if record =~ /.*#{Regexp.escape(s.strip)}$/i # we have a match ..
- matches << s.strip
- end
- end
-
- # then find the longest match
- if matches.count > 0
- longest_match = matches.sort_by{|x| x.split(".").length }.last
- return longest_match
- end
-
- rescue Errno::ENOENT => e
- _log_error "Unable to locate public suffix list, failing to check / create domain for #{lookup_name}"
- return nil
- end
-
- # unknown tld
- record
- end
-
-
- #TODO .. this method should only be called if we don't already have the entity in our project
- def self.traversable?(entity_type, entity_name, project)
-
- # by default things are not traversable
- out = false
-
- # first check to see if we know about this exact entity (type matters too)
- puts "Looking for global entity: #{entity_type} #{entity_name}"
- global_entity = Intrigue::Model::GlobalEntity.first(:name => entity_name, :type => entity_type)
-
- # If we know it exists, is it in our project (cool) or someone else (no traverse!)
- if global_entity
- puts "Global entity found: #{entity_type} #{entity_name}!"
-
- # we need to have a namespace to validate against
- if project.allowed_namespaces
- project.allowed_namespaces.each do |namespace|
- # if the entity's' namespace matches one of ours, we're good!
- if global_entity.namespace.downcase == namespace.downcase
- puts "Matches our namespace!"
- return true # we can immediately return
- end
- end
- else
- puts "No Allowed Namespaces, but this is a claimed entity!"
- return false
- end
-
- else
- puts "No Global entity found!"
- end
-
- # okay so if we made it this far, we may or may not have a matching entiy, so now
- # we need to find if it matches based on regex... since entities can have a couple
- # different forms (uri, dns_record, domain, etc)
-
- # TODAY this only works on domains... and things that have a domain (like a uri)
-
- # then check each for a match
- found_entity = nil
-
- ## Okay let's get smart by getting down to the smallest searchable unit first
- searchable_name = nil
-
- #include Intrigue::Task::Dns # useful for parsing domain names
-
- if entity_type == "Domain"
- # this should have gotten caught above...
- searchable_name = parse_domain_name(entity_name)
- elsif entity_type == "DnsRecord"
- searchable_name = parse_domain_name(entity_name)
- elsif entity_type == "EmailAddress"
- searchable_name = parse_domain_name(entity_name.split("@").last)
- elsif entity_type == "Nameserver"
- searchable_name = parse_domain_name(entity_name)
- elsif entity_type == "Uri"
- searchable_name = parse_domain_name(URI.parse(entity_name).host)
- end
-
- # now form the query, taking into acount the filter if we can
- if searchable_name
- found_entity = Intrigue::Model::GlobalEntity.first(:type => "Domain", :name => searchable_name)
- else
- global_entities = Intrigue::Model::GlobalEntity.all
-
- global_entities.each do |ge|
- # this needs a couple (3) cases:
- # 1) case where we're an EXACT match (ey.com)
- # 2) case where we're a subdomain of an exception domain (x.ey.com)
- # 3) case where we're a uri and should match an exception domain (https://ey.com)
- # none of these cases should match the case: jcpenney.com
- if (entity_name.downcase =~ /^#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/ ||
- entity_name.downcase =~ /^.*\.#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/ ||
- entity_name.downcase =~ /^https?:\/\/#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/)
-
- #okay we found it... now we need to check if it's an allowed project
- found_entity = ge
- end
- end
- end
-
- if found_entity && !project.allowed_namespaces.empty? # now lets check if we have an allowance for it
-
- (project.allowed_namespaces || []).each do |namespace|
- if found_entity.namespace.downcase == namespace.downcase # Good!
- return true
- end
- end
-
- out = false
- else # we never found it or we don't care (no namespaces)!
- out = true
- end
-
- #puts "Result for: #{entity_type} #{entity_name} in project #{project.name}: #{out}"
-
- out
- end
-
def self.load_global_namespace(data)
(data["entities"] || []).each do |x|
Intrigue::Model::GlobalEntity.update_or_create(:name => x["name"], :type => x["type"], :namespace => x["namespace"])
diff --git a/app/models/project.rb b/app/models/project.rb
index fe62555eb..a9131cd95 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -11,6 +11,7 @@ class Project < Sequel::Model
one_to_many :issues
include Intrigue::Model::Mixins::Handleable
+ include Intrigue::System::DnsHelpers
def validate
super
@@ -157,13 +158,107 @@ def export_applications_csv
out
end
- # Method gives us a true/false, depending on whether the entity is in an
- # exception list. Currently only used on project, but could be included
- # in task_result or scan_result. Note that they'd need the "additional_exception_list"
- # to be populated (automated by bootstrap)
- def traversable_entity?(entity_name, type_string)
+ def globally_traversable_entity?(entity_type, entity_name)
+
+ # by default things are not traversable
+ out = false
+
+ # first check to see if we know about this exact entity (type matters too)
+ puts "Looking for global entity: #{entity_type} #{entity_name}"
+ global_entity = Intrigue::Model::GlobalEntity.first(:name => entity_name, :type => entity_type)
+
+ # If we know it exists, is it in our project (cool) or someone else (no traverse!)
+ if global_entity
+ puts "Global entity found: #{entity_type} #{entity_name}!"
+
+ # we need to have a namespace to validate against
+ if self.allowed_namespaces
+ self.allowed_namespaces.each do |namespace|
+ # if the entity's' namespace matches one of ours, we're good!
+ if global_entity.namespace.downcase == namespace.downcase
+ puts "Matches our namespace!"
+ return true # we can immediately return
+ end
+ end
+ else
+ puts "No allowed namespaces, and this is a claimed entity but not a seed!"
+ return false
+ end
+
+ else
+ puts "No Global entity found, trying harder!"
+ end
+
+ # okay so if we made it this far, we may or may not have a matching entiy, so now
+ # we need to find if it matches based on regex... since entities can have a couple
+ # different forms (uri, dns_record, domain, etc)
+
+ # then check each for a match
+ found_entity = nil
+
+ ## Okay let's get smart by getting down to the smallest searchable unit first
+ searchable_name = nil
+
+ if entity_type == "Domain"
+ # this should have gotten caught above...
+ searchable_name = parse_domain_name(entity_name)
+ elsif entity_type == "DnsRecord"
+ searchable_name = parse_domain_name(entity_name)
+ elsif entity_type == "EmailAddress"
+ searchable_name = parse_domain_name(entity_name.split("@").last)
+ elsif entity_type == "Nameserver"
+ searchable_name = parse_domain_name(entity_name)
+ elsif entity_type == "Uri"
+ searchable_name = parse_domain_name(URI.parse(entity_name).host)
+ end
+
+ # now form the query, taking into acount the filter if we can
+ if searchable_name
+ found_entity = Intrigue::Model::GlobalEntity.first(:type => "Domain", :name => searchable_name)
+ else
+ global_entities = Intrigue::Model::GlobalEntity.all
+
+ global_entities.each do |ge|
+ # this needs a couple (3) cases:
+ # 1) case where we're an EXACT match (ey.com)
+ # 2) case where we're a subdomain of an exception domain (x.ey.com)
+ # 3) case where we're a uri and should match an exception domain (https://ey.com)
+ # none of these cases should match the case: jcpenney.com
+ if (entity_name.downcase =~ /^#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/ ||
+ entity_name.downcase =~ /^.*\.#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/ ||
+ entity_name.downcase =~ /^https?:\/\/#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/)
+
+ #okay we found it... now we need to check if it's an allowed project
+ found_entity = ge
+ end
+ end
+ end
+
+ if found_entity && !self.allowed_namespaces.empty? # now lets check if we have an allowance for it
- # if it's an explicit seed, it's always traversable
+ (self.allowed_namespaces || []).each do |namespace|
+ if found_entity.namespace.downcase == namespace.downcase # Good!
+ return true
+ end
+ end
+
+ out = false
+ else # we never found it or we don't care (no namespaces)!
+ out = true
+ end
+
+ #puts "Result for: #{entity_type} #{entity_name} in project #{project.name}: #{out}"
+
+ out
+ end
+
+
+ ###
+ ### Use this when you wan to scope in stuff
+ ###
+ def allow_list_entity?(type_string, entity_name)
+
+ # if it's an explicit seed, it's whitelisted
return true if seed_entity?(type_string,entity_name)
### CHECK OUR SEED ENTITIES TO SEE IF THE TEXT MATCHES A DOMAIN
@@ -175,44 +270,113 @@ def traversable_entity?(entity_name, type_string)
"Intrigue::Entity::EmailAddress",
"Intrigue::Entity::Organization",
"Intrigue::Entity::Nameserver",
- "Intrigue::Entity::Uri"
+ "Intrigue::Entity::Uri"
]
- # skip anything else!!
- return true unless scope_check_entity_types.include? "Intrigue::Entity::#{type_string}"
+ # skip anything else thats not verifiable!!
+ return false unless scope_check_entity_types.include? "Intrigue::Entity::#{type_string}"
- seeds.each do |s|
+ seeds.each do |s|
if entity_name =~ /[\.\s\@]#{Regexp.escape(s.name)}/i
- #puts "matched a seed, returning true"
- return true
+ return true # matches a seed pattern, it's whitelisted
end
end
- # Check standard exceptions (hardcoded list) first if we show up here (and we werent' a seed), we should skip
+ # Check standard exceptions (hardcoded list) first if we
+ # show up here (and we werent' a seed), we should skip
if use_standard_exceptions
if standard_no_traverse?(entity_name, type_string)
- #puts 'Matched a standard exception, returning false'
+ #puts 'Matched a standard exception, not whitelisted'
return false
end
end
- # unless we can verify it against a domain, it's probably not that helpful to do this
- # just assume we can't go any further
+ # now check the global intel
verifiable_entity_types = ["DnsRecord", "Domain", "EmailAddress", "NameServer" "Uri"]
if verifiable_entity_types.include? type_string
# if we don't have a list, safe to return false now, otherwise proceed to
# additional exceptions which are provided as an attribute on the object
- unless Intrigue::Model::GlobalEntity.traversable?(type_string, entity_name, self)
+ unless globally_traversable_entity?(type_string, entity_name)
puts 'Global intel says not traversable, returning false'
return false
end
end
- #puts "Defaulting to not traversable (Whitelist approach!)"
+ ###
+ # Defaulting to not traversable (Whitelist approach!)
+ ###
+ false
+ end
+
+ ###
+ ### Use this when you wan to scope out stuff based on rules or global intel
+ ###
+ def deny_list_entity?(type_string, entity_name)
+
+ # if it's an explicit seed, it's not blacklisted
+ return false if seed_entity?(type_string,entity_name)
+
+ ### CHECK OUR SEED ENTITIES TO SEE IF THE TEXT MATCHES A DOMAIN
+ ######################################################
+ # if it matches an explicit seed pattern
+ scope_check_entity_types = [
+ "Intrigue::Entity::DnsRecord",
+ "Intrigue::Entity::Domain",
+ "Intrigue::Entity::EmailAddress",
+ "Intrigue::Entity::Organization",
+ "Intrigue::Entity::Nameserver",
+ "Intrigue::Entity::Uri"
+ ]
+
+ # not blacklisted if we're not one of the check types
+ return false unless scope_check_entity_types.include? "Intrigue::Entity::#{type_string}"
+
+ seeds.each do |s|
+ if entity_name =~ /[\.\s\@]#{Regexp.escape(s.name)}/i
+ return false # not blacklisted if we're matching a seed derivative
+ end
+ end
+
+ # Check standard exceptions (hardcoded list) first if we
+ # show up here (and we werent' a seed), we should skip
+ if use_standard_exceptions
+ if standard_no_traverse?(entity_name, type_string)
+ return true # matched a blacklist
+ end
+ end
+
+ # now check the global intel
+ verifiable_entity_types = ["DnsRecord", "Domain", "EmailAddress", "NameServer" "Uri"]
+ if verifiable_entity_types.include? type_string
+ # if we don't have a list, safe to return false now, otherwise proceed to
+ # additional exceptions which are provided as an attribute on the object
+ if !globally_traversable_entity?(type_string, entity_name)
+ puts 'Global intel says not traversable so we are blacklisted, returning true'
+ return true
+ end
+ end
+
+ ###
+ # we made it this far, not blacklisted!
+ ###
false
end
+ # Method gives us a true/false, depending on whether the entity is in an
+ # exception list. Currently only used on project, but could be included
+ # in task_result or scan_result. Note that they'd need the "additional_exception_list"
+ # to be populated (automated by bootstrap)
+ ###
+ ### DEFAULTS TO FALSE!!! WHITELIST APPROACH
+ ###
+ def traversable_entity?(type_string, entity_name)
+ return true if allow_list_entity?(type_string, entity_name)
+ return false if deny_list_entity?(type_string, entity_name)
+ # otherwise, perimissive
+ true
+ end
+
# TODO - there must be a cleaner way?
def get_option(option_name)
opt = options.detect{|h| h[option_name] } if options
diff --git a/app/views/entities/detail.erb b/app/views/entities/detail.erb
index d19a38d52..bcb080eeb 100644
--- a/app/views/entities/detail.erb
+++ b/app/views/entities/detail.erb
@@ -10,7 +10,8 @@
Scoped: <%= @entity.scoped %>
Enriched: <%= @entity.enriched %>
-Hidden: <%= @entity.hidden %>
+Deny List Status: <%= @entity.deny_list %>
+Allow List Status: <%= @entity.allow_list %>
diff --git a/core.rb b/core.rb
index 4c566b38b..ec491f3da 100644
--- a/core.rb
+++ b/core.rb
@@ -53,6 +53,9 @@
#require 'pry-byebug'
require 'logger'
+# disable annoying redis messages
+Redis.exists_returns_integer = false
+
#
# Simple configuration check to ensure we have configs in place
def sanity_check_system
diff --git a/db/060_entity_allow_deny_list.rb b/db/060_entity_allow_deny_list.rb
new file mode 100644
index 000000000..574a24bc8
--- /dev/null
+++ b/db/060_entity_allow_deny_list.rb
@@ -0,0 +1,10 @@
+Sequel.migration do
+ change do
+
+ alter_table(:entities) do
+ add_column :allow_list, TrueClass
+ add_column :deny_list, TrueClass
+ end
+
+ end
+end
\ No newline at end of file
diff --git a/lib/all.rb b/lib/all.rb
index 289fde48f..58194f4f0 100644
--- a/lib/all.rb
+++ b/lib/all.rb
@@ -19,6 +19,11 @@
require_relative 'system/helpers'
include Intrigue::System::Helpers
+# Intrigue System-wide Helpers (both app and backend)
+require_relative 'system/dns_helpers'
+#include Intrigue::System::DnsHelpers
+
+
# Intrigue Export Format
require_relative 'system/json_data_export_file'
diff --git a/lib/entities/analytics_id.rb b/lib/entities/analytics_id.rb
index 47a1bfede..0f1831863 100644
--- a/lib/entities/analytics_id.rb
+++ b/lib/entities/analytics_id.rb
@@ -34,8 +34,8 @@ def supported_hash_types
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true # otherwise just default to true
end
diff --git a/lib/entities/autonomous_system.rb b/lib/entities/autonomous_system.rb
index aed5bba80..83f0f7d3e 100644
--- a/lib/entities/autonomous_system.rb
+++ b/lib/entities/autonomous_system.rb
@@ -16,9 +16,9 @@ def validate_entity
end
def scoped?
- return true if self.seed
- return false if self.hidden
- true # otherwise just default to true
+ return true if self.allow_list
+ return false if self.deny_list
+ false # otherwise false
end
end
diff --git a/lib/entities/aws_credential.rb b/lib/entities/aws_credential.rb
index 437ec239a..34f756199 100644
--- a/lib/entities/aws_credential.rb
+++ b/lib/entities/aws_credential.rb
@@ -33,8 +33,8 @@ def validate_entity
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true # otherwise just default to true
end
diff --git a/lib/entities/aws_iam_account.rb b/lib/entities/aws_iam_account.rb
index cf3bd32b4..213e5065d 100644
--- a/lib/entities/aws_iam_account.rb
+++ b/lib/entities/aws_iam_account.rb
@@ -19,8 +19,8 @@ def detail_string
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true # otherwise just default to true
end
diff --git a/lib/entities/aws_region.rb b/lib/entities/aws_region.rb
index 87412c158..aa71546c9 100644
--- a/lib/entities/aws_region.rb
+++ b/lib/entities/aws_region.rb
@@ -40,8 +40,8 @@ def enrichment_tasks
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true # otherwise just default to true
end
diff --git a/lib/entities/aws_s3_bucket.rb b/lib/entities/aws_s3_bucket.rb
index 10d85507e..b502c88e6 100644
--- a/lib/entities/aws_s3_bucket.rb
+++ b/lib/entities/aws_s3_bucket.rb
@@ -24,8 +24,8 @@ def enrichment_tasks
end
def scoped?(conditions={})
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true # otherwise just default to true
end
diff --git a/lib/entities/credential.rb b/lib/entities/credential.rb
index d31d02aa9..376955adf 100644
--- a/lib/entities/credential.rb
+++ b/lib/entities/credential.rb
@@ -18,8 +18,8 @@ def validate_entity
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true # otherwise just default to true
end
diff --git a/lib/entities/dns_record.rb b/lib/entities/dns_record.rb
index 4c808b3fe..f3a87b3ca 100644
--- a/lib/entities/dns_record.rb
+++ b/lib/entities/dns_record.rb
@@ -30,15 +30,15 @@ def enrichment_tasks
### SCOPING
###
def scoped?(conditions={})
- return true if self.seed
- return false if self.hidden # hit our blacklist so definitely false
+ return true if self.allow_list
+ return false if self.deny_list
#self.project.seeds.each do |s|
# return true if self.name =~ /[\.\s\@]#{Regexp.escape(s.name)}/i
#end
# check hidden on-demand
- return false unless self.project.traversable_entity?(parse_domain_name(self.name), "Domain")
+ return false unless self.project.traversable_entity?("Domain", parse_domain_name(self.name))
# if we didnt match the above and we were asked, default to true
true
diff --git a/lib/entities/domain.rb b/lib/entities/domain.rb
index cfca7a025..4c26f8f52 100644
--- a/lib/entities/domain.rb
+++ b/lib/entities/domain.rb
@@ -29,11 +29,8 @@ def enrichment_tasks
### SCOPING
###
def scoped?(conditions={})
- return true if self.seed
- return false if self.hidden
-
- # check hidden on-demand
- return true if self.project.traversable_entity?(self.name, "Domain")
+ return true if self.allow_list
+ return false if self.deny_list
# if we didnt match the above and we were asked, let's not allow it
false
diff --git a/lib/entities/email_address.rb b/lib/entities/email_address.rb
index c5ed78df0..59c07b572 100644
--- a/lib/entities/email_address.rb
+++ b/lib/entities/email_address.rb
@@ -29,11 +29,8 @@ def enrichment_tasks
### SCOPING
###
def scoped?(conditions={})
- return true if self.seed
- return false if self.hidden
-
- # check hidden on-demand
- return false unless self.project.traversable_entity?(self.name.split("@").last, "Domain")
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/entities/file_hash.rb b/lib/entities/file_hash.rb
index 3a014f97b..ade91d775 100644
--- a/lib/entities/file_hash.rb
+++ b/lib/entities/file_hash.rb
@@ -25,8 +25,8 @@ def supported_hash_types
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/entities/github_account.rb b/lib/entities/github_account.rb
index d199187b8..298aad366 100644
--- a/lib/entities/github_account.rb
+++ b/lib/entities/github_account.rb
@@ -20,8 +20,8 @@ def enrichment_tasks
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/entities/github_repository.rb b/lib/entities/github_repository.rb
index fa54211c4..3e7765efd 100644
--- a/lib/entities/github_repository.rb
+++ b/lib/entities/github_repository.rb
@@ -16,8 +16,8 @@ def validate_entity
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/entities/ip_address.rb b/lib/entities/ip_address.rb
index 30f00ff35..b62da8a66 100644
--- a/lib/entities/ip_address.rb
+++ b/lib/entities/ip_address.rb
@@ -31,8 +31,8 @@ def enrichment_tasks
### SCOPING
###
def scoped?(conditions={})
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
# if we have aliases and they're all false, we don't really want this thing
if self.aliases.count > 0
diff --git a/lib/entities/mailserver.rb b/lib/entities/mailserver.rb
index c5f73e389..5a26ec716 100644
--- a/lib/entities/mailserver.rb
+++ b/lib/entities/mailserver.rb
@@ -25,11 +25,8 @@ def enrichment_tasks
### SCOPING
###
def scoped?(conditions={})
- return true if self.seed
- return false if self.hidden # hit our blacklist so definitely false
-
- # check hidden on-demand
- return true if self.project.traversable_entity?(parse_domain_name(self.name), "Domain")
+ return true if self.allow_list
+ return false if self.deny_list
# if we didnt match the above and we were asked, it's false
false
diff --git a/lib/entities/nameserver.rb b/lib/entities/nameserver.rb
index 2f9e6f5cf..10ed36657 100644
--- a/lib/entities/nameserver.rb
+++ b/lib/entities/nameserver.rb
@@ -25,11 +25,8 @@ def enrichment_tasks
### SCOPING
###
def scoped?(conditions={})
- return true if self.seed
- return false if self.hidden # hit our blacklist so definitely false
-
- # check hidden on-demand
- return true if self.project.traversable_entity?(parse_domain_name(self.name), "Domain")
+ return true if self.allow_list
+ return false if self.deny_list
# if we didnt match the above and we were asked, it's false
false
diff --git a/lib/entities/net_block.rb b/lib/entities/net_block.rb
index ba20e50ae..2743d1dbb 100644
--- a/lib/entities/net_block.rb
+++ b/lib/entities/net_block.rb
@@ -27,8 +27,8 @@ def enrichment_tasks
### SCOPING
###
def scoped?(conditions={})
- return true if self.seed
- return false if self.hidden # hit our blacklist so definitely false
+ return true if self.allow_list
+ return false if self.deny_list
our_ip = self.name.split("/").first
our_route = self.name.split("/").last.to_i
diff --git a/lib/entities/network_service.rb b/lib/entities/network_service.rb
index 74608fe46..aa2e93b75 100644
--- a/lib/entities/network_service.rb
+++ b/lib/entities/network_service.rb
@@ -23,8 +23,8 @@ def enrichment_tasks
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/entities/organization.rb b/lib/entities/organization.rb
index 0785abed0..c698caee1 100644
--- a/lib/entities/organization.rb
+++ b/lib/entities/organization.rb
@@ -20,10 +20,10 @@ def enrichment_tasks
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
- true
+ false
end
end
diff --git a/lib/entities/person.rb b/lib/entities/person.rb
index 43bd8dd7f..3cb16f47d 100644
--- a/lib/entities/person.rb
+++ b/lib/entities/person.rb
@@ -20,8 +20,8 @@ def detail_string
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/entities/phone_number.rb b/lib/entities/phone_number.rb
index 1624a7680..047a5a961 100644
--- a/lib/entities/phone_number.rb
+++ b/lib/entities/phone_number.rb
@@ -20,8 +20,8 @@ def detail_string
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/entities/physical_location.rb b/lib/entities/physical_location.rb
index 3b2049211..ca25bbc9e 100644
--- a/lib/entities/physical_location.rb
+++ b/lib/entities/physical_location.rb
@@ -17,8 +17,8 @@ def validate_entity
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/entities/software_package.rb b/lib/entities/software_package.rb
index bf0b85796..87f632625 100644
--- a/lib/entities/software_package.rb
+++ b/lib/entities/software_package.rb
@@ -20,8 +20,8 @@ def detail_string
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/entities/ssl_certificate.rb b/lib/entities/ssl_certificate.rb
index d4fdade71..9a898a985 100644
--- a/lib/entities/ssl_certificate.rb
+++ b/lib/entities/ssl_certificate.rb
@@ -31,10 +31,9 @@ def detail_string
### SCOPING
###
def scoped?(conditions={})
- return true if self.seed
- return false if self.hidden # hit our blacklist so definitely false
-
- # if we didnt match the above and we were asked, it's still true
+ return true if self.allow_list
+ return false if self.deny_list
+
true
end
diff --git a/lib/entities/string.rb b/lib/entities/string.rb
index 896834a10..9cebd9559 100644
--- a/lib/entities/string.rb
+++ b/lib/entities/string.rb
@@ -20,8 +20,8 @@ def enrichment_tasks
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/entities/uri.rb b/lib/entities/uri.rb
index 2f45638b5..56524797e 100644
--- a/lib/entities/uri.rb
+++ b/lib/entities/uri.rb
@@ -42,17 +42,9 @@ def detail_string
### SCOPING
###
def scoped?(conditions={})
- return true if self.seed
- return true if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
- ## CHECK IF DOMAIN NAME IS KNOWN
- # =================================
- hostname = URI.parse(self.name).host.to_s
- if !hostname.is_ip_address?
- domain_name = parse_domain_name(hostname)
- return false unless self.project.traversable_entity?(domain_name, "Domain")
- end
-
# if we didnt match the above and we were asked, it's still true
true
end
diff --git a/lib/entities/web_account.rb b/lib/entities/web_account.rb
index 298544d54..fb9360a6e 100644
--- a/lib/entities/web_account.rb
+++ b/lib/entities/web_account.rb
@@ -18,8 +18,8 @@ def validate_entity
end
def scoped?
- return true if self.seed
- return false if self.hidden
+ return true if self.allow_list
+ return false if self.deny_list
true
end
diff --git a/lib/system/dns_helpers.rb b/lib/system/dns_helpers.rb
new file mode 100644
index 000000000..e4f47a1da
--- /dev/null
+++ b/lib/system/dns_helpers.rb
@@ -0,0 +1,66 @@
+module Intrigue
+module System
+module DnsHelpers
+
+###
+ ### TODO ... system helper
+ ###
+ def parse_domain_name(record)
+ return nil unless record
+ split_tld = parse_tld(record).split(".")
+ if (split_tld.last == "com" || split_tld.last == "net") && split_tld.count > 1 # handle cases like amazonaws.com, netlify.com
+ length = split_tld.count
+ else
+ length = split_tld.count + 1
+ end
+
+ record.split(".").last(length).join(".")
+ end
+
+
+ ###
+ ### TODO ... system helper
+ ###
+ # assumes we get a dns name of arbitrary length
+ def parse_tld(record)
+ return nil unless record
+
+ # first check if we're not long enough to split, just returning the domain
+ return record if record && record.split(".").length < 2
+
+ # Make sure we're comparing bananas to bananas
+ record = "#{record}".downcase
+
+ # now one at a time, check all known TLDs and match
+ begin
+ raw_suffix_list = File.open("#{$intrigue_basedir}/data/public_suffix_list.clean.txt").read.split("\n")
+ suffix_list = raw_suffix_list.map{|l| "#{l.downcase}".strip }
+
+ # first find all matches
+ matches = []
+ suffix_list.each do |s|
+ if record =~ /.*#{Regexp.escape(s.strip)}$/i # we have a match ..
+ matches << s.strip
+ end
+ end
+
+ # then find the longest match
+ if matches.count > 0
+ longest_match = matches.sort_by{|x| x.split(".").length + x.split(".").last.length }.last
+ puts "Returning: #{longest_match} from #{matches}"
+ return longest_match
+ end
+
+ rescue Errno::ENOENT => e
+ _log_error "Unable to locate public suffix list, failing to check / create domain for #{lookup_name}"
+ return nil
+ end
+
+ # unknown tld
+ record
+ end
+
+
+end
+end
+end
\ No newline at end of file
diff --git a/lib/tasks/base.rb b/lib/tasks/base.rb
index e1aaa6e9f..e1613e23c 100644
--- a/lib/tasks/base.rb
+++ b/lib/tasks/base.rb
@@ -98,27 +98,15 @@ def perform(task_result_id)
end
end
- ### ENRICHMENT TASK SPECIFICS
+ ### POST ENRICHMENT
+ #
# Now, if this is an enrichment task, we want to do some things
if @task_result.task_name =~ /^enrich\/.*/
-
- # MARK ENTITY AS ENRICHED
- _log "Marking entity as enriched!"
- @entity.enriched = true
-
- ### HANDLE SCOPING - We should have enough info now that enrichment is complete
- # check the scoped? method on the entity and set the attribute
- if @entity.scoped?
- _log "Marking entity as scoped!"
- @entity.scoped = true
- else
- _log "Marking entity as unscoped!"
- @entity.scoped = false
- end
-
- # Save it!
- _log "Saving entity!"
- @entity.save_changes
+
+ # entity should now be enriched!
+ #
+ @entity.enriched = true
+ @entity.save_changes
# MACHINE LAUNCH (ONLY IF WE ARE ATTACHED TO A MACHINE)
# if this is part of a scan and we're in depth
@@ -132,6 +120,7 @@ def perform(task_result_id)
end
machine.start(@entity, @task_result)
+
else
@task_result.log "No machine configured for #{@entity.name}!"
end
diff --git a/lib/tasks/dns_transfer_zone.rb b/lib/tasks/dns_transfer_zone.rb
index e1e28697d..ff8bee23f 100644
--- a/lib/tasks/dns_transfer_zone.rb
+++ b/lib/tasks/dns_transfer_zone.rb
@@ -46,18 +46,7 @@ def run
zt.server = nameserver
zone = zt.transfer(domain_name)
- ############################################
- ### Old Issue ###
- ###########################################
- # create an issue to track this
- #_create_issue({
- # name: "DNS Zone (AXFR) Transfer Enabled",
- # type: "dns_zone_transfer",
- # severity: 4,
- # status: "confirmed",
- # description: "Zone transfer on #{domain_name} using #{nameserver} resulted in leak of #{zone.count} records.",
- # details: { records: zone.map{|r| r.name.to_s } }
- # })
+
description = "Zone transfer on #{domain_name} using #{nameserver} resulted in leak of #{zone.count} records.",
############################################
### New Issue ###
diff --git a/lib/tasks/gitrob.rb b/lib/tasks/gitrob.rb
index 3062dd194..2e1547549 100644
--- a/lib/tasks/gitrob.rb
+++ b/lib/tasks/gitrob.rb
@@ -88,26 +88,10 @@ def run
next if (f["Description"] == "Contains word: credential" && f["FilePath"] =~ /credential.html/i )
next if (f["Description"] == "Contains word: password" && f["FilePath"] =~ /password.html/i )
- ############################################
- ### Old Issue ###
- ###########################################
- # _create_issue({
- # name: "Suspicious #{f["Action"]} Commit Found In Github Repository",
- # type: "suspicious_commit",
- # severity: 4,
- # status: "potential",
- # description: "A suspicious commit was found in a public Github repository.\n" +
- # "Repository URL: #{f['RepositoryUrl']}\n" +
- # "Commit Author: #{f["CommitAuthor"]}\n" +
- # "Commit Message #{f["CommitMessage"]}\n" +
- # "Details: #{f["Action"]} #{f["Description"]} at #{f["FileUrl"]}\n\n#{f["Comment"]}",
- # details: f.merge({uri: "#{f["CommitUrl"]}"})
- # })
-
############################################
### New Issue ###
###########################################
- _create_linked_issue({
+ _create_linked_issue("suspicious_commit", {
name: "Suspicious #{f["Action"]} Commit Found In Github Repository",
detailed_description: "A suspicious commit was found in a public Github repository.\n" +
"Repository URL: #{f['RepositoryUrl']}\n" +
diff --git a/lib/tasks/helpers/dns.rb b/lib/tasks/helpers/dns.rb
index 799389e75..19b9ef7df 100644
--- a/lib/tasks/helpers/dns.rb
+++ b/lib/tasks/helpers/dns.rb
@@ -3,6 +3,7 @@ module Task
module Dns
include Intrigue::Task::Generic
+ include Intrigue::System::DnsHelpers # parse_tld, parse_domain_name
def create_unscoped_dns_entity_from_string(s)
create_dns_entity_from_string(s, nil, true)
@@ -27,54 +28,6 @@ def create_dns_entity_from_string(s, alias_entity=nil, unscoped=false)
end
end
- def parse_domain_name(record)
- split_tld = parse_tld(record).split(".")
- if (split_tld.last == "com" || split_tld.last == "net") && split_tld.count > 1 # handle cases like amazonaws.com, netlify.com
- length = split_tld.count
- else
- length = split_tld.count + 1
- end
-
- record.split(".").last(length).join(".")
- end
-
- # assumes we get a dns name of arbitrary length
- def parse_tld(record)
-
- # first check if we're not long enough to split, just returning the domain
- return record if record.split(".").length < 2
-
- # Make sure we're comparing bananas to bananas
- record = record.downcase
-
- # now one at a time, check all known TLDs and match
- begin
- raw_suffix_list = File.open("#{$intrigue_basedir}/data/public_suffix_list.clean.txt").read.split("\n")
- suffix_list = raw_suffix_list.map{|l| "#{l.downcase}".strip }
-
- # first find all matches
- matches = []
- suffix_list.each do |s|
- if record =~ /.*#{Regexp.escape(s.strip)}$/i # we have a match ..
- matches << s.strip
- end
- end
-
- # then find the longest match
- if matches.count > 0
- longest_match = matches.sort_by{|x| x.split(".").length }.sort_by{|x| x.length }.last
- return longest_match
- end
-
- rescue Errno::ENOENT => e
- _log_error "Unable to locate public suffix list, failing to check / create domain"
- return nil
- end
-
- # unknown tld
- record
- end
-
# Check for wildcard DNS
def check_wildcard(suffix)
_log "Checking for wildcards on #{suffix}."
diff --git a/lib/tasks/helpers/issue.rb b/lib/tasks/helpers/issue.rb
index 95d1d51bf..c4ebc4f3c 100644
--- a/lib/tasks/helpers/issue.rb
+++ b/lib/tasks/helpers/issue.rb
@@ -143,6 +143,9 @@ def _check_requests_for_mixed_content(uri, requests)
resource_url = req["url"]
+ # skip data
+ return if uri =~ /^data:.*$/
+
# skip this for anything other than hostnames
hostname = URI(resource_url).hostname
return if hostname =~ ipv4_regex || hostname =~ /ipv6_regex/
diff --git a/lib/tasks/helpers/services.rb b/lib/tasks/helpers/services.rb
index 346d35d74..aa309a44f 100644
--- a/lib/tasks/helpers/services.rb
+++ b/lib/tasks/helpers/services.rb
@@ -123,7 +123,6 @@ def _create_network_service_entity(ip_entity,port_num,protocol="tcp",generic_det
end
entity_details = {
- "scoped" => true, # always scope in
"name" => uri,
"uri" => uri,
"service" => prefix
@@ -141,7 +140,6 @@ def _create_network_service_entity(ip_entity,port_num,protocol="tcp",generic_det
name = "#{h.name.strip}:#{port_num}"
entity_details = {
- "scoped" => true, # always scope in
"name" => name,
"service" => service
}
@@ -168,7 +166,6 @@ def _create_network_service_entity(ip_entity,port_num,protocol="tcp",generic_det
name = "#{h.name.strip}:#{port_num}"
entity_details = {
- "scoped" => true, # always scope in
"name" => name,
"service" => service
}
diff --git a/lib/tasks/helpers/web_content.rb b/lib/tasks/helpers/web_content.rb
index 60c2c1312..2adacfb56 100644
--- a/lib/tasks/helpers/web_content.rb
+++ b/lib/tasks/helpers/web_content.rb
@@ -7,7 +7,16 @@ def extract_and_fingerprint_scripts(script_list, host)
components = []
script_list.each do |s|
- uri = URI.parse(s)
+ # skip anything that's not http
+ next unless s =~ /^http/
+
+ begin
+ uri = URI.parse(s)
+ rescue URI::InvalidURIError => e
+ @task_result.logger.log "Unable to parse improperly formatted URI: #{s}"
+ next # unable to parse
+ end
+
next unless uri.host && uri.port && uri.scheme =~ /^http/
###
### Determine who's hosting