Skip to content

Commit

Permalink
adjust scoping to use deny/accept lists, and push the individual deci…
Browse files Browse the repository at this point in the history
…sions further down to enities
  • Loading branch information
jcran committed Jun 17, 2020
1 parent 2883ace commit eaa8393
Show file tree
Hide file tree
Showing 43 changed files with 356 additions and 351 deletions.
6 changes: 3 additions & 3 deletions Rakefile
Original file line number Diff line number Diff line change
Expand Up @@ -60,17 +60,17 @@ task :update do
end

def _get_global_entities
uri = "https://app.intrigue.io/api/global/entities?key=#{$intrigueio_api_key}"
uri = "https://app.intrigue.io/api/system/entities/global/entities/?key=#{$intrigueio_api_key}"
begin
puts "[+] Making request for global entities"
puts "[+] Making request for global entities!"
response = RestClient.get(uri)

# handle missing data
return -1 unless response && response.length > 0

j = JSON.parse(response.body)
rescue JSON::ParserError => e
puts "[+] Unable to parse bootstrap json"
puts "[+] Unable to parse json: #{e}"
return -1
end
j
Expand Down
3 changes: 3 additions & 0 deletions app/all.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
# must be brought in first, system should be skipped as a directive
require_relative "routes/system"

# useful to bring in generic helper functions
require_relative '../lib/system/dns_helpers'

require_relative "routes/analysis"
require_relative "routes/entities"
require_relative "routes/issues"
Expand Down
150 changes: 0 additions & 150 deletions app/models/global_entity.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,156 +11,6 @@ def validate
validates_unique([:namespace, :type, :name])
end

def self.parse_domain_name(record)
return nil unless record
split_tld = parse_tld(record).split(".")
if (split_tld.last == "com" || split_tld.last == "net") && split_tld.count > 1 # handle cases like amazonaws.com, netlify.com
length = split_tld.count
else
length = split_tld.count + 1
end

record.split(".").last(length).join(".")
end

# assumes we get a dns name of arbitrary length
def self.parse_tld(record)
return nil unless record

# first check if we're not long enough to split, just returning the domain
return record if record && record.split(".").length < 2

# Make sure we're comparing bananas to bananas
record = "#{record}".downcase

# now one at a time, check all known TLDs and match
begin
raw_suffix_list = File.open("#{$intrigue_basedir}/data/public_suffix_list.clean.txt").read.split("\n")
suffix_list = raw_suffix_list.map{|l| "#{l.downcase}".strip }

# first find all matches
matches = []
suffix_list.each do |s|
if record =~ /.*#{Regexp.escape(s.strip)}$/i # we have a match ..
matches << s.strip
end
end

# then find the longest match
if matches.count > 0
longest_match = matches.sort_by{|x| x.split(".").length }.last
return longest_match
end

rescue Errno::ENOENT => e
_log_error "Unable to locate public suffix list, failing to check / create domain for #{lookup_name}"
return nil
end

# unknown tld
record
end


#TODO .. this method should only be called if we don't already have the entity in our project
def self.traversable?(entity_type, entity_name, project)

# by default things are not traversable
out = false

# first check to see if we know about this exact entity (type matters too)
puts "Looking for global entity: #{entity_type} #{entity_name}"
global_entity = Intrigue::Model::GlobalEntity.first(:name => entity_name, :type => entity_type)

# If we know it exists, is it in our project (cool) or someone else (no traverse!)
if global_entity
puts "Global entity found: #{entity_type} #{entity_name}!"

# we need to have a namespace to validate against
if project.allowed_namespaces
project.allowed_namespaces.each do |namespace|
# if the entity's' namespace matches one of ours, we're good!
if global_entity.namespace.downcase == namespace.downcase
puts "Matches our namespace!"
return true # we can immediately return
end
end
else
puts "No Allowed Namespaces, but this is a claimed entity!"
return false
end

else
puts "No Global entity found!"
end

# okay so if we made it this far, we may or may not have a matching entiy, so now
# we need to find if it matches based on regex... since entities can have a couple
# different forms (uri, dns_record, domain, etc)

# TODAY this only works on domains... and things that have a domain (like a uri)

# then check each for a match
found_entity = nil

## Okay let's get smart by getting down to the smallest searchable unit first
searchable_name = nil

#include Intrigue::Task::Dns # useful for parsing domain names

if entity_type == "Domain"
# this should have gotten caught above...
searchable_name = parse_domain_name(entity_name)
elsif entity_type == "DnsRecord"
searchable_name = parse_domain_name(entity_name)
elsif entity_type == "EmailAddress"
searchable_name = parse_domain_name(entity_name.split("@").last)
elsif entity_type == "Nameserver"
searchable_name = parse_domain_name(entity_name)
elsif entity_type == "Uri"
searchable_name = parse_domain_name(URI.parse(entity_name).host)
end

# now form the query, taking into acount the filter if we can
if searchable_name
found_entity = Intrigue::Model::GlobalEntity.first(:type => "Domain", :name => searchable_name)
else
global_entities = Intrigue::Model::GlobalEntity.all

global_entities.each do |ge|
# this needs a couple (3) cases:
# 1) case where we're an EXACT match (ey.com)
# 2) case where we're a subdomain of an exception domain (x.ey.com)
# 3) case where we're a uri and should match an exception domain (https://ey.com)
# none of these cases should match the case: jcpenney.com
if (entity_name.downcase =~ /^#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/ ||
entity_name.downcase =~ /^.*\.#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/ ||
entity_name.downcase =~ /^https?:\/\/#{Regexp.escape(ge.name.downcase)}(:[0-9]*)?$/)

#okay we found it... now we need to check if it's an allowed project
found_entity = ge
end
end
end

if found_entity && !project.allowed_namespaces.empty? # now lets check if we have an allowance for it

(project.allowed_namespaces || []).each do |namespace|
if found_entity.namespace.downcase == namespace.downcase # Good!
return true
end
end

out = false
else # we never found it or we don't care (no namespaces)!
out = true
end

#puts "Result for: #{entity_type} #{entity_name} in project #{project.name}: #{out}"

out
end

def self.load_global_namespace(data)
(data["entities"] || []).each do |x|
Intrigue::Model::GlobalEntity.update_or_create(:name => x["name"], :type => x["type"], :namespace => x["namespace"])
Expand Down
Loading

0 comments on commit eaa8393

Please sign in to comment.