diff --git a/Gemfile.lock b/Gemfile.lock index 579491c9..20195e33 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,7 +2,7 @@ PATH remote: . specs: synapse (0.14.3) - aws-sdk (~> 1.39) + aws-sdk (~> 2) docker-api (~> 1.7) logging (~> 1.8) zk (~> 1.9.4) @@ -17,35 +17,38 @@ GEM thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) addressable (2.4.0) - aws-sdk (1.66.0) - aws-sdk-v1 (= 1.66.0) - aws-sdk-v1 (1.66.0) - json (~> 1.4) - nokogiri (>= 1.4.4) + aws-sdk (2.9.2) + aws-sdk-resources (= 2.9.2) + aws-sdk-core (2.9.2) + aws-sigv4 (~> 1.0) + jmespath (~> 1.0) + aws-sdk-resources (2.9.2) + aws-sdk-core (= 2.9.2) + aws-sigv4 (1.0.0) coderay (1.1.0) crack (0.4.3) safe_yaml (~> 1.0.0) diff-lcs (1.2.5) - docker-api (1.29.0) + docker-api (1.33.3) excon (>= 0.38.0) json - excon (0.49.0) + excon (0.55.0) factory_girl (4.7.0) activesupport (>= 3.0.0) + ffi (1.9.10) ffi (1.9.10-java) hashdiff (0.2.3) i18n (0.7.0) - json (1.8.3) - little-plugger (1.1.3) + jmespath (1.3.1) + json (1.8.6) + json (1.8.6-java) + little-plugger (1.1.4) logging (1.8.2) little-plugger (>= 1.1.3) multi_json (>= 1.8.4) method_source (0.8.2) - mini_portile2 (2.0.0) minitest (5.9.0) - multi_json (1.11.2) - nokogiri (1.6.7.2) - mini_portile2 (~> 2.0.0.rc2) + multi_json (1.12.1) pry (0.10.3) coderay (~> 1.1.0) method_source (~> 0.8.1) @@ -72,9 +75,12 @@ GEM rspec-support (3.1.2) safe_yaml (1.0.4) slop (3.6.0) + slyphon-log4j (1.2.15) + slyphon-zookeeper_jar (3.3.5-java) spoon (0.0.4) ffi thread_safe (0.3.5) + thread_safe (0.3.5-java) timecop (0.8.1) tzinfo (1.2.2) thread_safe (~> 0.1) @@ -85,6 +91,9 @@ GEM zk (1.9.6) zookeeper (~> 1.4.0) zookeeper (1.4.11) + zookeeper (1.4.11-java) + slyphon-log4j (= 1.2.15) + slyphon-zookeeper_jar (= 3.3.5) PLATFORMS java @@ -101,4 +110,4 @@ DEPENDENCIES webmock BUNDLED WITH - 1.11.2 + 1.14.3 diff --git a/README.md b/README.md index 764c9272..fc262825 100644 --- a/README.md +++ b/README.md @@ -259,13 +259,18 @@ It takes the following options: ##### AWS EC2 tags ##### This watcher retrieves a list of Amazon EC2 instances that have a tag -with particular value using the AWS API. +with particular value using the AWS API, or if you specify a tag_hash instead, +it will filter the intersection of those tags and their values. + +"tag_name"=>"tag_value" will be appended to any values in "tag_hash" as +a concession to backwards compatibility It takes the following options: * `method`: ec2tag * `tag_name`: the name of the tag to inspect. As per the AWS docs, this is case-sensitive. * `tag_value`: the value to match on. Case-sensitive. +* `tag_hash`: A hash map of names=> values to filter instances by tags with. Additionally, you MUST supply [`backend_port_override`](#backend_port_override) in the service configuration as this watcher does not know which port the @@ -415,7 +420,7 @@ For example: - "balance roundrobin" services: service1: - discovery: + discovery: method: "zookeeper" path: "/nerve/services/service1" hosts: diff --git a/lib/synapse/service_watcher/ec2tag.rb b/lib/synapse/service_watcher/ec2tag.rb index c14cac07..aae27033 100644 --- a/lib/synapse/service_watcher/ec2tag.rb +++ b/lib/synapse/service_watcher/ec2tag.rb @@ -10,15 +10,24 @@ def start region = @discovery['aws_region'] || ENV['AWS_REGION'] log.info "Connecting to EC2 region: #{region}" - @ec2 = AWS::EC2.new( + @ec2 = Aws::EC2::Resource.new( region: region, access_key_id: @discovery['aws_access_key_id'] || ENV['AWS_ACCESS_KEY_ID'], secret_access_key: @discovery['aws_secret_access_key'] || ENV['AWS_SECRET_ACCESS_KEY'] ) @check_interval = @discovery['check_interval'] || 15.0 + # Backwards compatibility for single-tag ec2tag watcher - log.info "synapse: ec2tag watcher looking for instances " + - "tagged with #{@discovery['tag_name']}=#{@discovery['tag_value']}" + if @discovery.key?('tag_name') + if @discovery.key?('tag_hash') + @discovery['tag_hash'][@discovery['tag_name']] = @discovery['tag_value'] + else + @discovery['tag_hash'] = {@discovery['tag_name']=>@discovery['tag_value']} + end + end + tag_filter_list = @discovery['tag_hash'].collect { |t, v| "#{t}=#{v}" } + log.info "synapse: ec2tag watcher looking for instances tagged with " + + tag_filter_list.join(' AND ') @watcher = Thread.new { watch } end @@ -29,10 +38,9 @@ def validate_discovery_opts # Required, via options only. raise ArgumentError, "invalid discovery method #{@discovery['method']}" \ unless @discovery['method'] == 'ec2tag' - raise ArgumentError, "aws tag name is required for service #{@name}" \ - unless @discovery['tag_name'] - raise ArgumentError, "aws tag value required for service #{@name}" \ - unless @discovery['tag_value'] + raise ArgumentError, "'tag_hash' or 'tag_name' and 'tag_value' is required for service #{@name}" \ + unless not @discovery.key?('tag_hash') \ + or (not @discovery.key?('tag_name') && !@discovery.key?('tag_value')) # As we're only looking up instances with hostnames/IPs, need to # be explicitly told which port the service we're balancing for listens on. @@ -78,30 +86,28 @@ def sleep_until_next_check(start_time) end def discover_instances - AWS.memoize do - instances = instances_with_tags(@discovery['tag_name'], @discovery['tag_value']) - - new_backends = [] - - # choice of private_dns_name, dns_name, private_ip_address or - # ip_address, for now, just stick with the private fields. - instances.each do |instance| - new_backends << { - 'name' => instance.private_dns_name, - 'host' => instance.private_ip_address, - } - end + instances = instances_with_tags(@discovery['tag_hash']) + + new_backends = [] - new_backends + # choice of private_dns_name, dns_name, private_ip_address or + # ip_address, for now, just stick with the private fields. + instances.each do |instance| + new_backends << { + 'name' => instance.private_dns_name, + 'host' => instance.private_ip_address, + } end + + new_backends end - def instances_with_tags(tag_name, tag_value) - @ec2.instances - .tagged(tag_name) - .tagged_values(tag_value) - .select { |i| i.status == :running } + def instances_with_tags(tag_hash) + filters = [{ name: 'instance-state-name', values: ['running'] }] + tag_hash.each do |tag, val| + filters << { name: "tag:#{tag}", values: ["#{val}"]} + end + @ec2.instances(filters: filters) end end end - diff --git a/spec/lib/synapse/service_watcher_ec2tags_spec.rb b/spec/lib/synapse/service_watcher_ec2tags_spec.rb index ef17d534..d54e2136 100644 --- a/spec/lib/synapse/service_watcher_ec2tags_spec.rb +++ b/spec/lib/synapse/service_watcher_ec2tags_spec.rb @@ -58,6 +58,22 @@ def fake_address } end + let(:new_config) do + { 'name' => 'ec2tagtest', + 'haproxy' => { + 'port' => '8080', + 'server_port_override' => '8081' + }, + "discovery" => { + "method" => "ec2tag", + "tag_hash" => {"fuNNy_tag_name"=> "funkyTagValue"}, + "aws_region" => 'eu-test-1', + "aws_access_key_id" => 'ABCDEFGHIJKLMNOPQRSTU', + "aws_secret_access_key" => 'verylongfakekeythatireallyneedtogenerate' + } + } + end + before(:all) do # Clean up ENV so we don't inherit any actual AWS config. %w[AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION].each { |k| ENV.delete(k) } @@ -66,7 +82,7 @@ def fake_address before(:each) do # https://ruby.awsblog.com/post/Tx2SU6TYJWQQLC3/Stubbing-AWS-Responses # always returns empty results, so data may have to be faked. - AWS.stub! + Aws.config[:stub_responses] = true end def remove_discovery_arg(name) @@ -100,6 +116,12 @@ def munge_arg(name, new_value) expect { subject }.not_to raise_error end + it 'instantiates cleanly with new config' do + expect { + Synapse::ServiceWatcher::Ec2tagWatcher.new(new_config, mock_synapse) + }.not_to raise_error + end + context 'when missing arguments' do it 'does not break if aws_region is missing' do expect { @@ -172,8 +194,8 @@ def munge_arg(name, new_value) end context 'using the AWS API' do - let(:ec2_client) { double('AWS::EC2') } - let(:instance_collection) { double('AWS::EC2::InstanceCollection') } + let(:ec2_client) { double('Aws::EC2::Resource') } + let(:instance_collection) { double('Aws::Resources::Collection') } before do subject.ec2 = ec2_client @@ -187,11 +209,7 @@ def munge_arg(name, new_value) expect(subject.ec2).to receive(:instances).and_return(instance_collection) - expect(instance_collection).to receive(:tagged).with('foo').and_return(instance_collection) - expect(instance_collection).to receive(:tagged_values).with('bar').and_return(instance_collection) - expect(instance_collection).to receive(:select).and_return(instance_collection) - - subject.send(:instances_with_tags, 'foo', 'bar') + subject.send(:instances_with_tags, {'foo'=> 'bar'}) end end @@ -240,4 +258,3 @@ def munge_arg(name, new_value) end end end - diff --git a/synapse.gemspec b/synapse.gemspec index 84a430f0..c3d8cd16 100644 --- a/synapse.gemspec +++ b/synapse.gemspec @@ -21,7 +21,7 @@ Gem::Specification.new do |gem| gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) - gem.add_runtime_dependency "aws-sdk", "~> 1.39" + gem.add_runtime_dependency "aws-sdk", "~> 2" gem.add_runtime_dependency "docker-api", "~> 1.7" gem.add_runtime_dependency "zk", "~> 1.9.4" gem.add_runtime_dependency "logging", "~> 1.8"