Adding tag_hash functionality to ec2tag watcher.

Gemfile.lock

@@ -2,7 +2,7 @@ PATH
   remote: .
   specs:
     synapse (0.14.3)
-      aws-sdk (~> 1.39)
+      aws-sdk (~> 2)
       docker-api (~> 1.7)
       logging (~> 1.8)
       zk (~> 1.9.4)
@@ -17,35 +17,38 @@ GEM
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     addressable (2.4.0)
-    aws-sdk (1.66.0)
-      aws-sdk-v1 (= 1.66.0)
-    aws-sdk-v1 (1.66.0)
-      json (~> 1.4)
-      nokogiri (>= 1.4.4)
+    aws-sdk (2.9.2)
+      aws-sdk-resources (= 2.9.2)
+    aws-sdk-core (2.9.2)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.9.2)
+      aws-sdk-core (= 2.9.2)
+    aws-sigv4 (1.0.0)
     coderay (1.1.0)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.2.5)
-    docker-api (1.29.0)
+    docker-api (1.33.3)
       excon (>= 0.38.0)
       json
-    excon (0.49.0)
+    excon (0.55.0)
     factory_girl (4.7.0)
       activesupport (>= 3.0.0)
+    ffi (1.9.10)
     ffi (1.9.10-java)
     hashdiff (0.2.3)
     i18n (0.7.0)
-    json (1.8.3)
-    little-plugger (1.1.3)
+    jmespath (1.3.1)
+    json (1.8.6)
+    json (1.8.6-java)
+    little-plugger (1.1.4)
     logging (1.8.2)
       little-plugger (>= 1.1.3)
       multi_json (>= 1.8.4)
     method_source (0.8.2)
-    mini_portile2 (2.0.0)
     minitest (5.9.0)
-    multi_json (1.11.2)
-    nokogiri (1.6.7.2)
-      mini_portile2 (~> 2.0.0.rc2)
+    multi_json (1.12.1)
     pry (0.10.3)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -72,9 +75,12 @@ GEM
     rspec-support (3.1.2)
     safe_yaml (1.0.4)
     slop (3.6.0)
+    slyphon-log4j (1.2.15)
+    slyphon-zookeeper_jar (3.3.5-java)
     spoon (0.0.4)
       ffi
     thread_safe (0.3.5)
+    thread_safe (0.3.5-java)
     timecop (0.8.1)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
@@ -85,6 +91,9 @@ GEM
     zk (1.9.6)
       zookeeper (~> 1.4.0)
     zookeeper (1.4.11)
+    zookeeper (1.4.11-java)
+      slyphon-log4j (= 1.2.15)
+      slyphon-zookeeper_jar (= 3.3.5)
 
 PLATFORMS
   java
@@ -101,4 +110,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   1.11.2
+   1.14.3

README.md

@@ -259,13 +259,18 @@ It takes the following options:
 ##### AWS EC2 tags #####
 
 This watcher retrieves a list of Amazon EC2 instances that have a tag
-with particular value using the AWS API.
+with particular value using the AWS API, or if you specify a tag_hash instead,
+it will filter the intersection of those tags and their values.
+
+"tag_name"=>"tag_value" will be appended to any values in "tag_hash" as
+a concession to backwards compatibility
 It takes the following options:
 
 * `method`: ec2tag
 * `tag_name`: the name of the tag to inspect. As per the AWS docs,
   this is case-sensitive.
 * `tag_value`: the value to match on. Case-sensitive.
+* `tag_hash`: A hash map of names=> values to filter instances by tags with. 
 
 Additionally, you MUST supply [`backend_port_override`](#backend_port_override)
 in the service configuration as this watcher does not know which port the
@@ -415,7 +420,7 @@ For example:
    - "balance  roundrobin"
  services:
   service1:
-   discovery: 
+   discovery:
     method: "zookeeper"
     path:  "/nerve/services/service1"
     hosts:

lib/synapse/service_watcher/ec2tag.rb

@@ -10,15 +10,24 @@ def start
       region = @discovery['aws_region'] || ENV['AWS_REGION']
       log.info "Connecting to EC2 region: #{region}"
 
-      @ec2 = AWS::EC2.new(
+      @ec2 = Aws::EC2::Resource.new(
         region:            region,
         access_key_id:     @discovery['aws_access_key_id']     || ENV['AWS_ACCESS_KEY_ID'],
         secret_access_key: @discovery['aws_secret_access_key'] || ENV['AWS_SECRET_ACCESS_KEY'] )
 
       @check_interval = @discovery['check_interval'] || 15.0
+      # Backwards compatibility for single-tag ec2tag watcher
 
-      log.info "synapse: ec2tag watcher looking for instances " +
-        "tagged with #{@discovery['tag_name']}=#{@discovery['tag_value']}"
+      if @discovery.key?('tag_name')
+        if @discovery.key?('tag_hash')
+          @discovery['tag_hash'][@discovery['tag_name']] = @discovery['tag_value']
+        else
+          @discovery['tag_hash'] = {@discovery['tag_name']=>@discovery['tag_value']}
+        end
+      end
+      tag_filter_list =  @discovery['tag_hash'].collect { |t, v| "#{t}=#{v}" }
+      log.info "synapse: ec2tag watcher looking for instances tagged with " +
+        tag_filter_list.join(' AND ')
 
       @watcher = Thread.new { watch }
     end
@@ -29,10 +38,9 @@ def validate_discovery_opts
       # Required, via options only.
       raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
         unless @discovery['method'] == 'ec2tag'
-      raise ArgumentError, "aws tag name is required for service #{@name}" \
-        unless @discovery['tag_name']
-      raise ArgumentError, "aws tag value required for service #{@name}" \
-        unless @discovery['tag_value']
+      raise ArgumentError, "'tag_hash' or 'tag_name' and 'tag_value' is required for service #{@name}" \
+        unless not @discovery.key?('tag_hash') \
+            or (not @discovery.key?('tag_name') && [email protected]?('tag_value'))
 
       # As we're only looking up instances with hostnames/IPs, need to
       # be explicitly told which port the service we're balancing for listens on.
@@ -78,30 +86,28 @@ def sleep_until_next_check(start_time)
     end
 
     def discover_instances
-      AWS.memoize do
-        instances = instances_with_tags(@discovery['tag_name'], @discovery['tag_value'])
-
-        new_backends = []
-
-        # choice of private_dns_name, dns_name, private_ip_address or
-        # ip_address, for now, just stick with the private fields.
-        instances.each do |instance|
-          new_backends << {
-            'name' => instance.private_dns_name,
-            'host' => instance.private_ip_address,
-          }
-        end
+      instances = instances_with_tags(@discovery['tag_hash'])
+
+      new_backends = []
 
-        new_backends
+      # choice of private_dns_name, dns_name, private_ip_address or
+      # ip_address, for now, just stick with the private fields.
+      instances.each do |instance|
+        new_backends << {
+          'name' => instance.private_dns_name,
+          'host' => instance.private_ip_address,
+        }
       end
+
+      new_backends
     end
 
-    def instances_with_tags(tag_name, tag_value)
-      @ec2.instances
-        .tagged(tag_name)
-        .tagged_values(tag_value)
-        .select { |i| i.status == :running }
+    def instances_with_tags(tag_hash)
+      filters = [{ name: 'instance-state-name', values: ['running'] }]
+      tag_hash.each do |tag, val|
+        filters << { name: "tag:#{tag}", values: ["#{val}"]}
+      end
+      @ec2.instances(filters: filters)
     end
   end
 end
-

spec/lib/synapse/service_watcher_ec2tags_spec.rb

@@ -58,6 +58,22 @@ def fake_address
     }
   end
 
+  let(:new_config) do
+    { 'name' => 'ec2tagtest',
+      'haproxy' => {
+        'port' => '8080',
+        'server_port_override' => '8081'
+      },
+      "discovery" => {
+        "method" => "ec2tag",
+        "tag_hash"   => {"fuNNy_tag_name"=> "funkyTagValue"},
+        "aws_region" => 'eu-test-1',
+        "aws_access_key_id" => 'ABCDEFGHIJKLMNOPQRSTU',
+        "aws_secret_access_key" => 'verylongfakekeythatireallyneedtogenerate'
+      }
+    }
+  end
+
   before(:all) do
     # Clean up ENV so we don't inherit any actual AWS config.
     %w[AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION].each { |k| ENV.delete(k) }
@@ -66,7 +82,7 @@ def fake_address
   before(:each) do
     # https://ruby.awsblog.com/post/Tx2SU6TYJWQQLC3/Stubbing-AWS-Responses
     # always returns empty results, so data may have to be faked.
-    AWS.stub!
+    Aws.config[:stub_responses] = true
   end
 
   def remove_discovery_arg(name)
@@ -100,6 +116,12 @@ def munge_arg(name, new_value)
       expect { subject }.not_to raise_error
     end
 
+    it 'instantiates cleanly with new config' do
+      expect {
+        Synapse::ServiceWatcher::Ec2tagWatcher.new(new_config, mock_synapse)
+      }.not_to raise_error
+    end
+
     context 'when missing arguments' do
       it 'does not break if aws_region is missing' do
         expect {
@@ -172,8 +194,8 @@ def munge_arg(name, new_value)
     end
 
     context 'using the AWS API' do
-      let(:ec2_client) { double('AWS::EC2') }
-      let(:instance_collection) { double('AWS::EC2::InstanceCollection') }
+      let(:ec2_client) { double('Aws::EC2::Resource') }
+      let(:instance_collection) { double('Aws::Resources::Collection') }
 
       before do
         subject.ec2 = ec2_client
@@ -187,11 +209,7 @@ def munge_arg(name, new_value)
 
         expect(subject.ec2).to receive(:instances).and_return(instance_collection)
 
-        expect(instance_collection).to receive(:tagged).with('foo').and_return(instance_collection)
-        expect(instance_collection).to receive(:tagged_values).with('bar').and_return(instance_collection)
-        expect(instance_collection).to receive(:select).and_return(instance_collection)
-
-        subject.send(:instances_with_tags, 'foo', 'bar')
+        subject.send(:instances_with_tags, {'foo'=> 'bar'})
       end
     end
 
@@ -240,4 +258,3 @@ def munge_arg(name, new_value)
     end
   end
 end
-

synapse.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
 
-  gem.add_runtime_dependency "aws-sdk", "~> 1.39"
+  gem.add_runtime_dependency "aws-sdk", "~> 2"
   gem.add_runtime_dependency "docker-api", "~> 1.7"
   gem.add_runtime_dependency "zk", "~> 1.9.4"
   gem.add_runtime_dependency "logging", "~> 1.8"