Add Floating IP mass reassignment tests

This ensures that moving a large number of Floating IPs in one go
propagates in a reasonable amount of time.

README.md

@@ -20,11 +20,12 @@ These tests are run regularly against our public infrastructure as well as our i
 | **Custom Image**    | [test_custom_image_with_slug](./test_custom_image.py#L11)                        | custom   |
 |                     | [test_custom_image_with_uuid](./test_custom_image.py#L22)                        | custom   |
 |                     | [test_custom_image_with_uefi](./test_custom_image.py#L33)                        | custom   |
-| **Floating IP**     | [test_floating_ip_connectivity](./test_floating_ip.py#L14)                       | default  |
-|                     | [test_multiple_floating_ips](./test_floating_ip.py#L32)                          | default  |
-|                     | [test_floating_ip_stability](./test_floating_ip.py#L54)                          | default  |
-|                     | [test_floating_ip_failover](./test_floating_ip.py#L97)                           | default  |
-|                     | [test_floating_network](./test_floating_ip.py#L140)                              | default  |
+| **Floating IP**     | [test_floating_ip_connectivity](./test_floating_ip.py#L15)                       | default  |
+|                     | [test_multiple_floating_ips](./test_floating_ip.py#L33)                          | default  |
+|                     | [test_floating_ip_stability](./test_floating_ip.py#L55)                          | default  |
+|                     | [test_floating_ip_failover](./test_floating_ip.py#L98)                           | default  |
+|                     | [test_floating_network](./test_floating_ip.py#L141)                              | default  |
+|                     | [test_floating_ip_mass_failover](./test_floating_ip.py#L179)                     | default  |
 | **Load Balancer**   | [test_simple_tcp_load_balancer](./test_load_balancer.py#L24)                     | default  |
 |                     | [test_load_balancer_end_to_end](./test_load_balancer.py#L48)                     | default  |
 |                     | [test_multiple_listeners](./test_load_balancer.py#L81)                           | default  |

api.py

@@ -103,6 +103,9 @@ def request(self, method, url, *args, **kwargs):
 
         return super().request(method, url, *args, **kwargs)
 
+    def resources(self, path):
+        return self.get(f'{path}?tag:runner={RUNNER_ID}').json()
+
     def runner_resources(self):
         """ Returns all resources created by the current API token as part
         of an acceptance test.

resources.py

@@ -534,6 +534,32 @@ def resolve(self, name, version):
 
         return tuple(addrs)
 
+    def identify(self, address, servers=None):
+        """ Gets the SSH host key of the given address and compares it to the
+        servers found in the API. If there is a match, the server UUID is
+        returned.
+
+        """
+        key = self.output_of(
+            f'ssh-keyscan -t ed25519 {address} | cut -d " " -f 2-3')
+
+        if not key:
+            return None
+
+        for server in (servers or self.api.resources('/servers')):
+            if isinstance(server, dict):
+                keys = server['ssh_host_keys']
+            else:
+                if not server.ssh_host_keys:
+                    server.refresh()
+
+                keys = server.ssh_host_keys
+
+            if key in keys or ():
+                return server
+
+        return None
+
     def file_path_exists(self, path):
         """ Returns true if the given path exists. """
 

test_floating_ip.py

@@ -9,6 +9,7 @@
 """
 
 from util import in_parallel
+from util import retry_for
 
 
 def test_floating_ip_connectivity(prober, server, floating_ip):
@@ -173,3 +174,55 @@ def test_floating_network(prober, server, floating_network):
 
         # Make sure it is gone
         prober.ping(address, expect_failure=True)
+
+
+def test_floating_ip_mass_failover(prober, create_server, server_group,
+                                   create_floating_ip):
+    """ Floating IP addresses can be re-assigned en masse. """
+
+    # Create two server to assign the Floating IPs
+    s1, s2 = in_parallel(create_server, instances=(
+        {'name': 's1', 'server_groups': [server_group.uuid]},
+        {'name': 's2', 'server_groups': [server_group.uuid]},
+    ))
+
+    # Create, assign, and configure 15 Floating IPs
+    ips = [
+        create_floating_ip(
+            ip_version=4,
+            server=s1.uuid
+        ) for _ in range(15)
+    ]
+
+    for ip in ips:
+        s1.configure_floating_ip(ip)
+        s2.configure_floating_ip(ip)
+
+    # Identify the associated servers by looking at the SSH host keys
+    def assert_assigned_to_server(server):
+        identified = in_parallel(
+            prober.identify, [(ip.address, (s1, s2)) for ip in ips])
+
+        for s in identified:
+            assert server is s
+
+    retry_for(seconds=15).or_fail(
+        lambda: assert_assigned_to_server(s1),
+        msg=f'Failed to assign addresses to {s1.name}',
+    )
+
+    # Move the Floating IPs to s2 at once
+    in_parallel(lambda ip: ip.assign(s2), ips)
+
+    retry_for(seconds=15).or_fail(
+        lambda: assert_assigned_to_server(s2),
+        msg=f'Failed to assign addresses to {s2.name}',
+    )
+
+    # Move the Floating IPs back to s1
+    in_parallel(lambda ip: ip.assign(s1), ips)
+
+    retry_for(seconds=15).or_fail(
+        lambda: assert_assigned_to_server(s1),
+        msg=f'Failed to assign addresses to {s1.name}',
+    )

util.py

@@ -32,6 +32,7 @@
 from psutil import Process
 from testinfra.backend.paramiko import ParamikoBackend
 from types import SimpleNamespace
+from typing import Generator
 from uuid import uuid4
 from warnings import warn
 
@@ -162,12 +163,18 @@ def in_parallel(factory, instances=None, count=None):
 
     Or if the function doesn't take any arguments:
 
-        s1, s2 = in_parallel(some_function, count=3)
+        s1, s2 = in_parallel(some_function, count=2)
 
     """
 
     def create(instance):
-        return factory(**instance)
+        if isinstance(instance, dict):
+            return factory(**instance)
+
+        if isinstance(instance, (list, tuple, Generator)):
+            return factory(*instance)
+
+        return factory(instance)
 
     # Require instances or a count, disallow both together
     assert instances or count