Merge pull request #35 from cloudscale-ch/denis/two-servers-in-a-subnet

Increase robustness of two_servers_in_same_subnet

conftest.py

@@ -451,6 +451,13 @@ def two_servers_in_same_subnet(create_server, prober, image):
 
     """
 
+    # How many servers to launch at once. As we add more public subnets,
+    # this number should be increased.
+    parallel = 6
+
+    # How many times to try launching a set of servers
+    tries = 4
+
     def network_id(server):
         targets = (i for i in server.interfaces if i['type'] == 'public')
         return next(i['network']['uuid'] for i in targets)
@@ -462,21 +469,32 @@ def two_in_same_subnet(servers):
 
         return None, None
 
-    for _ in range(4):
+    # Ignore assertion errors during server creation, as we don't want
+    # flakiness to stop us from finding two servers (we will abort if no server
+    # was created, which would indicate a more serious problem).
+    def try_create_server(**args):
+        try:
+            return create_server(**args)
+        except AssertionError:
+            return None
+
+    for _ in range(tries):
 
         server_args = {
             'image': image['slug'],
             'use_public_network': True,
             'use_private_network': True,
             'jump_host': prober,
         }
-        servers = in_parallel(create_server, instances=(
-            {'name': 's1', **server_args},
-            {'name': 's2', **server_args},
-            {'name': 's3', **server_args},
-            {'name': 's4', **server_args},
+
+        servers = in_parallel(try_create_server, instances=(
+            {'name': f's{n}', **server_args} for n in range(1, parallel + 1)
         ))
 
+        # Fail of no servers can be created
+        servers = [s for s in servers if s is not None]
+        assert servers
+
         a, b = two_in_same_subnet(servers)
 
         for s in servers:

constants.py

@@ -50,7 +50,7 @@
 SERVER_START_TIMEOUT = 240
 
 # How many resources may be spawned in parallel in a single call
-RESOURCE_CREATION_CONCURRENCY_LIMIT = 4
+RESOURCE_CREATION_CONCURRENCY_LIMIT = 2
 
 # Where events are logged
 EVENTS_PATH = 'events'

test_public_network.py

@@ -102,8 +102,8 @@ def test_public_network_port_security(ip_version, two_servers_in_same_subnet):
     Attention: Due to it's nature this test can cause havoc on the network in
     the absence of effective port security to prevent ARP and ND spoofing!
 
-    This is notoriously unreliable in failing in the absence of port security
-    rules for IPv6. It often succeeds even if it should fail.
+    This test can reliably detect port security issues in IPv4, but it is
+    unreliable when it comes to IPv6, where it can produce false positives.
     """
 
     # We need two servers in the same subnet