Add LBaaS API Reliability Test and LoadBalancer Remove Helpers

README.md

@@ -27,22 +27,23 @@ These tests are run regularly against our public infrastructure as well as our i
 |                           | [test_floating_ip_failover](./test_floating_ip.py#L98)                           | default  |
 |                           | [test_floating_ip_mass_failover](./test_floating_ip.py#L141)                     | default  |
 |                           | [test_floating_network](./test_floating_ip.py#L180)                              | default  |
-| **Load Balancer**         | [test_simple_tcp_load_balancer](./test_load_balancer.py#L25)                     | default  |
-|                           | [test_simple_udp_load_balancer](./test_load_balancer.py#L49)                     | default  |
-|                           | [test_load_balancer_end_to_end](./test_load_balancer.py#L79)                     | default  |
-|                           | [test_multiple_listeners](./test_load_balancer.py#L153)                          | default  |
-|                           | [test_multiple_listeners_multiple_pools](./test_load_balancer.py#L185)           | default  |
-|                           | [test_balancing_algorithm_round_robin](./test_load_balancer.py#L238)             | default  |
-|                           | [test_balancing_algorithm_source_ip](./test_load_balancer.py#L274)               | default  |
-|                           | [test_balancing_algorithm_least_connections](./test_load_balancer.py#L323)       | default  |
-|                           | [test_backend_health_monitors](./test_load_balancer.py#L364)                     | default  |
-|                           | [test_pool_member_change](./test_load_balancer.py#L444)                          | default  |
-|                           | [test_private_load_balancer_frontend](./test_load_balancer.py#L536)              | default  |
-|                           | [test_floating_ip](./test_load_balancer.py#L580)                                 | default  |
-|                           | [test_floating_ip_reassign](./test_load_balancer.py#L614)                        | default  |
-|                           | [test_frontend_allowed_cidr](./test_load_balancer.py#L737)                       | default  |
-|                           | [test_proxy_protocol](./test_load_balancer.py#L812)                              | default  |
-|                           | [test_ping](./test_load_balancer.py#L855)                                        | default  |
+| **Load Balancer**         | [test_simple_tcp_load_balancer](./test_load_balancer.py#L26)                     | default  |
+|                           | [test_simple_udp_load_balancer](./test_load_balancer.py#L50)                     | default  |
+|                           | [test_load_balancer_end_to_end](./test_load_balancer.py#L80)                     | default  |
+|                           | [test_multiple_listeners](./test_load_balancer.py#L154)                          | default  |
+|                           | [test_multiple_listeners_multiple_pools](./test_load_balancer.py#L186)           | default  |
+|                           | [test_balancing_algorithm_round_robin](./test_load_balancer.py#L239)             | default  |
+|                           | [test_balancing_algorithm_source_ip](./test_load_balancer.py#L275)               | default  |
+|                           | [test_balancing_algorithm_least_connections](./test_load_balancer.py#L324)       | default  |
+|                           | [test_backend_health_monitors](./test_load_balancer.py#L365)                     | default  |
+|                           | [test_pool_member_change](./test_load_balancer.py#L445)                          | default  |
+|                           | [test_private_load_balancer_frontend](./test_load_balancer.py#L537)              | default  |
+|                           | [test_floating_ip](./test_load_balancer.py#L581)                                 | default  |
+|                           | [test_floating_ip_reassign](./test_load_balancer.py#L615)                        | default  |
+|                           | [test_frontend_allowed_cidr](./test_load_balancer.py#L738)                       | default  |
+|                           | [test_proxy_protocol](./test_load_balancer.py#L813)                              | default  |
+|                           | [test_load_balancer_api_reliability](./test_load_balancer.py#L856)               | default  |
+|                           | [test_ping](./test_load_balancer.py#L939)                                        | default  |
 | **Nested Virtualization** | [test_virtualization_support](./test_nested_virtualization.py#L14)               | default  |
 |                           | [test_run_nested_vm](./test_nested_virtualization.py#L40)                        | default  |
 | **Private Network**       | [test_private_ip_address_on_all_images](./test_private_network.py#L14)           | all      |

resources.py

@@ -1448,6 +1448,23 @@ def remove_pool_member(self, pool, member):
         self.pool_members = list(filter(lambda x: x['uuid'] != member["uuid"],
                                         self.pool_members))
 
+    @with_trigger('load-balancer.remove-pool')
+    def remove_pool(self, pool):
+        self.api.delete(f'load-balancers/pools/{pool["uuid"]}')
+        self.pools = [p for p in self.pools if p['uuid'] != pool['uuid']]
+
+    @with_trigger('load-balancer.remove-listener')
+    def remove_listener(self, listener):
+        self.api.delete(f'load-balancers/listeners/{listener["uuid"]}')
+        self.listeners = [lis for lis in self.listeners
+                          if lis['uuid'] != listener['uuid']]
+
+    @with_trigger('load-balancer.remove-health-monitor')
+    def remove_health_monitor(self, hm):
+        self.api.delete(f'load-balancers/health-monitors/{hm["uuid"]}')
+        self.health_monitors = [h for h in self.health_monitors
+                                if h['uuid'] != hm['uuid']]
+
     @with_trigger('load-balancer.disable-pool-member')
     def toggle_pool_member(self, pool, member, enabled=True):
         self.api.patch(

test_load_balancer.py

@@ -17,6 +17,7 @@
 from util import setup_lbaas_http_test_server
 from util import setup_lbaas_udp_test_server
 from util import start_persistent_download
+from util import FakeBackend
 from util import unique
 from util import wait_for_load_balancer_ready
 from util import wait_for_url_ready
@@ -298,7 +299,7 @@ def test_balancing_algorithm_source_ip(
     clients = in_parallel(
         create_server,
         instances=({
-            'name': f'client{i+1}',
+            'name': f'client{i + 1}',
             'image': image,
         } for i in range(4))
     )
@@ -852,6 +853,89 @@ def test_proxy_protocol(prober, create_load_balancer_scenario, proxy_protocol):
     assert expected_log_line[proxy_protocol] in logs
 
 
+def test_load_balancer_api_reliability(
+        create_load_balancer, create_private_network,
+):
+    """ Verify that pools, members, listeners and health monitors can be
+    created, deleted, and recreated without any API calls failing.
+
+    Uses FakeBackend instances with fixed addresses instead of real VMs,
+    testing the LBaaS control-plane API without standing up backend servers.
+
+    """
+
+    pools = 5
+    members_per_pool = 10
+    listeners_per_pool = 2
+    runs = 2
+    subnet_cidr = '10.0.0.0/24'
+    member_ip_start = 5
+    listener_base_port = 8000
+
+    assert pools * members_per_pool <= 249, 'Too many members (IP exhaustion)'
+
+    # Create a private network with a subnet for the fixed member addresses
+    private_network = create_private_network()
+    subnet = private_network.add_subnet(cidr=subnet_cidr)
+
+    # Create a load balancer and wait for it to reach running state
+    load_balancer = create_load_balancer()
+    load_balancer.wait_for('running', seconds=120)
+
+    for run_n in range(runs):
+        ip_counter = member_ip_start
+        port_counter = listener_base_port + run_n * pools * listeners_per_pool
+        created = []  # tracks resources created in this run for deletion
+
+        for pool_n in range(pools):
+
+            # Create a pool
+            pool = load_balancer.add_pool(
+                f'bench-{pool_n}', 'round_robin', 'tcp',
+            )
+
+            # Add members using fixed addresses, no real VMs needed
+            members = []
+            for member_n in range(members_per_pool):
+                ip = f'10.0.0.{ip_counter}'
+                ip_counter += 1
+                fake = FakeBackend(
+                    f'bench-{pool_n}-{member_n}', ip, subnet.uuid)
+                member = load_balancer.add_pool_member(
+                    pool, fake, private_network)
+                members.append(member)
+
+            # Add listeners
+            listeners = []
+            for listener_n in range(listeners_per_pool):
+                listener = load_balancer.add_listener(
+                    pool,
+                    port_counter,
+                    name=f'bench-{pool_n}-{listener_n}',
+                )
+                listeners.append(listener)
+                port_counter += 1
+
+            # Add a health monitor
+            hm = load_balancer.add_health_monitor(pool, 'tcp', None)
+
+            created.append({
+                'pool': pool,
+                'members': members,
+                'listeners': listeners,
+                'hm': hm,
+            })
+
+        # Delete the resources created in this run
+        for item in reversed(created):
+            for listener in reversed(item['listeners']):
+                load_balancer.remove_listener(listener)
+            load_balancer.remove_health_monitor(item['hm'])
+            for member in reversed(item['members']):
+                load_balancer.remove_pool_member(item['pool'], member)
+            load_balancer.remove_pool(item['pool'])
+
+
 def test_ping(prober, create_load_balancer_scenario, floating_ipv4,
               floating_ipv6):
     """ The load balancer answers to ICMP echo requests (ping) on all VIP

util.py

@@ -147,7 +147,7 @@ def generate_server_name(request, original_name=''):
     # Truncate name to 63 characters, but keep the caller supplied name. This
     # part might be important to distinguish different servers in a test
     if len(name) > 63:
-        name = f'{name[:63-len(original_name)-1]}-{original_name.lower()}'
+        name = f'{name[:63 - len(original_name) - 1]}-{original_name.lower()}'
 
     # Remove - at the start or end
     name = name.strip('-')
@@ -824,6 +824,28 @@ def unique(iterable):
     return set(iterable)
 
 
+class FakeBackend:
+    """ A fake Server for use as an LBaaS pool member with a fixed address,
+    for tests that exercise the control-plane API without real VMs.
+
+    Satisfies the duck-type expected by LoadBalancer.add_pool_member:
+    - name: used in the member resource name
+    - ip_address_config(): returns the address and subnet UUID
+
+    """
+
+    def __init__(self, name, address, subnet_uuid):
+        self.name = name
+        self._address = address
+        self._subnet_uuid = subnet_uuid
+
+    def ip_address_config(self, *args, **kwargs):
+        return {
+            'address': self._address,
+            'subnet': {'uuid': self._subnet_uuid},
+        }
+
+
 @contextmanager
 def assert_takes_no_longer_than(seconds):
     """ Asserts that inside of the "with" block takes no longer than the