diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..ab72bcdd8 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,19 @@ +name: Unit tests + +on: [push, pull_request] + +jobs: + unittest: + runs-on: ubuntu-22.04 + strategy: + matrix: + python-version: + - "3.9" + - "3.10" + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - run: pip install tox + - run: tox -e py diff --git a/.gitignore b/.gitignore index 3a64ce30d..72f965149 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,7 @@ pip-log.txt # Unit test / coverage reports .coverage .tox -nosetests.xml +.stestr/ cover # Translations @@ -35,8 +35,6 @@ cover AUTHORS ChangeLog -.testrepository - # generated policy file etc/blazar/policy.yaml.sample diff --git a/.gitreview b/.gitreview index 1500a8d6d..451928396 100644 --- a/.gitreview +++ b/.gitreview @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/blazar.git +defaultbranch=stable/2023.1 diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 000000000..604900fbf --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=${TEST_PATH:-./blazar/tests} +top_dir=./ diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index 4da4d7dba..000000000 --- a/.testr.conf +++ /dev/null @@ -1,7 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ - ${PYTHON:-python} -m subunit.run discover $DISCOVER_DIRECTORY $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/.zuul.yaml b/.zuul.yaml index cef278d65..c1fadeb63 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,21 +1,39 @@ +- job: + name: blazar-tempest-py3 + parent: blazar-tempest-plugin-base + description: | + Local variant of blazar-tempest-plugin-base which forces py3 + (the default for this branch is py2) + vars: + devstack_localrc: + USE_PYTHON3: True + +- job: + name: blazar-tempest-ipv6-only-py3 + parent: blazar-tempest-plugin-ipv6-only + description: | + Local variant of blazar-tempest-plugin-ipv6-only which forces py3 + (the default for this branch is py2) + vars: + devstack_localrc: + USE_PYTHON3: True + - project: + queue: blazar templates: - check-requirements - openstack-cover-jobs - - openstack-lower-constraints-jobs - - openstack-python3-yoga-jobs + - openstack-python3-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: jobs: - blazar-tempest-plugin-base - blazar-tempest-plugin-ipv6-only - - openstack-tox-lower-constraints: - voting: false - openstack-tox-pylint: voting: false gate: - queue: blazar jobs: - blazar-tempest-plugin-base - - blazar-tempest-plugin-ipv6-only + - blazar-tempest-py3 + - blazar-tempest-ipv6-only-py3 diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..30dfe979a --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,35 @@ +pipeline { + agent any + + options { + copyArtifactPermission(projectNames: 'blazar*') + } + + stages { + stage('test') { + parallel { + stage('pep8') { + steps { + sh 'tox -e pep8' + } + } + // stage('py36') { + // steps { + // sh 'tox -e py36' + // } + // } + } + } + + stage('package') { + steps { + dir('dist') { + deleteDir() + } + sh 'python setup.py sdist' + sh 'find dist -type f -exec cp {} dist/blazar.tar.gz \\;' + archiveArtifacts(artifacts: 'dist/blazar.tar.gz', onlyIfSuccessful: true) + } + } + } +} diff --git a/api-ref/source/conf.py b/api-ref/source/conf.py index f4a69d167..daa937fc1 100644 --- a/api-ref/source/conf.py +++ b/api-ref/source/conf.py @@ -65,7 +65,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +#language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/api-ref/source/v1/hosts.inc b/api-ref/source/v1/hosts.inc index 26c3e1966..620e33603 100644 --- a/api-ref/source/v1/hosts.inc +++ b/api-ref/source/v1/hosts.inc @@ -128,6 +128,7 @@ Request .. rest_parameters:: parameters.yaml - host_id: host_id_path + - private: property_private Response -------- @@ -324,3 +325,90 @@ Response .. literalinclude:: ../../../doc/api_samples/hosts/allocation-get-resp.json :language: javascript + +List Resource Properties +======================== + +.. rest_method:: GET v1/os-hosts/properties + +Get all resource properties from host + +**Response codes** + +Normal response code: 200 + +Error response codes: Bad Request(400), Unauthorized(401), Forbidden(403), +Internal Server Error(500) + +Request +------- + +.. rest_parameters:: parameters.yaml + + - detail: resource_property_detail + - all: resource_property_all + +Response +-------- + +.. rest_parameters:: parameters.yaml + + - resource_properties: resource_properties + - property: resource_properties_property + - private: resource_properties_private + - values: resource_properties_values + +**Example of List Resource Properties Response** + +.. literalinclude:: ../../../doc/api_samples/hosts/host-property-list.json + :language: javascript + +**Example of List Resource Properties With Detail Response** + +.. literalinclude:: ../../../doc/api_samples/hosts/host-property-list-detail.json + :language: javascript + +Update Resource Properties +========================== + +.. rest_method:: PATCH v1/os-hosts/properties/{property_name} + +Update a host resource properties + +**Response codes** + +Normal response code: 200 + +Error response codes: Bad Request(400), Unauthorized(401), Forbidden(403), +Internal Server Error(500) + +Request +------- + +.. rest_parameters:: parameters.yaml + + - property_name: property_name + - private: property_private + +**Example of Update Resource Properties** + +.. literalinclude:: ../../../doc/api_samples/hosts/host-property-update.json + :language: javascript + +Response +-------- + +.. rest_parameters:: parameters.yaml + + - created_at: created_at + - updated_at: updated_at + - id: resource_property_id + - resource_type: resource_property_resource_type + - property_name: resource_properties_property + - private: resource_property_private + +**Example of List Resource Properties Response** + +.. literalinclude:: ../../../doc/api_samples/hosts/host-property-update-res.json + :language: javascript + diff --git a/api-ref/source/v1/parameters.yaml b/api-ref/source/v1/parameters.yaml index e757e8183..a8d3edf59 100644 --- a/api-ref/source/v1/parameters.yaml +++ b/api-ref/source/v1/parameters.yaml @@ -42,6 +42,12 @@ lease_id_path: in: path required: true type: string +property_name: + description: | + The name of the property. + in: path + required: true + type: string # variables in query @@ -57,6 +63,19 @@ allocation_reservation_id_query: in: query required: false type: string +resource_property_all: + description: | + Whether to include all resource properties, public and private. + in: query + required: false + type: string +resource_property_detail: + description: | + Whether to include values along for each property and if the property + is private. + in: query + required: false + type: string # variables in body @@ -394,6 +413,13 @@ leases: in: body required: true type: array +property_private: + description: | + Whether the property is private. + + in: body + required: true + type: boolean reservation: description: | A ``reservation`` object. @@ -615,6 +641,69 @@ reservations_optional: in: body required: false type: array +resource_properties: + description: | + A list of ``resource_property`` objects. + + in: body + required: true + type: array +resource_properties_private: + description: | + Whether the property is private. + + in: body + required: false + type: boolean +resource_properties_property: + description: | + The name of the property. + + in: body + required: true + type: any +resource_properties_values: + description: | + A list of values for the property. + + in: body + required: false + type: array +resource_property: + description: | + The updated ``resource_property`` object. + + in: body + required: true + type: any +resource_property_id: + description: | + The updated ``resource_property`` UUID. + + in: body + required: true + type: string +resource_property_private: + description: | + Whether the updated ``resource_property`` is private. + + in: body + required: true + type: boolean +resource_property_property_name: + description: | + The updated ``resource_property`` property_name. + + in: body + required: true + type: string +resource_property_resource_type: + description: | + The updated ``resource_property`` resource type. + + in: body + required: true + type: string updated_at: description: | The date and time when the object was updated. diff --git a/blazar/api/context.py b/blazar/api/context.py index 9b2ea0df2..e8fb5ae56 100644 --- a/blazar/api/context.py +++ b/blazar/api/context.py @@ -27,18 +27,5 @@ def ctx_from_headers(headers): except TypeError: raise exceptions.WrongFormat() - kwargs = {"user_id": headers['X-User-Id'], - "project_id": headers['X-Project-Id'], - "auth_token": headers['X-Auth-Token'], - "service_catalog": service_catalog, - "user_name": headers['X-User-Name'], - "project_name": headers['X-Project-Name'], - "roles": list( - map(str.strip, headers['X-Roles'].split(',')))} - - # For v1 only, request_id and global_request_id will be available. - if headers.environ['PATH_INFO'].startswith('/v1'): - kwargs['request_id'] = headers.environ['openstack.request_id'] - kwargs['global_request_id'] = headers.environ.get( - 'openstack.global_request_id') - return context.BlazarContext(**kwargs) + return context.BlazarContext.from_environ(headers.environ, + service_catalog=service_catalog) diff --git a/blazar/api/v1/devices/__init__.py b/blazar/api/v1/devices/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/blazar/api/v1/devices/service.py b/blazar/api/v1/devices/service.py new file mode 100644 index 000000000..171482d37 --- /dev/null +++ b/blazar/api/v1/devices/service.py @@ -0,0 +1,110 @@ +# Copyright (c) 2018 StackHPC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from blazar import context +from blazar.manager.service import get_plugins +from blazar import policy +from blazar.utils import trusts + + +class API(object): + def __init__(self): + self.plugin = get_plugins()["device"] + + @policy.authorize('devices', 'get') + def get_devices(self): + """List all existing devices.""" + return self.plugin.list_devices() + + @policy.authorize('devices', 'post') + @trusts.use_trust_auth() + def create_device(self, data): + """Create new device. + + :param data: New device characteristics. + :type data: dict + """ + + return self.plugin.create_device(data) + + @policy.authorize('devices', 'get') + def get_device(self, device_id): + """Get device by its ID. + + :param device_id: ID of the device in Blazar DB. + :type device_id: str + """ + return self.plugin.get_device(device_id) + + @policy.authorize('devices', 'put') + def update_device(self, device_id, data): + """Update device. + + :param device_id: ID of the device in Blazar DB. + :type device_id: str + :param data: New device characteristics. + :type data: dict + """ + return self.plugin.update_device(device_id, data) + + @policy.authorize('devices', 'delete') + def delete_device(self, device_id): + """Delete specified device. + + :param device_id: ID of the device in Blazar DB. + :type device_id: str + """ + self.plugin.delete_device(device_id) + + @policy.authorize('devices', 'reallocate') + def reallocate(self, device_id, data): + """Exchange device from allocations.""" + return self.plugin.reallocate_device(device_id, data) + + @policy.authorize('devices', 'get_allocations') + def list_allocations(self, query): + """List all allocations on all devices. + + :param query: parameter to query allocations + :type query: dict + """ + ctx = context.current() + detail = False + + if policy.enforce(ctx, 'admin', {}, do_raise=False): + detail = True + + return self.plugin.list_allocations(query, detail=detail) + + @policy.authorize('devices', 'get_allocations') + def get_allocations(self, device_id, query): + """List all allocations on a specificied device. + + :param device_id: ID of the device in Blazar BDself. + :type device_id: str + :param query: parameters to query allocation + :type query: dict + """ + return self.plugin.get_allocations(device_id, query) + + @policy.authorize('devices', 'get_resource_properties') + def list_resource_properties(self, query): + """List resource properties for devices.""" + return self.plugin.list_resource_properties(query) + + @policy.authorize('devices', 'update_resource_properties') + def update_resource_property(self, property_name, data): + """Update a device resource property.""" + return self.plugin.update_resource_property(property_name, data) diff --git a/blazar/api/v1/devices/v1_0.py b/blazar/api/v1/devices/v1_0.py new file mode 100644 index 000000000..54d1b37a0 --- /dev/null +++ b/blazar/api/v1/devices/v1_0.py @@ -0,0 +1,103 @@ +# Copyright (c) 2018 StackHPC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from blazar.api.v1.devices import service +from blazar.api.v1 import utils as api_utils +from blazar.api.v1 import validation +from blazar import utils + + +def get_rest(): + """Return Rest app""" + return rest + + +rest = api_utils.Rest('device_v1_0', __name__, url_prefix='/v1/devices') +_api = utils.LazyProxy(service.API) + + +# devices operations + +@rest.get('') +def devices_list(req): + """List all existing devices.""" + return api_utils.render(devices=_api.get_devices()) + + +@rest.post('') +def devices_create(req, data): + """Create new device.""" + return api_utils.render(device=_api.create_device(data)) + + +@rest.get('/') +@validation.check_exists(_api.get_device, device_id='device_id') +def devices_get(req, device_id): + """Get device by its ID.""" + return api_utils.render(device=_api.get_device(device_id)) + + +@rest.put('/') +@validation.check_exists(_api.get_device, device_id='device_id') +def devices_update(req, device_id, data): + """Update device. Only name changing may be proceeded.""" + if len(data) == 0: + return api_utils.internal_error(status_code=400, + descr="No data to update") + else: + return api_utils.render(device=_api.update_device(device_id, data)) + + +@rest.delete('/') +@validation.check_exists(_api.get_device, device_id='device_id') +def devices_delete(req, device_id): + """Delete specified device.""" + _api.delete_device(device_id) + return api_utils.render(status=200) + + +@rest.put('//allocation') +@validation.check_exists(_api.get_device, device_id='device_id') +def reallocate(req, device_id, data): + """Exhange device in a lease.""" + return api_utils.render(allocation=_api.reallocate(device_id, data)) + + +@rest.get('/allocations', query=True) +def allocations_list(req, query, detail=False): + """List all allocations on all device segments.""" + return api_utils.render(allocations=_api.list_allocations(query)) + + +@rest.get('//allocation') +@validation.check_exists(_api.get_device, device_id='device_id') +def allocations_get(req, device_id, query): + """List all allocations on a specific device segment.""" + return api_utils.render(allocation=_api.get_allocations(device_id, + query)) + + +@rest.get('/properties', query=True) +def resource_properties_list(req, query=None): + """List device resource properties.""" + return api_utils.render( + resource_properties=_api.list_resource_properties(query)) + + +@rest.patch('/properties/') +def resource_property_update(req, property_name, data): + """Update a device resource property.""" + return api_utils.render( + resource_property=_api.update_resource_property(property_name, data)) diff --git a/blazar/api/v1/floatingips/service.py b/blazar/api/v1/floatingips/service.py index 8181061bc..f8305d0c8 100644 --- a/blazar/api/v1/floatingips/service.py +++ b/blazar/api/v1/floatingips/service.py @@ -13,19 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from blazar.manager.floatingips import rpcapi as manager_rpcapi +from blazar import context +from blazar.manager.service import get_plugins from blazar import policy from blazar.utils import trusts class API(object): def __init__(self): - self.manager_rpcapi = manager_rpcapi.ManagerRPCAPI() + self.plugin = get_plugins()["virtual:floatingip"] @policy.authorize('floatingips', 'get') def get_floatingips(self): """List all existing floatingip.""" - return self.manager_rpcapi.list_floatingips() + return self.plugin.list_floatingip() @policy.authorize('floatingips', 'post') @trusts.use_trust_auth() @@ -36,7 +37,7 @@ def create_floatingip(self, data): :type data: dict """ - return self.manager_rpcapi.create_floatingip(data) + return self.plugin.create_floatingip(data) @policy.authorize('floatingips', 'get') def get_floatingip(self, floatingip_id): @@ -45,7 +46,7 @@ def get_floatingip(self, floatingip_id): :param floatingip_id: ID of the floatingip in Blazar DB. :type floatingip_id: str """ - return self.manager_rpcapi.get_floatingip(floatingip_id) + return self.plugin.get_floatingip(floatingip_id) @policy.authorize('floatingips', 'delete') def delete_floatingip(self, floatingip_id): @@ -54,4 +55,19 @@ def delete_floatingip(self, floatingip_id): :param floatingip_id: ID of the floatingip in Blazar DB. :type floatingip_id: str """ - self.manager_rpcapi.delete_floatingip(floatingip_id) + self.plugin.delete_floatingip(floatingip_id) + + @policy.authorize('floatingips', 'get_allocations') + def list_allocations(self, query): + """List all allocations on all computehosts. + + :param query: parameters to query allocations + :type query: dict + """ + ctx = context.current() + detail = False + + if policy.enforce(ctx, 'admin', {}, do_raise=False): + detail = True + + return self.plugin.list_allocations(query, detail=detail) diff --git a/blazar/api/v1/floatingips/v1_0.py b/blazar/api/v1/floatingips/v1_0.py index 436494d33..0fa477b21 100644 --- a/blazar/api/v1/floatingips/v1_0.py +++ b/blazar/api/v1/floatingips/v1_0.py @@ -55,4 +55,9 @@ def floatingips_get(req, floatingip_id): def floatingips_delete(req, floatingip_id): """Delete specified floatingip.""" _api.delete_floatingip(floatingip_id) - return api_utils.render() + return api_utils.render(status=200) + +@rest.get('/allocations', query=True) +def allocations_list(req, query): + """List all allocations on all computehosts.""" + return api_utils.render(allocations=_api.list_allocations(query)) diff --git a/blazar/api/v1/leases/service.py b/blazar/api/v1/leases/service.py index 0a63762b9..9cab08969 100644 --- a/blazar/api/v1/leases/service.py +++ b/blazar/api/v1/leases/service.py @@ -15,8 +15,8 @@ from oslo_log import log as logging +from blazar.cmd.manager import ManagerServiceSingleton from blazar import context -from blazar.manager.leases import rpcapi as manager_rpcapi from blazar import policy from blazar.utils import trusts @@ -26,7 +26,7 @@ class API(object): def __init__(self): - self.manager_rpcapi = manager_rpcapi.ManagerRPCAPI() + self.manager_service = ManagerServiceSingleton() # Leases operations @@ -34,12 +34,16 @@ def __init__(self): def get_leases(self, query): """List all existing leases.""" ctx = context.current() - if policy.enforce(ctx, 'admin', {}, do_raise=False): + is_admin = policy.enforce(ctx, 'admin', {}, do_raise=False) + if query.get("all_tenants") and is_admin: project_id = None + elif query.get("project_id") and is_admin: + # User can only filter by project_id if they are an admin + project_id = query.get("project_id") else: project_id = ctx.project_id - return self.manager_rpcapi.list_leases(project_id=project_id, - query=query) + return self.manager_service.list_leases(project_id=project_id, + query=query) @policy.authorize('leases', 'post') @trusts.use_trust_auth() @@ -54,7 +58,7 @@ def create_lease(self, data): # two lines ctx = context.current() data['user_id'] = ctx.user_id - return self.manager_rpcapi.create_lease(data) + return self.manager_service.create_lease(data) @policy.authorize('leases', 'get') def get_lease(self, lease_id): @@ -63,7 +67,34 @@ def get_lease(self, lease_id): :param lease_id: ID of the lease in Blazar DB. :type lease_id: str """ - return self.manager_rpcapi.get_lease(lease_id) + return self.manager_service.get_lease(lease_id) + + @policy.authorize('leases', 'get') + def hosts_in_lease(self, lease_id): + """List all hosts in lease by its ID. + + :param lease_id: ID of the lease in Blazar DB. + :type lease_id: str + """ + return self.manager_service.hosts_in_lease(lease_id) + + @policy.authorize('leases', 'get') + def networks_in_lease(self, lease_id): + """List all networks in lease by its ID. + + :param lease_id: ID of the lease in Blazar DB. + :type lease_id: str + """ + return self.manager_service.networks_in_lease(lease_id) + + @policy.authorize('leases', 'get') + def devices_in_lease(self, lease_id): + """List all devices in lease by its ID. + + :param lease_id: ID of the lease in Blazar DB. + :type lease_id: str + """ + return self.manager_service.devices_in_lease(lease_id) @policy.authorize('leases', 'put') def update_lease(self, lease_id, data): @@ -74,7 +105,7 @@ def update_lease(self, lease_id, data): :param data: New lease characteristics. :type data: dict """ - return self.manager_rpcapi.update_lease(lease_id, data) + return self.manager_service.update_lease(lease_id, data) @policy.authorize('leases', 'delete') def delete_lease(self, lease_id): @@ -83,7 +114,7 @@ def delete_lease(self, lease_id): :param lease_id: ID of the lease in Blazar DB. :type lease_id: str """ - self.manager_rpcapi.delete_lease(lease_id) + self.manager_service.delete_lease(lease_id) # Plugins operations diff --git a/blazar/api/v1/leases/v1_0.py b/blazar/api/v1/leases/v1_0.py index ee41c3595..0f447872b 100644 --- a/blazar/api/v1/leases/v1_0.py +++ b/blazar/api/v1/leases/v1_0.py @@ -65,7 +65,23 @@ def leases_update(req, lease_id, data): def leases_delete(req, lease_id): """Delete specified lease.""" _api.delete_lease(lease_id) - return api_utils.render() + return api_utils.render(status=200) + + +@rest.get('/leases//hosts') +def hosts_in_lease_get(req, lease_id): + """Get hosts in lease by its ID.""" + return api_utils.render(hosts=_api.hosts_in_lease(lease_id)) + +@rest.get('/leases//networks') +def networks_in_lease_get(req, lease_id): + """Get networks in lease by its ID.""" + return api_utils.render(networks=_api.networks_in_lease(lease_id)) + +@rest.get('/leases//devices') +def devices_in_lease_get(req, lease_id): + """Get devices in lease by its ID.""" + return api_utils.render(devices=_api.devices_in_lease(lease_id)) # Plugins operations diff --git a/blazar/api/v1/networks/__init__.py b/blazar/api/v1/networks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/blazar/api/v1/networks/service.py b/blazar/api/v1/networks/service.py new file mode 100644 index 000000000..c1964cbe0 --- /dev/null +++ b/blazar/api/v1/networks/service.py @@ -0,0 +1,105 @@ +# Copyright (c) 2018 StackHPC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from blazar import context +from blazar.manager.service import get_plugins +from blazar import policy +from blazar.utils import trusts + + +class API(object): + def __init__(self): + self.plugin = get_plugins()["network"] + + @policy.authorize('networks', 'get') + def get_networks(self): + """List all existing networks.""" + return self.plugin.list_networks() + + @policy.authorize('networks', 'post') + @trusts.use_trust_auth() + def create_network(self, data): + """Create new network. + + :param data: New network characteristics. + :type data: dict + """ + + return self.plugin.create_network(data) + + @policy.authorize('networks', 'get') + def get_network(self, network_id): + """Get network by its ID. + + :param network_id: ID of the network in Blazar DB. + :type network_id: str + """ + return self.plugin.get_network(network_id) + + @policy.authorize('networks', 'put') + def update_network(self, network_id, data): + """Update network. Only name changing may be proceeded. + + :param network_id: ID of the network in Blazar DB. + :type network_id: str + :param data: New network characteristics. + :type data: dict + """ + return self.plugin.update_network(network_id, data) + + @policy.authorize('networks', 'delete') + def delete_network(self, network_id): + """Delete specified network. + + :param network_id: ID of the network in Blazar DB. + :type network_id: str + """ + self.plugin.delete_network(network_id) + + @policy.authorize('networks', 'get_allocations') + def list_allocations(self, query): + """List all allocations on all network segments. + + :param query: parameter to query allocations + :type query: dict + """ + ctx = context.current() + detail = False + + if policy.enforce(ctx, 'admin', {}, do_raise=False): + detail = True + + return self.plugin.list_allocations(query, detail=detail) + + @policy.authorize('networks', 'get_allocations') + def get_allocations(self, network_id, query): + """List all allocations on a specificied network segment. + + :param network_id: ID of the network segment in Blazar BDself. + :type network_id: str + :param query: parameters to query allocation + :type query: dict + """ + return self.plugin.get_allocations(network_id, query) + + @policy.authorize('networks', 'get_resource_properties') + def list_resource_properties(self, query): + """List resource properties for networks.""" + return self.plugin.list_resource_properties(query) + + @policy.authorize('networks', 'update_resource_properties') + def update_resource_property(self, property_name, data): + """Update a network resource property.""" + return self.plugin.update_resource_property(property_name, data) diff --git a/blazar/api/v1/networks/v1_0.py b/blazar/api/v1/networks/v1_0.py new file mode 100644 index 000000000..1575a439d --- /dev/null +++ b/blazar/api/v1/networks/v1_0.py @@ -0,0 +1,96 @@ +# Copyright (c) 2018 StackHPC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from blazar.api.v1.networks import service +from blazar.api.v1 import utils as api_utils +from blazar.api.v1 import validation +from blazar import utils + + +def get_rest(): + """Return Rest app""" + return rest + + +rest = api_utils.Rest('network_v1_0', __name__, url_prefix='/v1/networks') +_api = utils.LazyProxy(service.API) + + +# networks operations + +@rest.get('') +def networks_list(req): + """List all existing networks.""" + return api_utils.render(networks=_api.get_networks()) + + +@rest.post('') +def networks_create(req, data): + """Create new network.""" + return api_utils.render(network=_api.create_network(data)) + + +@rest.get('/') +@validation.check_exists(_api.get_network, network_id='network_id') +def networks_get(req, network_id): + """Get network by its ID.""" + return api_utils.render(network=_api.get_network(network_id)) + + +@rest.put('/') +@validation.check_exists(_api.get_network, network_id='network_id') +def networks_update(req, network_id, data): + """Update network. Only name changing may be proceeded.""" + if len(data) == 0: + return api_utils.internal_error(status_code=400, + descr="No data to update") + else: + return api_utils.render(network=_api.update_network(network_id, data)) + + +@rest.delete('/') +@validation.check_exists(_api.get_network, network_id='network_id') +def networks_delete(req, network_id): + """Delete specified network.""" + _api.delete_network(network_id) + return api_utils.render(status=200) + + +@rest.get('/allocations', query=True) +def allocations_list(req, query, detail=False): + """List all allocations on all network segments.""" + return api_utils.render(allocations=_api.list_allocations(query)) + + +@rest.get('//allocation', query=True) +@validation.check_exists(_api.get_network, network_id='network_id') +def allocations_get(req, network_id, query): + """List all allocations on a specific network segment.""" + return api_utils.render(allocation=_api.get_allocations(network_id, + query)) + + +@rest.get('/properties', query=True) +def resource_properties_list(req, query=None): + """List network resource properties.""" + return api_utils.render( + resource_properties=_api.list_resource_properties(query)) + + +@rest.patch('/properties/') +def resource_property_update(req, property_name, data): + """Update a network resource property.""" + return api_utils.render( + resource_property=_api.update_resource_property(property_name, data)) diff --git a/blazar/api/v1/oshosts/service.py b/blazar/api/v1/oshosts/service.py index 565a50349..9038fccaf 100644 --- a/blazar/api/v1/oshosts/service.py +++ b/blazar/api/v1/oshosts/service.py @@ -13,19 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from blazar.manager.oshosts import rpcapi as manager_rpcapi +from blazar import context +from blazar.manager.service import get_plugins from blazar import policy from blazar.utils import trusts class API(object): def __init__(self): - self.manager_rpcapi = manager_rpcapi.ManagerRPCAPI() + self.plugin = get_plugins()["physical:host"] @policy.authorize('oshosts', 'get') def get_computehosts(self, query): """List all existing computehosts.""" - return self.manager_rpcapi.list_computehosts(query=query) + return self.plugin.list_computehosts(query=query) @policy.authorize('oshosts', 'post') @trusts.use_trust_auth() @@ -36,7 +37,7 @@ def create_computehost(self, data): :type data: dict """ - return self.manager_rpcapi.create_computehost(data) + return self.plugin.create_computehost(data) @policy.authorize('oshosts', 'get') def get_computehost(self, host_id): @@ -45,7 +46,7 @@ def get_computehost(self, host_id): :param host_id: ID of the computehost in Blazar DB. :type host_id: str """ - return self.manager_rpcapi.get_computehost(host_id) + return self.plugin.get_computehost(host_id) @policy.authorize('oshosts', 'put') def update_computehost(self, host_id, data): @@ -56,7 +57,7 @@ def update_computehost(self, host_id, data): :param data: New computehost characteristics. :type data: dict """ - return self.manager_rpcapi.update_computehost(host_id, data) + return self.plugin.update_computehost(host_id, data) @policy.authorize('oshosts', 'delete') def delete_computehost(self, host_id): @@ -65,7 +66,7 @@ def delete_computehost(self, host_id): :param host_id: ID of the computehost in Blazar DB. :type host_id: str """ - self.manager_rpcapi.delete_computehost(host_id) + self.plugin.delete_computehost(host_id) @policy.authorize('oshosts', 'get_allocations') def list_allocations(self, query): @@ -74,7 +75,13 @@ def list_allocations(self, query): :param query: parameters to query allocations :type query: dict """ - return self.manager_rpcapi.list_allocations(query) + ctx = context.current() + detail = False + + if policy.enforce(ctx, 'admin', {}, do_raise=False): + detail = True + + return self.plugin.list_allocations(query, detail=detail) @policy.authorize('oshosts', 'get_allocations') def get_allocations(self, host_id, query): @@ -85,4 +92,18 @@ def get_allocations(self, host_id, query): :param query: parameters to query allocations :type query: dict """ - return self.manager_rpcapi.get_allocations(host_id, query) + return self.plugin.get_allocations(host_id, query) + + def reallocate(self, host_id, data): + """Exchange host from allocations.""" + return self.plugin.reallocate_computehost(host_id, data) + + @policy.authorize('oshosts', 'get_resource_properties') + def list_resource_properties(self, query): + """List resource properties for hosts.""" + return self.plugin.list_resource_properties(query) + + @policy.authorize('oshosts', 'update_resource_properties') + def update_resource_property(self, property_name, data): + """Update a host resource property.""" + return self.plugin.update_resource_property(property_name, data) diff --git a/blazar/api/v1/oshosts/v1_0.py b/blazar/api/v1/oshosts/v1_0.py index 1123c502a..219c19088 100644 --- a/blazar/api/v1/oshosts/v1_0.py +++ b/blazar/api/v1/oshosts/v1_0.py @@ -65,7 +65,7 @@ def computehosts_update(req, host_id, data): def computehosts_delete(req, host_id): """Delete specified computehost.""" _api.delete_computehost(host_id) - return api_utils.render() + return api_utils.render(status=200) @rest.get('/allocations', query=True) @@ -79,3 +79,24 @@ def allocations_list(req, query): def allocations_get(req, host_id, query): """List all allocations on a specific host.""" return api_utils.render(allocation=_api.get_allocations(host_id, query)) + + +@rest.put('//allocation') +@validation.check_exists(_api.get_computehost, host_id='host_id') +def reallocate(req, host_id, data): + """Exhange host in a lease.""" + return api_utils.render(allocation=_api.reallocate(host_id, data)) + + +@rest.get('/properties', query=True) +def resource_properties_list(req, query=None): + """List computehost resource properties.""" + return api_utils.render( + resource_properties=_api.list_resource_properties(query)) + + +@rest.patch('/properties/') +def resource_property_update(req, property_name, data): + """Update a computehost resource property.""" + return api_utils.render( + resource_property=_api.update_resource_property(property_name, data)) diff --git a/blazar/api/v1/utils.py b/blazar/api/v1/utils.py index 1825ed419..1892caf98 100644 --- a/blazar/api/v1/utils.py +++ b/blazar/api/v1/utils.py @@ -53,6 +53,9 @@ def post(self, rule, status_code=201): def put(self, rule, status_code=200): return self._mroute('PUT', rule, status_code) + def patch(self, rule, status_code=200): + return self._mroute('PATCH', rule, status_code) + def delete(self, rule, status_code=204): return self._mroute('DELETE', rule, status_code) @@ -79,7 +82,7 @@ def handler(**kwargs): if status: flask.request.status_code = status - if flask.request.method in ['POST', 'PUT']: + if flask.request.method in ['POST', 'PUT', 'PATCH']: kwargs['data'] = request_data() if flask.request.endpoint in self.routes_with_query_support: @@ -293,11 +296,11 @@ def get_request_args(): def abort_and_log(status_code, descr, exc=None): """Process occurred errors.""" - LOG.error("Request aborted with status code %(code)s and " + LOG.warning("Request aborted with status code %(code)s and " "message '%(msg)s'", {'code': status_code, 'msg': descr}) if exc is not None: - LOG.error(traceback.format_exc()) + LOG.warning(traceback.format_exc()) flask.abort(status_code, description=descr) diff --git a/blazar/cmd/api.py b/blazar/cmd/api.py index c2b7381d2..ef4d4eb6b 100644 --- a/blazar/cmd/api.py +++ b/blazar/cmd/api.py @@ -28,6 +28,7 @@ from blazar.api import app as wsgi_app from blazar.api.v2 import app as v2_app +from blazar.notification import notifier from blazar.utils import service as service_utils @@ -50,6 +51,7 @@ def main(): """Entry point to start Blazar API wsgi server.""" cfg.CONF(sys.argv[1:], project='blazar', prog='blazar-api') + notifier.init() service_utils.prepare_service(sys.argv) if not CONF.enable_v1_api: app = v2_app.make_app() diff --git a/blazar/cmd/manager.py b/blazar/cmd/manager.py index 4052fe961..17a925c1a 100644 --- a/blazar/cmd/manager.py +++ b/blazar/cmd/manager.py @@ -16,6 +16,7 @@ import eventlet eventlet.monkey_patch() +from functools import partial import gettext import sys @@ -24,20 +25,34 @@ gettext.install('blazar') -from blazar.db import api as db_api from blazar.manager import service as manager_service from blazar.notification import notifier from blazar.utils import service as service_utils +class ManagerServiceSingleton: + _instance = None + + def __new__(self, resource_type=None): + if not ManagerServiceSingleton._instance: + ManagerServiceSingleton._instance = \ + manager_service.ManagerService() + if resource_type: + return partial( + ManagerServiceSingleton._instance.call, resource_type) + return ManagerServiceSingleton._instance + + +manager_service_instance = None + + def main(): cfg.CONF(project='blazar', prog='blazar-manager') service_utils.prepare_service(sys.argv) - db_api.setup_db() notifier.init() service.launch( cfg.CONF, - manager_service.ManagerService(), + ManagerServiceSingleton(), restart_method='mutate' ).wait() diff --git a/blazar/cmd/usage.py b/blazar/cmd/usage.py new file mode 100644 index 000000000..6bd5a38c2 --- /dev/null +++ b/blazar/cmd/usage.py @@ -0,0 +1,109 @@ +import sys +from oslo_config import cfg +from blazar.db import api as db_api +from datetime import datetime + +CONF = cfg.CONF + + +def list_leases(args): + print("start,end,created_at,deleted_at,hours_before_start,number_of_hosts,numer_of_networks,id,user_id,project_id,host_id,hypervisor_hostname,node_name,node_type") + since_datetime = None + if args.since: + since_datetime = datetime.strptime(args.since, "%Y-%m-%d %H:%M") + hosts_by_id = {} + for lease in db_api.lease_list( + args.project_id, + status=None, + lease_id=None, + lease_name=None, + marker=None, + limit=None, + sort_dir="desc", + sort_key="end_date" + ): + try: + if since_datetime and lease["start_date"] < since_datetime: + continue + + created_at = datetime.strptime(lease["created_at"], "%Y-%m-%d %H:%M:%S") + td = max(0, (lease["start_date"] - created_at).total_seconds())/3600 + all_allocations = [] + for allocations in [ + db_api.host_allocation_get_all_by_values(reservation_id=x['id']) + for x in lease["reservations"] + if x["resource_type"] in ["physical:host", "flavor:instance"] + ]: + all_allocations.extend( + hosts_by_id.setdefault( + a["compute_host_id"], + db_api.host_get(a["compute_host_id"]) + ) + for a in allocations + ) + for host in all_allocations: + if not host.get("extras"): + host["extras"] = { + k: v.capability_value + for v, k in db_api.host_extra_capability_get_all_per_host(host["id"]) + } + print( + lease["start_date"], + lease["end_date"], + lease["created_at"], + lease["deleted_at"], + td, + len(db_api.hosts_in_lease(lease["id"])), + len(db_api.networks_in_lease(lease["id"])), + lease["id"], + lease["user_id"], + lease["project_id"], + host["id"], + host["hypervisor_hostname"], + host["extras"]["node_name"], + host["extras"]["node_type"], + sep=",", + ) + except Exception as e: + print(f"Error processing lease {lease['id']}: {e}", file=sys.stderr) + continue + + +def add_command_parsers(subparsers): + parser = subparsers.add_parser('list_leases') + parser.add_argument( + '--since', + type=str, + help='Start date for filtering leases (YYYY-MM-DD HH:MM format)' + ) + parser.add_argument( + '--project-id', + type=str, + default=None, + help='Project ID to filter by' + ) + parser.set_defaults(func=list_leases) + + +command_opts = [ + cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) +] +CONF.register_cli_opts(command_opts) + + +def main(): + CONF(project='blazar', prog='blazar-manager') + CONF.set_override("include_deleted", True) + + if hasattr(CONF.command, 'func'): + CONF.command.func(CONF.command) + else: + print("No valid command specified.") + CONF.print_help() + + +if __name__ == "__main__": + main() diff --git a/blazar/config.py b/blazar/config.py index c52f35ff6..89b971e16 100644 --- a/blazar/config.py +++ b/blazar/config.py @@ -88,6 +88,11 @@ CONF.register_opts(os_opts) CONF.register_opts(api_opts) CONF.register_opts(lease_opts) +CONF.register_opts([ + cfg.ListOpt('allocation_extras', + default=[], + help='What extra information to include with allocations.'), +], 'api') logging.register_options(cfg.CONF) diff --git a/blazar/context.py b/blazar/context.py index 524ec6e56..80424b3c6 100644 --- a/blazar/context.py +++ b/blazar/context.py @@ -15,31 +15,29 @@ import threading +from oslo_config import cfg from oslo_context import context +CONF = cfg.CONF + class BlazarContext(context.RequestContext): + # service_catalog is not by default read from a dict + # when deserializing a context. + FROM_DICT_EXTRA_KEYS = ['service_catalog'] + _context_stack = threading.local() - def __init__(self, user_id=None, project_id=None, project_name=None, - service_catalog=None, user_name=None, **kwargs): + def __init__(self, service_catalog=None, **kwargs): # NOTE(neha-alhat): During serializing/deserializing context object # over the RPC layer, below extra parameters which are passed by # `oslo.messaging` are popped as these parameters are not required. kwargs.pop('client_timeout', None) kwargs.pop('user_identity', None) - kwargs.pop('project', None) - - if user_id: - kwargs['user_id'] = user_id - if project_id: - kwargs['project_id'] = project_id super(BlazarContext, self).__init__(**kwargs) - self.project_name = project_name - self.user_name = user_name self.service_catalog = service_catalog or [] if self.is_admin and 'admin' not in self.roles: @@ -65,28 +63,35 @@ def current(cls): except (AttributeError, IndexError): raise RuntimeError("Context isn't available here") - # NOTE(yorik-sar): as long as oslo.rpc requires this def to_dict(self): result = super(BlazarContext, self).to_dict() - result['user_id'] = self.user_id - result['user_name'] = self.user_name - result['project_id'] = self.project_id - result['project_name'] = self.project_name result['service_catalog'] = self.service_catalog return result @classmethod - def elevated(cls): + def admin(cls): try: - ctx = cls.current() + cur = cls.current() + request_id = cur.request_id + global_request_id = cur.global_request_id + service_catalog = cur.service_catalog except RuntimeError: - ctx = None - return cls(ctx, is_admin=True) + request_id = global_request_id = service_catalog = None + return cls( + user_name=CONF.os_admin_username, + user_domain_name=CONF.os_admin_user_domain_name, + project_name=CONF.os_admin_project_name, + project_domain_name=CONF.os_admin_project_domain_name, + is_admin=True, + service_catalog=service_catalog, + request_id=request_id, + global_request_id=global_request_id + ) def current(): return BlazarContext.current() -def elevated(): - return BlazarContext.elevated() +def admin(): + return BlazarContext.admin() diff --git a/blazar/db/api.py b/blazar/db/api.py index 8deeb78b4..acbe08faa 100644 --- a/blazar/db/api.py +++ b/blazar/db/api.py @@ -179,9 +179,45 @@ def lease_get(lease_id): @to_dict -def lease_list(project_id=None): +def hosts_in_lease(lease_id): + """Return hosts in a lease.""" + return IMPL.hosts_in_lease(lease_id) + + +@to_dict +def networks_in_lease(lease_id): + """Return networks in a lease.""" + return IMPL.networks_in_lease(lease_id) + + +@to_dict +def devices_in_lease(lease_id): + """Return devices in a lease.""" + return IMPL.devices_in_lease(lease_id) + + +@to_dict +def lease_list( + project_id, + status, + lease_id, + lease_name, + marker, + limit, + sort_dir, + sort_key +): """Return a list of all existing leases.""" - return IMPL.lease_list(project_id) + return IMPL.lease_list( + project_id, + status, + lease_id, + lease_name, + marker, + limit, + sort_dir, + sort_key + ) def lease_destroy(lease_id): @@ -385,13 +421,11 @@ def host_extra_capability_create(values): return IMPL.host_extra_capability_create(values) -@to_dict def host_extra_capability_get(host_extra_capability_id): """Return a specific Host Extracapability.""" return IMPL.host_extra_capability_get(host_extra_capability_id) -@to_dict def host_extra_capability_get_all_per_host(host_id): """Return all extra_capabilities belonging to a specific Compute host.""" return IMPL.host_extra_capability_get_all_per_host(host_id) @@ -410,7 +444,6 @@ def host_extra_capability_update(host_extra_capability_id, values): def host_extra_capability_get_all_per_name(host_id, extra_capability_name): return IMPL.host_extra_capability_get_all_per_name(host_id, - extra_capability_name) @@ -421,6 +454,25 @@ def host_get_all_by_queries_including_extracapabilities(queries): return IMPL.host_get_all_by_queries_including_extracapabilities(queries) +# ComputeHostResourceInventory + +def host_resource_inventory_create(values): + """Create a host resource inventory from the values.""" + return IMPL.host_resource_inventory_create(values) + + +def host_resource_inventory_get_all_per_host(host_id): + """Return all resource inventories belonging to a specific Compute host.""" + return IMPL.host_resource_inventory_get_all_per_host(host_id) + + +# ComputeHostTrait + +def host_trait_create(values): + """Create a host trait from the values.""" + return IMPL.host_trait_create(values) + + # FloatingIP reservation def fip_reservation_create(fip_reservation_values): @@ -522,6 +574,310 @@ def reservable_fip_get_all_by_queries(queries): return IMPL.reservable_fip_get_all_by_queries(queries) +@to_dict +def unreservable_fip_get_all_by_queries(queries): + """Returns unreservable floating IPs filtered by an array of queries.""" + return IMPL.unreservable_fip_get_all_by_queries(queries) + + def floatingip_destroy(floatingip_id): """Delete specific floating ip.""" IMPL.floatingip_destroy(floatingip_id) + + +@to_dict +def floatingip_update(floatingip_id, values): + """Update specific floating ip.""" + IMPL.floatingip_update(floatingip_id, values) + + +# Networks + +def network_create(values): + """Create a network from the values.""" + return IMPL.network_create(values) + + +@to_dict +def network_get(network_id): + """Return a specific network.""" + return IMPL.network_get(network_id) + + +@to_dict +def network_list(): + """Return a list of networks.""" + return IMPL.network_list() + + +@to_dict +def network_get_all_by_filters(filters): + """Returns Compute networks filtered by name of the field.""" + return IMPL.network_get_all_by_filters(filters) + + +@to_dict +def network_get_all_by_queries(queries): + """Returns networks filtered by an array of queries.""" + return IMPL.network_get_all_by_queries(queries) + + +@to_dict +def reservable_network_get_all_by_queries(queries): + """Returns reservable networks filtered by an array of queries.""" + return IMPL.reservable_network_get_all_by_queries(queries) + + +@to_dict +def unreservable_network_get_all_by_queries(queries): + """Returns unreservable networks filtered by an array of queries.""" + return IMPL.unreservable_network_get_all_by_queries(queries) + + +def network_destroy(network_id): + """Delete specific network.""" + IMPL.network_destroy(network_id) + + +def network_update(network_id, values): + """Update network.""" + IMPL.network_update(network_id, values) + + +# Network allocations + +def network_allocation_create(allocation_values): + """Create an allocation from the values.""" + return IMPL.network_allocation_create(allocation_values) + + +@to_dict +def network_allocation_get_all_by_values(**kwargs): + """Returns all entries filtered by col=value.""" + return IMPL.network_allocation_get_all_by_values(**kwargs) + + +def network_allocation_destroy(allocation_id): + """Delete specific allocation.""" + IMPL.network_allocation_destroy(allocation_id) + + +# network reservation + +def network_reservation_create(network_reservation_values): + """Create a network reservation from the values.""" + return IMPL.network_reservation_create(network_reservation_values) + + +def network_reservation_get(network_reservation_id): + """Return specific network reservation.""" + return IMPL.network_reservation_get(network_reservation_id) + + +def network_reservation_update(network_reservation_id, + network_reservation_values): + """Update network reservation.""" + return IMPL.network_reservation_update(network_reservation_id, + network_reservation_values) + + +def network_reservation_destroy(network_reservation_id): + """Delete specific network reservation.""" + return IMPL.network_reservation_destroy(network_reservation_id) + + +# NetworkSegmentExtraCapabilities + +def network_extra_capability_create(values): + """Create a network ExtraCapability from the values.""" + return IMPL.network_extra_capability_create(values) + + +def network_extra_capability_get(network_extra_capability_id): + """Return a specific network Extracapability.""" + return IMPL.network_extra_capability_get(network_extra_capability_id) + + +def network_extra_capability_get_all_per_network(network_id): + """Return all extra_capabilities belonging to a specific network.""" + return IMPL.network_extra_capability_get_all_per_network(network_id) + + +def network_extra_capability_destroy(network_extra_capability_id): + """Delete specific network ExtraCapability.""" + IMPL.network_extra_capability_destroy(network_extra_capability_id) + + +def network_extra_capability_update(network_extra_capability_id, values): + """Update specific network ExtraCapability.""" + IMPL.network_extra_capability_update(network_extra_capability_id, values) + + +def network_extra_capability_get_all_per_name(network_id, + extra_capability_name): + return IMPL.network_extra_capability_get_all_per_name( + network_id, extra_capability_name) + + +def network_extra_capability_get_latest_per_name(network_id, + extra_capability_name): + return IMPL.network_extra_capability_get_latest_per_name( + network_id, extra_capability_name + ) + +# Devices + + +@to_dict +def device_create(values): + """Create a device from the values.""" + return IMPL.device_create(values) + + +@to_dict +def device_get(device_id): + """Return a specific device.""" + return IMPL.device_get(device_id) + + +@to_dict +def device_list(): + """Return a list of devices.""" + return IMPL.device_list() + + +@to_dict +def device_get_all_by_filters(filters): + """Returns Compute devices filtered by name of the field.""" + return IMPL.device_get_all_by_filters(filters) + + +@to_dict +def device_get_all_by_queries(queries): + """Returns devices filtered by an array of queries.""" + return IMPL.device_get_all_by_queries(queries) + + +@to_dict +def reservable_device_get_all_by_queries(queries): + """Returns reservable devices filtered by an array of queries.""" + return IMPL.reservable_device_get_all_by_queries(queries) + + +@to_dict +def unreservable_device_get_all_by_queries(queries): + """Returns unreservable devices filtered by an array of queries.""" + return IMPL.unreservable_device_get_all_by_queries(queries) + + +def device_destroy(device_id): + """Delete specific device.""" + IMPL.device_destroy(device_id) + + +def device_update(device_id, values): + """Update device.""" + IMPL.device_update(device_id, values) + + +# Device allocations +@to_dict +def device_allocation_create(allocation_values): + """Create an allocation from the values.""" + return IMPL.device_allocation_create(allocation_values) + + +@to_dict +def device_allocation_get_all_by_values(**kwargs): + """Returns all entries filtered by col=value.""" + return IMPL.device_allocation_get_all_by_values(**kwargs) + + +def device_allocation_destroy(allocation_id): + """Delete specific allocation.""" + IMPL.device_allocation_destroy(allocation_id) + + +def device_allocation_update(allocation_id, allocation_values): + """Update allocation.""" + IMPL.device_allocation_update(allocation_id, allocation_values) + + +# device reservation +@to_dict +def device_reservation_create(device_reservation_values): + """Create a device reservation from the values.""" + return IMPL.device_reservation_create(device_reservation_values) + + +@to_dict +def device_reservation_get(device_reservation_id): + """Return specific device reservation.""" + return IMPL.device_reservation_get(device_reservation_id) + + +def device_reservation_update(device_reservation_id, + device_reservation_values): + """Update device reservation.""" + IMPL.device_reservation_update(device_reservation_id, + device_reservation_values) + + +def device_reservation_destroy(device_reservation_id): + """Delete specific device reservation.""" + IMPL.device_reservation_destroy(device_reservation_id) + + +# DeviceExtraCapabilities + +def device_extra_capability_create(values): + """Create a device ExtraCapability from the values.""" + return IMPL.device_extra_capability_create(values) + + +@to_dict +def device_extra_capability_get(device_extra_capability_id): + """Return a specific device Extracapability.""" + return IMPL.device_extra_capability_get(device_extra_capability_id) + + +def device_extra_capability_get_all_per_device(device_id): + """Return all extra_capabilities belonging to a specific device.""" + return IMPL.device_extra_capability_get_all_per_device(device_id) + + +def device_extra_capability_destroy(device_extra_capability_id): + """Delete specific device ExtraCapability.""" + IMPL.device_extra_capability_destroy(device_extra_capability_id) + + +def device_extra_capability_update(device_extra_capability_id, values): + """Update specific device ExtraCapability.""" + IMPL.device_extra_capability_update(device_extra_capability_id, values) + + +def device_extra_capability_get_all_per_name(device_id, + extra_capability_name): + return IMPL.device_extra_capability_get_all_per_name( + device_id, extra_capability_name) + + +def device_extra_capability_get_latest_per_name(device_id, + extra_capability_name): + return IMPL.device_extra_capability_get_latest_per_name( + device_id, extra_capability_name + ) + +# Resource Properties + + +def resource_properties_list(resource_type): + return IMPL.resource_properties_list(resource_type) + + +def resource_property_update(resource_type, property_name, values): + return IMPL.resource_property_update(resource_type, property_name, values) + + +def resource_property_create(values): + return IMPL.resource_property_create(values) diff --git a/blazar/db/exceptions.py b/blazar/db/exceptions.py index bef0e54da..d5597537e 100644 --- a/blazar/db/exceptions.py +++ b/blazar/db/exceptions.py @@ -40,3 +40,24 @@ class BlazarDBInvalidFilter(BlazarDBException): class BlazarDBInvalidFilterOperator(BlazarDBException): msg_fmt = _('%(filter_operator)s is invalid') + + +class BlazarDBExtraCapabilitiesNotEnabled(BlazarDBException): + msq_fmt = _('%(resource_type)s does not have extra capabilities enabled.') + + +class BlazarDBInvalidExtraCapability(BlazarDBException): + msg_fmt = _('%(property_name)s does not exist for resource type ' + '%(resource_type)s.') + + +class BlazarDBForbiddenResourceProperty(BlazarDBException): + msg_fmt = _('%(property_name)s cannot be set as an extra capability') + +class BlazarDBResourcePropertiesNotEnabled(BlazarDBException): + msq_fmt = _('%(resource_type)s does not have resource properties enabled.') + + +class BlazarDBInvalidResourceProperty(BlazarDBException): + msg_fmt = _('%(property_name)s does not exist for resource type ' + '%(resource_type)s.') diff --git a/blazar/db/migration/alembic_migrations/versions/02e2f2186d98_extra_capability.py b/blazar/db/migration/alembic_migrations/versions/02e2f2186d98_extra_capability.py new file mode 100644 index 000000000..c18931a02 --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/02e2f2186d98_extra_capability.py @@ -0,0 +1,133 @@ +# Copyright 2014 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""extra capability + +Revision ID: 02e2f2186d98 +Revises: 520c9976941c +Create Date: 2020-04-17 15:51:40.542459 + +""" + +# revision identifiers, used by Alembic. +revision = '02e2f2186d98' +down_revision = '520c9976941c' + +from itertools import chain +import uuid + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + + +def upgrade(): + op.create_table('extra_capabilities', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('resource_type', sa.String(255), nullable=False), + sa.Column('capability_name', sa.String(255), + nullable=False), + sa.Column('private', sa.Boolean, nullable=False, + server_default=sa.false()), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('resource_type', 'capability_name')) + + if op.get_bind().engine.name != 'sqlite': + connection = op.get_bind() + + host_query = connection.execute(""" + SELECT DISTINCT "physical:host", capability_name + FROM computehost_extra_capabilities;""") + network_query = connection.execute(""" + SELECT DISTINCT "network", capability_name + FROM networksegment_extra_capabilities;""") + + capability_values = [ + (str(uuid.uuid4()), resource_type, capability_name) + for resource_type, capability_name + in chain(host_query.fetchall(), network_query.fetchall())] + + if capability_values: + insert = """ + INSERT INTO extra_capabilities + (id, resource_type, capability_name) + VALUES {};""" + connection.execute( + insert.format(', '.join(map(str, capability_values)))) + + op.add_column('computehost_extra_capabilities', + sa.Column('capability_id', sa.String(length=255), + nullable=False)) + op.add_column('networksegment_extra_capabilities', + sa.Column('capability_id', sa.String(length=255), + nullable=False)) + + connection.execute(""" + UPDATE computehost_extra_capabilities c + LEFT JOIN extra_capabilities e + ON e.capability_name = c.capability_name + SET c.capability_id = e.id;""") + connection.execute(""" + UPDATE networksegment_extra_capabilities n + LEFT JOIN extra_capabilities e + ON e.capability_name = n.capability_name + SET n.capability_id = e.id;""") + + op.create_foreign_key('computehost_extra_capability_id_fk', + 'computehost_extra_capabilities', + 'extra_capabilities', ['capability_id'], ['id']) + op.create_foreign_key('networksegment_extra_capability_id_fk', + 'networksegment_extra_capabilities', + 'extra_capabilities', ['capability_id'], ['id']) + + op.drop_column('computehost_extra_capabilities', 'capability_name') + op.drop_column('networksegment_extra_capabilities', 'capability_name') + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('networksegment_extra_capabilities', + sa.Column('capability_name', mysql.VARCHAR(length=64), + nullable=False)) + op.add_column('computehost_extra_capabilities', + sa.Column('capability_name', mysql.VARCHAR(length=64), + nullable=False)) + + if op.get_bind().engine.name != 'sqlite': + connection = op.get_bind() + connection.execute(""" + UPDATE networksegment_extra_capabilities n + LEFT JOIN extra_capabilities e + ON e.id = n.capability_id + SET n.capability_name = e.capability_name;""") + + connection.execute(""" + UPDATE computehost_extra_capabilities c + LEFT JOIN extra_capabilities e + ON e.id=c.capability_id + SET c.capability_name = e.capability_name;""") + + op.drop_constraint('networksegment_extra_capability_id_fk', + 'networksegment_extra_capabilities', + type_='foreignkey') + op.drop_column('networksegment_extra_capabilities', 'capability_id') + + op.drop_constraint('computehost_extra_capability_id_fk', + 'computehost_extra_capabilities', + type_='foreignkey') + op.drop_column('computehost_extra_capabilities', 'capability_id') + op.drop_table('extra_capabilities') diff --git a/blazar/db/migration/alembic_migrations/versions/42c7fd6e792e_add_device_reservation.py b/blazar/db/migration/alembic_migrations/versions/42c7fd6e792e_add_device_reservation.py new file mode 100644 index 000000000..f6670c520 --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/42c7fd6e792e_add_device_reservation.py @@ -0,0 +1,113 @@ +# Copyright 2021 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""add device reservation + +Revision ID: 42c7fd6e792e +Revises: 02e2f2186d98 +Create Date: 2021-06-22 15:27:00.239725 + +""" + +# revision identifiers, used by Alembic. +revision = '42c7fd6e792e' +down_revision = '02e2f2186d98' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('devices', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('device_type', + sa.Enum('container', 'vm', 'shell', + name='allowed_device_types'), + nullable=False), + sa.Column('device_driver', sa.Enum( + 'zun', name='allowed_device_drivers'), nullable=False), + sa.Column('reservable', sa.Boolean(), + server_default=sa.text('true'), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('device_extra_capabilities', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('device_id', sa.String( + length=36), nullable=False), + sa.Column('capability_id', sa.String( + length=255), nullable=False), + sa.Column('capability_value', sa.Text().with_variant( + mysql.MEDIUMTEXT(), 'mysql'), nullable=False), + sa.ForeignKeyConstraint( + ['capability_id'], ['extra_capabilities.id'], ), + sa.ForeignKeyConstraint(['device_id'], ['devices.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('device_allocations', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.String(length=36), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('device_id', sa.String( + length=36), nullable=True), + sa.Column('reservation_id', sa.String( + length=36), nullable=True), + sa.ForeignKeyConstraint(['device_id'], ['devices.id'], ), + sa.ForeignKeyConstraint(['reservation_id'], [ + 'reservations.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('device_reservations', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.String(length=36), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('reservation_id', sa.String( + length=36), nullable=True), + sa.Column('count_range', sa.String( + length=36), nullable=True), + sa.Column('resource_properties', sa.Text().with_variant( + mysql.MEDIUMTEXT(), 'mysql'), nullable=True), + sa.Column('before_end', sa.String( + length=36), nullable=True), + sa.ForeignKeyConstraint(['reservation_id'], [ + 'reservations.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.alter_column('instance_reservations', 'affinity', + existing_type=mysql.TINYINT(display_width=1), + nullable=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('instance_reservations', 'affinity', + existing_type=mysql.TINYINT(display_width=1), + nullable=True) + op.drop_table('device_reservations') + op.drop_table('device_allocations') + op.drop_table('device_extra_capabilities') + op.drop_table('devices') + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/4b7bdec3ae61_soft_delete_resources.py b/blazar/db/migration/alembic_migrations/versions/4b7bdec3ae61_soft_delete_resources.py new file mode 100644 index 000000000..5ba88b6dd --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/4b7bdec3ae61_soft_delete_resources.py @@ -0,0 +1,87 @@ +# Copyright 2022 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Soft delete resources + +Revision ID: 4b7bdec3ae61 +Revises: beebda67d373 +Create Date: 2022-05-04 17:19:47.103803 + +""" + +# revision identifiers, used by Alembic. +revision = '4b7bdec3ae61' +down_revision = '8fe921c50030' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('extra_capabilities', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('extra_capabilities', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('computehosts', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('computehosts', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('computehost_extra_capabilities', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('computehost_extra_capabilities', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('floatingips', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('floatingips', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('network_segments', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('network_segments', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('networksegment_extra_capabilities', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('networksegment_extra_capabilities', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('devices', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('devices', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('device_extra_capabilities', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('device_extra_capabilities', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('extra_capabilities', 'deleted') + op.drop_column('extra_capabilities', 'deleted_at') + op.drop_column('computehosts', 'deleted') + op.drop_column('computehosts', 'deleted_at') + op.drop_column('computehost_extra_capabilities', 'deleted') + op.drop_column('computehost_extra_capabilities', 'deleted_at') + op.drop_column('floatingips', 'deleted') + op.drop_column('floatingips', 'deleted_at') + op.drop_column('network_segments', 'deleted') + op.drop_column('network_segments', 'deleted_at') + op.drop_column('networksegment_extra_capabilities', 'deleted') + op.drop_column('networksegment_extra_capabilities', 'deleted_at') + op.drop_column('devices', 'deleted') + op.drop_column('devices', 'deleted_at') + op.drop_column('device_extra_capabilities', 'deleted') + op.drop_column('device_extra_capabilities', 'deleted_at') + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/4fe5e44880da_add_computehost_indexes.py b/blazar/db/migration/alembic_migrations/versions/4fe5e44880da_add_computehost_indexes.py new file mode 100644 index 000000000..b0df4ee11 --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/4fe5e44880da_add_computehost_indexes.py @@ -0,0 +1,45 @@ +# Copyright 2026 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""add_computehost_indexes + +Revision ID: 4fe5e44880da +Revises: bf2011753964 +Create Date: 2026-06-04 21:33:06.515431 + +""" + +# revision identifiers, used by Alembic. +revision = '4fe5e44880da' +down_revision = 'bf2011753964' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index('ix_computehost_allocations_compute_host_id', 'computehost_allocations', ['compute_host_id'], unique=False) + op.create_index('ix_computehost_allocations_reservation_id', 'computehost_allocations', ['reservation_id'], unique=False) + op.create_index('ix_computehost_extra_capabilities_computehost_id', 'computehost_extra_capabilities', ['computehost_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ix_computehost_extra_capabilities_computehost_id', table_name='computehost_extra_capabilities') + op.drop_index('ix_computehost_allocations_reservation_id', table_name='computehost_allocations') + op.drop_index('ix_computehost_allocations_compute_host_id', table_name='computehost_allocations') + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/520c9976941c_add_tables_for_network_segment_.py b/blazar/db/migration/alembic_migrations/versions/520c9976941c_add_tables_for_network_segment_.py new file mode 100644 index 000000000..5f27f445c --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/520c9976941c_add_tables_for_network_segment_.py @@ -0,0 +1,100 @@ +# Copyright 2018 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add tables for network segment reservation + +Revision ID: 520c9976941c +Revises: 6ef879d2080d +Create Date: 2019-01-07 13:47:08.073385 + +""" + +# revision identifiers, used by Alembic. +revision = '520c9976941c' +down_revision = '6ef879d2080d' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.mysql import MEDIUMTEXT + + +def MediumText(): + return sa.Text().with_variant(MEDIUMTEXT(), 'mysql') + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'network_segments', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('network_type', sa.String(length=255), nullable=False), + sa.Column('physical_network', sa.String(length=255), nullable=True), + sa.Column('segment_id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('network_type', 'physical_network', 'segment_id') + ) + op.create_table( + 'networksegment_extra_capabilities', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('network_id', sa.String(length=36), nullable=False), + sa.Column('capability_name', sa.String(length=64), nullable=False), + sa.Column('capability_value', MediumText(), nullable=False), + sa.ForeignKeyConstraint(['network_id'], ['network_segments.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table( + 'network_allocations', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.String(length=36), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('network_id', sa.String(length=36), nullable=True), + sa.Column('reservation_id', sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint(['network_id'], ['network_segments.id'], ), + sa.ForeignKeyConstraint(['reservation_id'], ['reservations.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table( + 'network_reservations', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.String(length=36), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('reservation_id', sa.String(length=36), nullable=True), + sa.Column('resource_properties', MediumText(), nullable=True), + sa.Column('network_properties', MediumText(), nullable=True), + sa.Column('before_end', sa.String(length=36), nullable=True), + sa.Column('network_name', sa.String(length=255), nullable=True), + sa.Column('network_description', sa.String(length=255), nullable=True), + sa.Column('network_id', sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint(['reservation_id'], ['reservations.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('network_reservations') + op.drop_table('network_allocations') + op.drop_table('networksegment_extra_capabilities') + op.drop_table('network_segments') + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/553383923ca0_add_compute_host_inventory_and_traits.py b/blazar/db/migration/alembic_migrations/versions/553383923ca0_add_compute_host_inventory_and_traits.py new file mode 100644 index 000000000..53d4a67a1 --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/553383923ca0_add_compute_host_inventory_and_traits.py @@ -0,0 +1,63 @@ +# Copyright 2024 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add compute host inventory and traits + +Revision ID: 553383923ca0 +Revises: 95bd85fe13f0 +Create Date: 2024-04-29 17:40:05.148493 + +""" + +# revision identifiers, used by Alembic. +revision = '553383923ca0' +down_revision = '95bd85fe13f0' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table( + 'computehost_resource_inventory', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('computehost_id', sa.String(length=36), nullable=True), + sa.Column('resource_class', sa.String(length=255), nullable=False), + sa.Column('total', sa.Integer(), nullable=False), + sa.Column('reserved', sa.Integer(), nullable=False), + sa.Column('min_unit', sa.Integer(), nullable=False), + sa.Column('max_unit', sa.Integer(), nullable=False), + sa.Column('step_size', sa.Integer(), nullable=False), + sa.Column('allocation_ratio', sa.Float(), nullable=False), + sa.ForeignKeyConstraint(['computehost_id'], ['computehosts.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table( + 'computehost_trait', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('computehost_id', sa.String(length=36), nullable=True), + sa.Column('trait', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['computehost_id'], ['computehosts.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + +def downgrade(): + op.drop_table('computehost_trait') + op.drop_table('computehost_resource_inventory') diff --git a/blazar/db/migration/alembic_migrations/versions/6ef879d2080d_add_on_start_into_computehost_.py b/blazar/db/migration/alembic_migrations/versions/6ef879d2080d_add_on_start_into_computehost_.py new file mode 100644 index 000000000..01509f481 --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/6ef879d2080d_add_on_start_into_computehost_.py @@ -0,0 +1,43 @@ +# Copyright 2018 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add on_start into computehost_reservations table + +Revision ID: 6ef879d2080d +Revises: cffa05fd6878 +Create Date: 2018-12-28 14:53:05.762902 + +""" + +# revision identifiers, used by Alembic. +revision = '6ef879d2080d' +down_revision = 'cffa05fd6878' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + 'computehost_reservations', + sa.Column('on_start', sa.String(length=50), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('computehost_reservations', 'on_start') + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/8fe921c50030_add_k3s_device_driver.py b/blazar/db/migration/alembic_migrations/versions/8fe921c50030_add_k3s_device_driver.py new file mode 100644 index 000000000..81b865f05 --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/8fe921c50030_add_k3s_device_driver.py @@ -0,0 +1,41 @@ +# Copyright 2022 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""add k8s device driver + +Revision ID: 8fe921c50030 +Revises: 42c7fd6e792e +Create Date: 2022-01-27 20:48:00.443041 + +""" + +# revision identifiers, used by Alembic. +revision = '8fe921c50030' +down_revision = '42c7fd6e792e' + +from alembic import op + + +def upgrade(): + connection = op.get_bind() + connection.execute( + "ALTER TABLE devices MODIFY COLUMN device_driver ENUM('zun', 'k8s');") + + +def downgrade(): + connection = op.get_bind() + connection.execute("UPDATE devices SET device_driver = 'zun';") + connection.execute( + "ALTER TABLE devices MODIFY COLUMN device_driver ENUM('zun')") diff --git a/blazar/db/migration/alembic_migrations/versions/95bd85fe13f0_instance_reservation_affinity_nullable.py b/blazar/db/migration/alembic_migrations/versions/95bd85fe13f0_instance_reservation_affinity_nullable.py new file mode 100644 index 000000000..5b6a6ac85 --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/95bd85fe13f0_instance_reservation_affinity_nullable.py @@ -0,0 +1,48 @@ +# Copyright 2025 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fix instance reservations affinity nullable + +Chameleon commit 73864039c1d1bc8efe709dd702e2badb7c2fa486 accidentally reverted +an upstream migration, revert the revert here. + +Revision ID: 95bd85fe13f0 +Revises: 4fe5e44880da +Create Date: 2025-01-17 16:05:26.255224 + +""" + +# revision identifiers, used by Alembic. +revision = '95bd85fe13f0' +down_revision = '4fe5e44880da' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('instance_reservations', 'affinity', + existing_type=mysql.TINYINT(display_width=1), + nullable=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('instance_reservations', 'affinity', + existing_type=mysql.TINYINT(display_width=1), + nullable=False) + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/9eec70cbc562_add_is_unique_field_in_extra_.py b/blazar/db/migration/alembic_migrations/versions/9eec70cbc562_add_is_unique_field_in_extra_.py new file mode 100644 index 000000000..f3c3242ef --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/9eec70cbc562_add_is_unique_field_in_extra_.py @@ -0,0 +1,41 @@ +# Copyright 2024 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add is_unique field in extra capabilities + +Revision ID: 9eec70cbc562 +Revises: d5a379ff6ba3 +Create Date: 2024-03-15 17:36:26.552008 + +""" + +# revision identifiers, used by Alembic. +revision = '9eec70cbc562' +down_revision = 'd5a379ff6ba3' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + op.create_unique_constraint(None, 'devices', ['name']) + op.add_column('extra_capabilities', sa.Column('is_unique', sa.Boolean(), server_default=sa.text('false'), nullable=False)) + # ### end Alembic commands ### + + +def downgrade(): + op.drop_column('extra_capabilities', 'is_unique') + op.drop_constraint(None, 'devices', type_='unique') + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/a167bcf47857_add_flavor_before_end.py b/blazar/db/migration/alembic_migrations/versions/a167bcf47857_add_flavor_before_end.py new file mode 100644 index 000000000..48399aa51 --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/a167bcf47857_add_flavor_before_end.py @@ -0,0 +1,41 @@ +# Copyright 2025 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add flavor before end column + +Revision ID: a167bcf47857 +Revises: 95bd85fe13f0 +Create Date: 2025-01-27 19:02:30.182818 + +""" + +# revision identifiers, used by Alembic. +revision = 'a167bcf47857' +down_revision = '553383923ca0' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('instance_reservations', sa.Column('before_end', sa.String(length=36), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('instance_reservations', 'before_end') + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/afd0a1c7748a_add_soft_delete_for_floating_ip_.py b/blazar/db/migration/alembic_migrations/versions/afd0a1c7748a_add_soft_delete_for_floating_ip_.py new file mode 100644 index 000000000..9f58c9ec3 --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/afd0a1c7748a_add_soft_delete_for_floating_ip_.py @@ -0,0 +1,57 @@ +# Copyright 2019 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add soft delete for floating IP reservation + +Revision ID: afd0a1c7748a +Revises: f4084140f608 +Create Date: 2019-03-28 14:14:46.125239 + +""" + +# revision identifiers, used by Alembic. +revision = 'afd0a1c7748a' +down_revision = 'f4084140f608' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('floatingip_allocations', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('floatingip_allocations', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('floatingip_reservations', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('floatingip_reservations', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('required_floatingips', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('required_floatingips', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('required_floatingips', 'deleted_at') + op.drop_column('required_floatingips', 'deleted') + op.drop_column('floatingip_reservations', 'deleted_at') + op.drop_column('floatingip_reservations', 'deleted') + op.drop_column('floatingip_allocations', 'deleted_at') + op.drop_column('floatingip_allocations', 'deleted') + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/bf2011753964_add_lease_indexes.py b/blazar/db/migration/alembic_migrations/versions/bf2011753964_add_lease_indexes.py new file mode 100644 index 000000000..5e32290ca --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/bf2011753964_add_lease_indexes.py @@ -0,0 +1,49 @@ +# Copyright 2026 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""add_lease_indexes + +Revision ID: bf2011753964 +Revises: ee3b2513b59f +Create Date: 2026-06-03 19:54:19.674143 + +""" + +# revision identifiers, used by Alembic. +revision = 'bf2011753964' +down_revision = 'ee3b2513b59f' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index('ix_leases_end_date', 'leases', ['end_date'], unique=False) + op.create_index('ix_leases_project_id', 'leases', ['project_id'], unique=False) + op.create_index('ix_leases_status', 'leases', ['status'], unique=False) + op.create_index('ix_leases_project_end_date', 'leases', ['project_id', 'end_date'], unique=False) + op.create_index('ix_leases_status_end_date', 'leases', ['status', 'end_date'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ix_leases_status', table_name='leases') + op.drop_index('ix_leases_project_id', table_name='leases') + op.drop_index('ix_leases_end_date', table_name='leases') + op.drop_index('ix_leases_status_end_date', table_name='leases') + op.drop_index('ix_leases_project_end_date', table_name='leases') + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/cffa05fd6878_add_soft_delete_to_most_blazar_models.py b/blazar/db/migration/alembic_migrations/versions/cffa05fd6878_add_soft_delete_to_most_blazar_models.py new file mode 100644 index 000000000..31f020bef --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/cffa05fd6878_add_soft_delete_to_most_blazar_models.py @@ -0,0 +1,76 @@ +# Copyright 2018 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add soft delete to most Blazar models + +Revision ID: cffa05fd6878 +Revises: 35b314cd39ee +Create Date: 2018-07-25 12:27:53.163504 + +""" + +# revision identifiers, used by Alembic. +revision = 'cffa05fd6878' +down_revision = 'afd0a1c7748a' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('computehost_allocations', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('computehost_allocations', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.drop_constraint(u'computehost_allocations_ibfk_1', + 'computehost_allocations', type_='foreignkey') + op.add_column('computehost_reservations', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('computehost_reservations', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('events', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('events', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('instance_reservations', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('instance_reservations', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('leases', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('leases', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + op.add_column('reservations', + sa.Column('deleted', sa.String(length=36), nullable=True)) + op.add_column('reservations', + sa.Column('deleted_at', sa.DateTime(), nullable=True)) + + +def downgrade(): + op.drop_column('reservations', 'deleted_at') + op.drop_column('reservations', 'deleted') + op.drop_column('leases', 'deleted_at') + op.drop_column('leases', 'deleted') + op.drop_column('instance_reservations', 'deleted_at') + op.drop_column('instance_reservations', 'deleted') + op.drop_column('events', 'deleted_at') + op.drop_column('events', 'deleted') + op.drop_column('computehost_reservations', 'deleted_at') + op.drop_column('computehost_reservations', 'deleted') + op.create_foreign_key(u'computehost_allocations_ibfk_1', + 'computehost_allocations', 'computehosts', + ['compute_host_id'], ['id']) + op.drop_column('computehost_allocations', 'deleted_at') + op.drop_column('computehost_allocations', 'deleted') diff --git a/blazar/db/migration/alembic_migrations/versions/d5a379ff6ba3_add_disabled_in_compute_host.py b/blazar/db/migration/alembic_migrations/versions/d5a379ff6ba3_add_disabled_in_compute_host.py new file mode 100644 index 000000000..c2d48cdbd --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/d5a379ff6ba3_add_disabled_in_compute_host.py @@ -0,0 +1,47 @@ +# Copyright 2023 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""add disabled in compute host + +Revision ID: d5a379ff6ba3 +Revises: 4b7bdec3ae61 +Create Date: 2023-11-17 10:48:50.403580 + +""" + +# revision identifiers, used by Alembic. +revision = 'd5a379ff6ba3' +down_revision = '4b7bdec3ae61' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('computehosts', sa.Column('disabled', sa.Boolean(), server_default=sa.text('false'), nullable=False)) + op.alter_column('devices', 'device_driver', + existing_type=mysql.ENUM('zun', 'k8s'), + nullable=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('devices', 'device_driver', + existing_type=mysql.ENUM('zun', 'k8s'), + nullable=True) + op.drop_column('computehosts', 'disabled') + # ### end Alembic commands ### diff --git a/blazar/db/migration/alembic_migrations/versions/ee3b2513b59f_rename_extra_capability.py b/blazar/db/migration/alembic_migrations/versions/ee3b2513b59f_rename_extra_capability.py new file mode 100644 index 000000000..676c9fb47 --- /dev/null +++ b/blazar/db/migration/alembic_migrations/versions/ee3b2513b59f_rename_extra_capability.py @@ -0,0 +1,100 @@ +# Copyright 2024 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""rename extra capability to resource property + +Revision ID: ee3b2513b59f +Revises: d5a379ff6ba3 +Create Date: 2024-01-06 00:22:00.128435 + +""" + +# revision identifiers, used by Alembic. +revision = 'ee3b2513b59f' +down_revision = '9eec70cbc562' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + # Drop existing foreign key constraints + op.drop_constraint('computehost_extra_capability_id_fk', + 'computehost_extra_capabilities', + type_='foreignkey') + op.drop_constraint('networksegment_extra_capability_id_fk', + 'networksegment_extra_capabilities', + type_='foreignkey') + op.drop_constraint('device_extra_capabilities_ibfk_2', + 'device_extra_capabilities', + type_='foreignkey') + # Rename the ExtraCapability model to ResourceProperty + op.rename_table('extra_capabilities', 'resource_properties') + # Update the column names and types + op.alter_column('resource_properties', 'capability_name', + new_column_name='property_name', existing_type=sa.String(255)) + op.alter_column('computehost_extra_capabilities', 'capability_id', + new_column_name='property_id', existing_type=sa.String(255)) + op.alter_column('networksegment_extra_capabilities', 'capability_id', + new_column_name='property_id', existing_type=sa.String(255)) + op.alter_column('device_extra_capabilities', 'capability_id', + new_column_name='property_id', existing_type=sa.String(255)) + # Recreate foreign key constraints + op.create_foreign_key('computehost_extra_capability_id_fk', + 'computehost_extra_capabilities', + 'resource_properties', ['property_id'], ['id']) + op.create_foreign_key('networksegment_extra_capability_id_fk', + 'networksegment_extra_capabilities', + 'resource_properties', ['property_id'], ['id']) + op.create_foreign_key('device_extra_capabilities_ibfk_2', + 'device_extra_capabilities', + 'resource_properties', ['property_id'], ['id']) + +def downgrade(): + # Drop foreign key constraints + op.drop_constraint('computehost_extra_capability_id_fk', + 'computehost_extra_capabilities', + type_='foreignkey') + op.drop_constraint('networksegment_extra_capability_id_fk', + 'networksegment_extra_capabilities', + type_='foreignkey') + op.drop_constraint('device_extra_capabilities_ibfk_2', + 'device_extra_capabilities', + type_='foreignkey') + + # Rename the ResourceProperty model back to ExtraCapability + op.rename_table('resource_properties', 'extra_capabilities') + + # Update the column names and types + op.alter_column('extra_capabilities', 'property_name', + new_column_name='capability_name', existing_type=sa.String(255)) + op.alter_column('computehost_extra_capabilities', 'property_id', + new_column_name='capability_id', existing_type=sa.String(255)) + op.alter_column('networksegment_extra_capabilities', 'property_id', + new_column_name='capability_id', existing_type=sa.String(255)) + op.alter_column('device_extra_capabilities', 'property_id', + new_column_name='capability_id', existing_type=sa.String(255)) + + # Recreate foreign key constraints + op.create_foreign_key('computehost_extra_capability_id_fk', + 'computehost_extra_capabilities', + 'extra_capabilities', ['capability_id'], ['id']) + op.create_foreign_key('networksegment_extra_capability_id_fk', + 'networksegment_extra_capabilities', + 'extra_capabilities', ['capability_id'], ['id']) + op.create_foreign_key('device_extra_capabilities_ibfk_2', + 'device_extra_capabilities', + 'extra_capabilities', ['capability_id'], ['id']) diff --git a/blazar/db/sqlalchemy/api.py b/blazar/db/sqlalchemy/api.py index 97752d1d9..051b8a13b 100644 --- a/blazar/db/sqlalchemy/api.py +++ b/blazar/db/sqlalchemy/api.py @@ -18,37 +18,59 @@ import sys from oslo_config import cfg + +from blazar.db import exceptions as db_exc +from blazar.db.sqlalchemy import facade_wrapper +from blazar.db.sqlalchemy import models from oslo_db import exception as common_db_exc from oslo_db.sqlalchemy import session as db_session +from oslo_db.sqlalchemy import utils as sqlalchemyutils from oslo_log import log as logging import sqlalchemy as sa from sqlalchemy.sql.expression import asc from sqlalchemy.sql.expression import desc -from blazar.db import exceptions as db_exc -from blazar.db.sqlalchemy import facade_wrapper -from blazar.db.sqlalchemy import models +RESOURCE_PROPERTY_MODELS = { + 'physical:host': models.ComputeHostExtraCapability, + 'network': models.NetworkSegmentExtraCapability, + 'device': models.DeviceExtraCapability, +} + +FORBIDDEN_RESOURCE_PROPERTY_NAMES = ["id", "reservable"] LOG = logging.getLogger(__name__) get_engine = facade_wrapper.get_engine -get_session = facade_wrapper.get_session +cfg.CONF.register_opt(cfg.BoolOpt( + 'include_deleted', default=False, help='Include deleted in queries (used by scripts)')) + +get_session = facade_wrapper.get_session def get_backend(): """The backend is this module itself.""" return sys.modules[__name__] -def model_query(model, session=None): +def _read_deleted_filter(query, db_model, deleted): + if 'deleted' not in db_model.__table__.columns: + return query + + default_deleted_value = None + if not deleted and not cfg.CONF.include_deleted: + query = query.filter(db_model.deleted == default_deleted_value) + return query + + +def model_query(model, session=None, deleted=False): """Query helper. :param model: base model to query """ session = session or get_session() - return session.query(model) + return _read_deleted_filter(session.query(model), model, deleted) def setup_db(): @@ -132,7 +154,7 @@ def reservation_get_all(): def reservation_get_all_by_lease_id(lease_id): reservations = (model_query(models.Reservation, - get_session()).filter_by(lease_id=lease_id)) + get_session()).filter_by(lease_id=lease_id)) return reservations.all() @@ -175,6 +197,34 @@ def reservation_update(reservation_id, values): return reservation_get(reservation_id) +def _reservation_destroy(session, reservation): + if reservation.instance_reservation: + reservation.instance_reservation.soft_delete(session=session) + + if reservation.computehost_reservation: + reservation.computehost_reservation.soft_delete(session=session) + + if reservation.network_reservation: + reservation.network_reservation.soft_delete(session=session) + + if reservation.floatingip_reservation: + reservation.floatingip_reservation.soft_delete(session=session) + + if reservation.computehost_allocations: + for computehost_allocation in reservation.computehost_allocations: + computehost_allocation.soft_delete(session=session) + + if reservation.network_allocations: + for network_allocation in reservation.network_allocations: + network_allocation.soft_delete(session=session) + + if reservation.floatingip_allocations: + for floatingip_allocation in reservation.floatingip_allocations: + floatingip_allocation.soft_delete(session=session) + + reservation.soft_delete(session=session) + + def reservation_destroy(reservation_id): session = get_session() with session.begin(): @@ -185,7 +235,7 @@ def reservation_destroy(reservation_id): raise db_exc.BlazarDBNotFound(id=reservation_id, model='Reservation') - session.delete(reservation) + _reservation_destroy(session, reservation) # Lease @@ -211,11 +261,107 @@ def lease_get_all_by_user(user_id): raise NotImplementedError -def lease_list(project_id=None): - query = model_query(models.Lease, get_session()) +def hosts_in_lease(lease_id): + query = model_query(models.ComputeHost, get_session()) + query = query.join( + models.ComputeHostAllocation, + models.ComputeHostAllocation.compute_host_id == models.ComputeHost.id + ).join( + models.ComputeHostReservation, + models.ComputeHostReservation.reservation_id == models.ComputeHostAllocation.reservation_id + ).join( + models.Reservation, + models.Reservation.id == models.ComputeHostReservation.reservation_id + ).join( + models.Lease, + models.Lease.id == models.Reservation.lease_id + ).filter(models.Lease.id == lease_id) + return query.all() + + +def devices_in_lease(lease_id): + query = model_query(models.Device, get_session()) + query = query.join( + models.DeviceAllocation, + models.DeviceAllocation.device_id == models.Device.id + ).join( + models.DeviceReservation, + models.DeviceReservation.reservation_id == models.DeviceAllocation.reservation_id + ).join( + models.Reservation, + models.Reservation.id == models.DeviceReservation.reservation_id + ).join( + models.Lease, + models.Lease.id == models.Reservation.lease_id + ).filter(models.Lease.id == lease_id) + return query.all() + + +def networks_in_lease(lease_id): + query = model_query(models.NetworkSegment, get_session()) + query = query.join( + models.NetworkAllocation, + models.NetworkAllocation.network_id == models.NetworkSegment.id + ).join( + models.NetworkReservation, + models.NetworkReservation.reservation_id == models.NetworkAllocation.reservation_id + ).join( + models.Reservation, + models.Reservation.id == models.NetworkReservation.reservation_id + ).join( + models.Lease, + models.Lease.id == models.Reservation.lease_id + ).filter(models.Lease.id == lease_id) + return query.all() + + +def lease_list( + project_id=None, + status=None, + lease_id=None, + lease_name=None, + marker=None, + limit=None, + sort_dir="desc", + sort_key="end_date" + ): + # Wait to select relations until after pagination to avoid the large join + query = model_query(models.Lease, get_session()).options( + sa.orm.noload(models.Lease.reservations), + sa.orm.noload(models.Lease.events), + ) if project_id is not None: query = query.filter_by(project_id=project_id) - return query.all() + if status is not None: + query = query.filter_by(status=status) + if lease_id is not None: + query = query.filter_by(id=lease_id) + if lease_name is not None: + query = query.filter_by(name=lease_name) + marker_obj = None + if marker: + marker_obj = lease_get(marker) + if not marker_obj: + # raise not found error + raise db_exc.BlazarDBNotFound(id=marker, model='Lease') + query = sqlalchemyutils.paginate_query( + query, + models.Lease, + limit, + sort_keys=[sort_key, "id"], + sort_dirs=[sort_dir, "asc"], + marker=marker_obj, + ) + lease_ids = [lease.id for lease in query.all()] + if not lease_ids: + return [] + query = model_query(models.Lease, get_session()).options( + sa.orm.selectinload(models.Lease.reservations), + sa.orm.selectinload(models.Lease.events), + ).filter(models.Lease.id.in_(lease_ids)) + leases = query.all() + lease_map = {lease.id: lease for lease in leases} + return [lease_map[lease_id] for lease_id in lease_ids if lease_id in lease_map] def lease_create(values): @@ -279,12 +425,18 @@ def lease_destroy(lease_id): # raise not found error raise db_exc.BlazarDBNotFound(id=lease_id, model='Lease') - session.delete(lease) + for reservation in lease.reservations: + _reservation_destroy(session, reservation) + + for event in lease.events: + event.soft_delete(session=session) + + lease.soft_delete(session=session) # Event -def _event_get(session, event_id): - query = model_query(models.Event, session) +def _event_get(session, event_id, deleted=False): + query = model_query(models.Event, session, deleted=deleted) return query.filter_by(id=event_id).first() @@ -374,7 +526,8 @@ def event_update(event_id, values): session = get_session() with session.begin(): - event = _event_get(session, event_id) + # NOTE(jason): Allow updating soft-deleted events + event = _event_get(session, event_id, deleted=True) event.update(values) event.save(session=session) @@ -390,7 +543,7 @@ def event_destroy(event_id): # raise not found error raise db_exc.BlazarDBNotFound(id=event_id, model='Event') - session.delete(event) + event.soft_delete(session=session) # ComputeHostReservation @@ -459,7 +612,7 @@ def host_reservation_destroy(host_reservation_id): raise db_exc.BlazarDBNotFound( id=host_reservation_id, model='ComputeHostReservation') - session.delete(host_reservation) + host_reservation.soft_delete(session=session) # InstanceReservation @@ -513,7 +666,8 @@ def instance_reservation_destroy(instance_reservation_id): if not instance: raise db_exc.BlazarDBNotFound( id=instance_reservation_id, model='InstanceReservations') - session.delete(instance) + + instance.soft_delete(session=session) # ComputeHostAllocation @@ -582,7 +736,7 @@ def host_allocation_destroy(host_allocation_id): raise db_exc.BlazarDBNotFound( id=host_allocation_id, model='ComputeHostAllocation') - session.delete(host_allocation) + host_allocation.soft_delete(session=session) # ComputeHost @@ -635,7 +789,8 @@ def host_get_all_by_queries(queries): '!=': ['ne', lambda a, b: a == b], } - hosts = [] + # loop over input queries. For each one, construct a sqlalchemy filter + # clause and append to hosts_query for query in queries: try: key, op, value = query.split(' ', 2) @@ -663,29 +818,46 @@ def host_get_all_by_queries(queries): hosts_query = hosts_query.filter(filt) else: - # looking for extra capabilities matches - extra_filter = model_query( - models.ComputeHostExtraCapability, get_session() - ).filter(models.ComputeHostExtraCapability.capability_name == key - ).all() - if not extra_filter: + # Since `key` does not map to a host column directly, check if it + # maps to a resource property joined to at least one host by extra + # capability. + cap = models.ComputeHostExtraCapability + prop = models.ResourceProperty + # capability rows for this property name + caps_query = (model_query(cap, get_session()) + .join(prop, cap.property_id == prop.id) + .filter(prop.property_name == key)) + if not caps_query.first(): raise db_exc.BlazarDBNotFound( id=key, model='ComputeHostExtraCapability') - for host in extra_filter: - if op in oper and oper[op][1](host.capability_value, value): - hosts.append(host.computehost_id) - elif op not in oper: - msg = 'Operator %s for extra capabilities not implemented' - raise NotImplementedError(msg % op) - - # We must also avoid selecting any host which doesn't have the - # extra capability present. - all_hosts = [h.id for h in hosts_query.all()] - extra_filter_hosts = [h.computehost_id for h in extra_filter] - hosts += [h for h in all_hosts if h not in extra_filter_hosts] - - return hosts_query.filter(~models.ComputeHost.id.in_(hosts)).all() + # Check if requested operator is supported for capabilities + if op not in oper: + msg = "Operator %s for resource properties not implemented" + raise NotImplementedError(msg % op) + + # the oper dict maps an input symbol, e.g. `>=` to a sqlalchemy + # operator, e.g. `ge`, and to a python lambda implementing the op. + # look up the sqlalchemy operator, then look up which prefix form + # is a method on the extra capabilites column + op_name = oper[op][0] + try: + attr = [ e for e in ["%s", "%s_", "__%s__"] + if hasattr(cap.capability_value, e % op_name)][0] % op_name + except IndexError: + raise db_exc.BlazarDBInvalidFilterOperator(filter_operator=op) + value_filter = getattr(cap.capability_value, attr)(value) + + # keep hosts that have a capability row matching that clause + hosts_query = hosts_query.filter( + caps_query.filter(cap.computehost_id == models.ComputeHost.id) + .filter(value_filter) + .exists() + ) + + # execute the constructed db query, containing filter clauses for each input + # query. + return hosts_query.all() def reservable_host_get_all_by_queries(queries): @@ -751,13 +923,27 @@ def host_destroy(host_id): # raise not found error raise db_exc.BlazarDBNotFound(id=host_id, model='Host') - session.delete(host) + host.soft_delete(session=session) + # Also delete this host's extra capabilities + for capability in host.computehost_extra_capabilities: + capability.soft_delete(session=session) # ComputeHostExtraCapability + + +def _host_resource_property_query(session): + return ( + model_query(models.ComputeHostExtraCapability, session) + .join(models.ResourceProperty) + .add_column(models.ResourceProperty.property_name)) + + def _host_extra_capability_get(session, host_extra_capability_id): - query = model_query(models.ComputeHostExtraCapability, session) - return query.filter_by(id=host_extra_capability_id).first() + query = _host_resource_property_query(session).filter( + models.ComputeHostExtraCapability.id == host_extra_capability_id) + + return query.first() def host_extra_capability_get(host_extra_capability_id): @@ -766,8 +952,10 @@ def host_extra_capability_get(host_extra_capability_id): def _host_extra_capability_get_all_per_host(session, host_id): - query = model_query(models.ComputeHostExtraCapability, session) - return query.filter_by(computehost_id=host_id) + query = _host_resource_property_query(session).filter( + models.ComputeHostExtraCapability.computehost_id == host_id) + + return query def host_extra_capability_get_all_per_host(host_id): @@ -777,10 +965,18 @@ def host_extra_capability_get_all_per_host(host_id): def host_extra_capability_create(values): values = values.copy() + + resource_property = resource_property_get_or_create( + 'physical:host', values.get('property_name')) + + del values['property_name'] + values['property_id'] = resource_property.id + host_extra_capability = models.ComputeHostExtraCapability() host_extra_capability.update(values) session = get_session() + with session.begin(): try: host_extra_capability.save(session=session) @@ -797,7 +993,7 @@ def host_extra_capability_update(host_extra_capability_id, values): session = get_session() with session.begin(): - host_extra_capability = ( + host_extra_capability, _ = ( _host_extra_capability_get(session, host_extra_capability_id)) host_extra_capability.update(values) @@ -809,9 +1005,8 @@ def host_extra_capability_update(host_extra_capability_id, values): def host_extra_capability_destroy(host_extra_capability_id): session = get_session() with session.begin(): - host_extra_capability = ( - _host_extra_capability_get(session, - host_extra_capability_id)) + host_extra_capability = _host_extra_capability_get( + session, host_extra_capability_id) if not host_extra_capability: # raise not found error @@ -819,15 +1014,67 @@ def host_extra_capability_destroy(host_extra_capability_id): id=host_extra_capability_id, model='ComputeHostExtraCapability') - session.delete(host_extra_capability) + host_extra_capability[0].soft_delete(session=session) -def host_extra_capability_get_all_per_name(host_id, capability_name): +def host_extra_capability_get_all_per_name(host_id, property_name): session = get_session() with session.begin(): query = _host_extra_capability_get_all_per_host(session, host_id) - return query.filter_by(capability_name=capability_name).all() + return query.filter( + models.ResourceProperty.property_name == property_name).all() + + +# ComputeHostResourceInventory + +def host_resource_inventory_create(values): + values = values.copy() + + host_resource_inventory = models.ComputeHostResourceInventory() + host_resource_inventory.update(values) + + session = get_session() + + with session.begin(): + try: + host_resource_inventory.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=host_resource_inventory.__class__.__name__, + columns=e.columns) + + return None + + +def host_resource_inventory_get_all_per_host(host_id): + session = get_session() + with session.begin(): + query = session.query(models.ComputeHostResourceInventory) + return query.filter_by(computehost_id=host_id).all() + + +# ComputeHostTrait + +def host_trait_create(values): + values = values.copy() + + host_trait = models.ComputeHostTrait() + host_trait.update(values) + + session = get_session() + + with session.begin(): + try: + host_trait.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=host_trait.__class__.__name__, + columns=e.columns) + + return None # FloatingIP reservation @@ -879,6 +1126,7 @@ def fip_reservation_destroy(fip_reservation_id): raise db_exc.BlazarDBNotFound( id=fip_reservation_id, model='FloatingIPReservation') + fip_reservation.soft_delete(session=session) session.delete(fip_reservation) @@ -931,6 +1179,7 @@ def required_fip_destroy(required_fip_id): raise db_exc.BlazarDBNotFound( id=required_fip_id, model='RequiredFloatingIP') + required_fip.soft_delete(session=session) session.delete(required_fip) @@ -992,6 +1241,7 @@ def fip_allocation_destroy(allocation_id): raise db_exc.BlazarDBNotFound( id=allocation_id, model='FloatingIPAllocation') + fip_allocation.soft_delete(session=session) session.delete(fip_allocation) @@ -1080,6 +1330,12 @@ def reservable_fip_get_all_by_queries(queries): return fip_get_all_by_queries(queries) +def unreservable_fip_get_all_by_queries(queries): + """Returns unreseravable fips filtered by an array of queries""" + queries.append("reservable == 0") + return fip_get_all_by_queries(queries) + + def floatingip_get(floatingip_id): return _floatingip_get(get_session(), floatingip_id) @@ -1114,4 +1370,921 @@ def floatingip_destroy(floatingip_id): # raise not found error raise db_exc.BlazarDBNotFound(id=floatingip_id, model='FloatingIP') - session.delete(floatingip) + floatingip.soft_delete(session=session) + + +def floatingip_update(fip_id, values): + session = get_session() + + with session.begin(): + fip = _floatingip_get(session, fip_id) + fip.update(values) + fip.save(session=session) + + return floatingip_get(fip_id) + +# Networks + +def _network_get(session, network_id): + query = model_query(models.NetworkSegment, session) + return query.filter_by(id=network_id).first() + + +def _network_get_all(session): + query = model_query(models.NetworkSegment, session) + return query + + +def network_get(network_id): + return _network_get(get_session(), network_id) + + +def network_list(): + return model_query(models.NetworkSegment, get_session()).all() + + +def network_create(values): + values = values.copy() + network = models.NetworkSegment() + network.update(values) + + session = get_session() + with session.begin(): + try: + network.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=network.__class__.__name__, columns=e.columns) + + return network_get(network.id) + + +def network_update(network_id, values): + session = get_session() + + with session.begin(): + network = _network_get(session, network_id) + network.update(values) + network.save(session=session) + + return network_get(network_id) + + +def network_destroy(network_id): + session = get_session() + with session.begin(): + network = _network_get(session, network_id) + + if not network: + # raise not found error + raise db_exc.BlazarDBNotFound( + id=network_id, model='Network segment') + + network.soft_delete(session=session) + + # Also delete this network's extra capabilities + for capability, platform_version in network_extra_capability_get_all_per_network(network_id): + capability.soft_delete(session=session) + + +# NetworkAllocation + +def _network_allocation_get(session, network_allocation_id): + query = model_query(models.NetworkAllocation, session) + return query.filter_by(id=network_allocation_id).first() + + +def network_allocation_get(network_allocation_id): + return _network_allocation_get(get_session(), + network_allocation_id) + + +def network_allocation_get_all(): + query = model_query(models.NetworkAllocation, get_session()) + return query.all() + + +def network_allocation_create(values): + values = values.copy() + network_allocation = models.NetworkAllocation() + network_allocation.update(values) + + session = get_session() + with session.begin(): + try: + network_allocation.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=network_allocation.__class__.__name__, columns=e.columns) + + return network_allocation_get(network_allocation.id) + + +def network_allocation_get_all_by_values(**kwargs): + """Returns all entries filtered by col=value.""" + allocation_query = model_query(models.NetworkAllocation, get_session()) + for name, value in kwargs.items(): + column = getattr(models.NetworkAllocation, name, None) + if column: + allocation_query = allocation_query.filter(column == value) + return allocation_query.all() + + +def network_allocation_destroy(network_allocation_id): + session = get_session() + with session.begin(): + network_allocation = _network_allocation_get(session, + network_allocation_id) + + if not network_allocation: + # raise not found error + raise db_exc.BlazarDBNotFound( + id=network_allocation_id, model='NetworkAllocation') + + network_allocation.soft_delete(session=session) + + +# NetworkReservation + +def network_reservation_create(values): + value = values.copy() + network_reservation = models.NetworkReservation() + network_reservation.update(value) + + session = get_session() + with session.begin(): + try: + network_reservation.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=network_reservation.__class__.__name__, + columns=e.columns) + + return network_reservation_get(network_reservation.id) + + +def network_reservation_get(network_reservation_id, session=None): + if not session: + session = get_session() + query = model_query(models.NetworkReservation, session) + return query.filter_by(id=network_reservation_id).first() + + +def network_reservation_update(network_reservation_id, values): + session = get_session() + + with session.begin(): + network_reservation = network_reservation_get( + network_reservation_id, session) + + if not network_reservation: + raise db_exc.BlazarDBNotFound( + id=network_reservation_id, model='NetworkReservation') + + network_reservation.update(values) + network_reservation.save(session=session) + + return network_reservation_get(network_reservation_id) + + +def network_reservation_destroy(network_reservation_id): + session = get_session() + with session.begin(): + network = network_reservation_get(network_reservation_id) + + if not network: + raise db_exc.BlazarDBNotFound( + id=network_reservation_id, model='NetworkReservation') + + network.soft_delete(session=session) + + +def network_get_all_by_filters(filters): + """Returns networks filtered by name of the field.""" + + networks_query = _network_get_all(get_session()) + + if 'status' in filters: + networks_query = networks_query.filter( + models.NetworkSegment.status == filters['status']) + + return networks_query.all() + + +def network_get_all_by_queries(queries): + """Return networks filtered by an array of queries. + + :param queries: array of queries "key op value" where op can be + http://docs.sqlalchemy.org/en/rel_0_7/core/expression_api.html + #sqlalchemy.sql.operators.ColumnOperators + """ + networks_query = model_query(models.NetworkSegment, get_session()) + + oper = { + '<': ['lt', lambda a, b: a >= b], + '>': ['gt', lambda a, b: a <= b], + '<=': ['le', lambda a, b: a > b], + '>=': ['ge', lambda a, b: a < b], + '==': ['eq', lambda a, b: a != b], + '!=': ['ne', lambda a, b: a == b], + } + + networks = [] + for query in queries: + try: + key, op, value = query.split(' ', 2) + except ValueError: + raise db_exc.BlazarDBInvalidFilter(query_filter=query) + + column = getattr(models.NetworkSegment, key, None) + if column is not None: + if op == 'in': + filt = column.in_(value.split(',')) + else: + if op in oper: + op = oper[op][0] + try: + attr = [e for e in ['%s', '%s_', '__%s__'] + if hasattr(column, e % op)][0] % op + except IndexError: + raise db_exc.BlazarDBInvalidFilterOperator( + filter_operator=op) + + if value == 'null': + value = None + + filt = getattr(column, attr)(value) + + networks_query = networks_query.filter(filt) + else: + # looking for extra capabilities matches + extra_filter = ( + _network_extra_capability_query(get_session()) + .filter(models.ResourceProperty.property_name == key) + ).all() + if not extra_filter: + raise db_exc.BlazarDBNotFound( + id=key, model='NetworkSegmentExtraCapability') + for network, capability_name in extra_filter: + if op in oper and oper[op][1](network.capability_value, value): + networks.append(network.network_id) + elif op not in oper: + msg = 'Operator %s for extra capabilities not implemented' + raise NotImplementedError(msg % op) + + # We must also avoid selecting any network which doesn't have the + # extra capability present. + all_networks = [h.id for h in networks_query.all()] + extra_filter_networks = [h.network_id for h, _ in extra_filter] + networks += [h for h in all_networks if h not in + extra_filter_networks] + + return networks_query.filter(~models.NetworkSegment.id.in_(networks)).all() + + +def reservable_network_get_all_by_queries(queries): + """Return reservable networks filtered by an array of queries. + + :param queries: array of queries "key op value" where op can be + http://docs.sqlalchemy.org/en/rel_0_7/core/expression_api.html + #sqlalchemy.sql.operators.ColumnOperators + """ + queries.append('reservable == 1') + return network_get_all_by_queries(queries) + + +def unreservable_network_get_all_by_queries(queries): + """Return unreservable networks filtered by an array of queries. + + :param queries: array of queries "key op value" where op can be + http://docs.sqlalchemy.org/en/rel_0_7/core/expression_api.html + #sqlalchemy.sql.operators.ColumnOperators + """ + # TODO(hiro-kobayashi): support the expression 'reservable == False' + queries.append('reservable == 0') + return network_get_all_by_queries(queries) + + +# NetworkSegmentExtraCapability + +def _network_extra_capability_query(session): + return ( + model_query(models.NetworkSegmentExtraCapability, session) + .join(models.ResourceProperty) + .add_column(models.ResourceProperty.property_name)) + + +def _network_extra_capability_get(session, network_extra_capability_id): + query = _network_extra_capability_query(session).filter( + models.NetworkSegmentExtraCapability.id == network_extra_capability_id) + + return query.first() + + +def network_extra_capability_get(network_extra_capability_id): + return _network_extra_capability_get(get_session(), + network_extra_capability_id) + + +def _network_extra_capability_get_all_per_network(session, network_id): + query = _network_extra_capability_query(session).filter( + models.NetworkSegmentExtraCapability.network_id == network_id) + + return query + + +def network_extra_capability_get_all_per_network(network_id): + return _network_extra_capability_get_all_per_network(get_session(), + network_id).all() + + +def network_extra_capability_create(values): + values = values.copy() + + resource_property = _resource_property_get_or_create( + get_session(), 'network', values.get('capability_name')) + + del values['capability_name'] + values['property_id'] = resource_property.id + + network_extra_capability = models.NetworkSegmentExtraCapability() + network_extra_capability.update(values) + + session = get_session() + + with session.begin(): + try: + network_extra_capability.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=network_extra_capability.__class__.__name__, + columns=e.columns) + + return network_extra_capability_get(network_extra_capability.id) + + +def network_extra_capability_update(network_extra_capability_id, values): + session = get_session() + + with session.begin(): + network_extra_capability, _ = ( + _network_extra_capability_get(session, + network_extra_capability_id)) + network_extra_capability.update(values) + network_extra_capability.save(session=session) + + return network_extra_capability_get(network_extra_capability_id) + + +def network_extra_capability_destroy(network_extra_capability_id): + session = get_session() + with session.begin(): + network_extra_capability = _network_extra_capability_get( + session, network_extra_capability_id) + + if not network_extra_capability: + # raise not found error + raise db_exc.BlazarDBNotFound( + id=network_extra_capability_id, + model='NetworkSegmentExtraCapability') + + network_extra_capability[0].soft_delete(session=session) + + +def network_extra_capability_get_all_per_name(network_id, capability_name): + session = get_session() + + with session.begin(): + query = _network_extra_capability_get_all_per_network( + session, network_id) + return query.filter_by(capability_name=capability_name).all() + + +def network_extra_capability_get_latest_per_name(network_id, capability_name): + session = get_session() + + with session.begin(): + query = _network_extra_capability_get_all_per_network(session, + network_id) + return ( + query + .filter(models.ResourceProperty.capability_name == capability_name) + .order_by(models.NetworkSegmentExtraCapability.created_at.desc()) + .first()) + +# Devices + + +def _device_get(session, device_id): + query = model_query(models.Device, session) + return query.filter_by(id=device_id).first() + + +def _device_get_all(session): + query = model_query(models.Device, session) + return query + + +def device_get(device_id): + return _device_get(get_session(), device_id) + + +def device_list(): + return model_query(models.Device, get_session()).all() + + +def device_create(values): + values = values.copy() + device = models.Device() + device.update(values) + + session = get_session() + with session.begin(): + try: + device.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=device.__class__.__name__, columns=e.columns) + + return device_get(device.id) + + +def device_update(device_id, values): + session = get_session() + + with session.begin(): + device = _device_get(session, device_id) + device.update(values) + device.save(session=session) + + return device_get(device_id) + + +def device_destroy(device_id): + session = get_session() + with session.begin(): + device = _device_get(session, device_id) + + if not device: + # raise not found error + raise db_exc.BlazarDBNotFound( + id=device_id, model='Device') + + device.soft_delete(session=session) + + # Also delete this device's extra capabilities + for capability,platform_version in device_extra_capability_get_all_per_device(device_id): + capability.soft_delete(session=session) + + +# DeviceAllocation + +def _device_allocation_get(session, device_allocation_id): + query = model_query(models.DeviceAllocation, session) + return query.filter_by(id=device_allocation_id).first() + + +def device_allocation_get(device_allocation_id): + return _device_allocation_get(get_session(), + device_allocation_id) + + +def device_allocation_get_all(): + query = model_query(models.DeviceAllocation, get_session()) + return query.all() + + +def device_allocation_create(values): + values = values.copy() + device_allocation = models.DeviceAllocation() + device_allocation.update(values) + + session = get_session() + with session.begin(): + try: + device_allocation.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=device_allocation.__class__.__name__, columns=e.columns) + + return device_allocation_get(device_allocation.id) + + +def device_allocation_get_all_by_values(**kwargs): + """Returns all entries filtered by col=value.""" + allocation_query = model_query(models.DeviceAllocation, get_session()) + for name, value in kwargs.items(): + column = getattr(models.DeviceAllocation, name, None) + if column: + allocation_query = allocation_query.filter(column == value) + return allocation_query.all() + + +def device_allocation_update(device_allocation_id, values): + session = get_session() + + with session.begin(): + device_allocation = _device_allocation_get(session, + device_allocation_id) + device_allocation.update(values) + device_allocation.save(session=session) + + return device_allocation_get(device_allocation_id) + + +def device_allocation_destroy(device_allocation_id): + session = get_session() + with session.begin(): + device_allocation = _device_allocation_get(session, + device_allocation_id) + + if not device_allocation: + # raise not found error + raise db_exc.BlazarDBNotFound( + id=device_allocation_id, model='DeviceAllocation') + + device_allocation.soft_delete(session=session) + + +# DeviceReservation + +def device_reservation_create(values): + value = values.copy() + device_reservation = models.DeviceReservation() + device_reservation.update(value) + + session = get_session() + with session.begin(): + try: + device_reservation.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=device_reservation.__class__.__name__, + columns=e.columns) + + return device_reservation_get(device_reservation.id) + + +def device_reservation_get(device_reservation_id, session=None): + if not session: + session = get_session() + query = model_query(models.DeviceReservation, session) + return query.filter_by(id=device_reservation_id).first() + + +def device_reservation_update(device_reservation_id, values): + session = get_session() + + with session.begin(): + device_reservation = device_reservation_get( + device_reservation_id, session) + + if not device_reservation: + raise db_exc.BlazarDBNotFound( + id=device_reservation_id, model='DeviceReservation') + + device_reservation.update(values) + device_reservation.save(session=session) + + return device_reservation_get(device_reservation_id) + + +def device_reservation_destroy(device_reservation_id): + session = get_session() + with session.begin(): + device = device_reservation_get(device_reservation_id) + + if not device: + raise db_exc.BlazarDBNotFound( + id=device_reservation_id, model='DeviceReservation') + + device.soft_delete(session=session) + + +def device_get_all_by_filters(filters): + """Returns devices filtered by name of the field.""" + + devices_query = _device_get_all(get_session()) + + if 'status' in filters: + devices_query = devices_query.filter( + models.Device.status == filters['status']) + + return devices_query.all() + + +def device_get_all_by_queries(queries): + """Return devices filtered by an array of queries. + + :param queries: array of queries "key op value" where op can be + http://docs.sqlalchemy.org/en/rel_0_7/core/expression_api.html + #sqlalchemy.sql.operators.ColumnOperators + """ + devices_query = model_query(models.Device, get_session()) + + oper = { + '<': ['lt', lambda a, b: a >= b], + '>': ['gt', lambda a, b: a <= b], + '<=': ['le', lambda a, b: a > b], + '>=': ['ge', lambda a, b: a < b], + '==': ['eq', lambda a, b: a != b], + '!=': ['ne', lambda a, b: a == b], + } + + devices = [] + for query in queries: + try: + key, op, value = query.split(' ', 2) + except ValueError: + raise db_exc.BlazarDBInvalidFilter(query_filter=query) + + column = getattr(models.Device, key, None) + if column is not None: + if op == 'in': + filt = column.in_(value.split(',')) + else: + if op in oper: + op = oper[op][0] + try: + attr = [e for e in ['%s', '%s_', '__%s__'] + if hasattr(column, e % op)][0] % op + except IndexError: + raise db_exc.BlazarDBInvalidFilterOperator( + filter_operator=op) + + if value == 'null': + value = None + + filt = getattr(column, attr)(value) + + devices_query = devices_query.filter(filt) + else: + # looking for extra capabilities matches + extra_filter = ( + _device_extra_capability_query(get_session()) + .filter(models.ResourceProperty.property_name == key) + ).all() + + if not extra_filter: + raise db_exc.BlazarDBNotFound( + id=key, model='DeviceExtraCapability') + + for device, capability_name in extra_filter: + if op in oper and oper[op][1](device.capability_value, value): + devices.append(device.device_id) + elif op not in oper: + msg = 'Operator %s for extra capabilities not implemented' + raise NotImplementedError(msg % op) + + # We must also avoid selecting any device which doesn't have the + # extra capability present. + all_devices = [h.id for h in devices_query.all()] + extra_filter_devices = [h.device_id for h, _ in extra_filter] + devices += [h for h in all_devices if h not in + extra_filter_devices] + + return devices_query.filter(~models.Device.id.in_(devices)).all() + + +def reservable_device_get_all_by_queries(queries): + """Return reservable devices filtered by an array of queries. + + :param queries: array of queries "key op value" where op can be + http://docs.sqlalchemy.org/en/rel_0_7/core/expression_api.html + #sqlalchemy.sql.operators.ColumnOperators + """ + queries.append('reservable == 1') + return device_get_all_by_queries(queries) + + +def unreservable_device_get_all_by_queries(queries): + """Return unreservable devices filtered by an array of queries. + + :param queries: array of queries "key op value" where op can be + http://docs.sqlalchemy.org/en/rel_0_7/core/expression_api.html + #sqlalchemy.sql.operators.ColumnOperators + """ + # TODO(hiro-kobayashi): support the expression 'reservable == False' + queries.append('reservable == 0') + return device_get_all_by_queries(queries) + + +# DeviceExtraCapability + +def _device_extra_capability_query(session): + return ( + model_query(models.DeviceExtraCapability, session) + .join(models.ResourceProperty) + .add_column(models.ResourceProperty.property_name)) + + +def _device_extra_capability_get(session, device_extra_capability_id): + query = _device_extra_capability_query(session).filter( + models.DeviceExtraCapability.id == device_extra_capability_id) + + return query.first() + + +def device_extra_capability_get(device_extra_capability_id): + return _device_extra_capability_get(get_session(), + device_extra_capability_id) + + +def _device_extra_capability_get_all_per_device(session, device_id): + query = _device_extra_capability_query(session).filter( + models.DeviceExtraCapability.device_id == device_id) + + return query + + +def device_extra_capability_get_all_per_device(device_id): + return _device_extra_capability_get_all_per_device(get_session(), + device_id).all() + + +def device_extra_capability_create(values): + values = values.copy() + + resource_property = _resource_property_get_or_create( + get_session(), 'device', values.get('capability_name')) + + del values['capability_name'] + values['property_id'] = resource_property.id + + device_extra_capability = models.DeviceExtraCapability() + device_extra_capability.update(values) + + session = get_session() + + with session.begin(): + try: + device_extra_capability.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=device_extra_capability.__class__.__name__, + columns=e.columns) + + return device_extra_capability_get(device_extra_capability.id) + + +def device_extra_capability_update(device_extra_capability_id, values): + session = get_session() + + with session.begin(): + device_extra_capability, _ = ( + _device_extra_capability_get(session, + device_extra_capability_id)) + device_extra_capability.update(values) + device_extra_capability.save(session=session) + + return device_extra_capability_get(device_extra_capability_id) + + +def device_extra_capability_destroy(device_extra_capability_id): + session = get_session() + with session.begin(): + device_extra_capability = _device_extra_capability_get( + session, device_extra_capability_id) + + if not device_extra_capability: + # raise not found error + raise db_exc.BlazarDBNotFound( + id=device_extra_capability_id, + model='DeviceExtraCapability') + + device_extra_capability[0].soft_delete(session=session) + + +def device_extra_capability_get_all_per_name(device_id, capability_name): + session = get_session() + + with session.begin(): + query = _device_extra_capability_get_all_per_device( + session, device_id) + return query.filter_by(capability_name=capability_name).all() + + +def device_extra_capability_get_latest_per_name(device_id, capability_name): + session = get_session() + + with session.begin(): + query = _device_extra_capability_get_all_per_device(session, + device_id) + return ( + query + .filter(models.ResourceProperty.property_name == capability_name) + .order_by(models.DeviceExtraCapability.created_at.desc()) + .first()) + +# Resource Properties + + +def _resource_property_get(session, resource_type, property_name): + query = ( + model_query(models.ResourceProperty, session) + .filter_by(resource_type=resource_type) + .filter_by(property_name=property_name)) + + return query.first() + + +def resource_property_get(resource_type, property_name): + return _resource_property_get(get_session(), resource_type, property_name) + + +def resource_properties_list(resource_type): + if resource_type not in RESOURCE_PROPERTY_MODELS: + raise db_exc.BlazarDBResourcePropertiesNotEnabled( + resource_type=resource_type) + + session = get_session() + + with session.begin(): + + resource_model = RESOURCE_PROPERTY_MODELS[resource_type] + query = _read_deleted_filter(session.query( + models.ResourceProperty.property_name, + models.ResourceProperty.private, + resource_model.capability_value, + models.ResourceProperty.is_unique, + ).join(resource_model), resource_model, deleted=False).distinct() + + return query.all() + + +def _resource_property_create(session, values): + values = values.copy() + + resource_property = models.ResourceProperty() + resource_property.update(values) + + with session.begin(): + try: + resource_property.save(session=session) + except common_db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise db_exc.BlazarDBDuplicateEntry( + model=resource_property.__class__.__name__, + columns=e.columns) + + return resource_property_get(values.get('resource_type'), + values.get('property_name')) + + +def resource_property_create(values): + return _resource_property_create(get_session(), values) + + +def resource_property_update(resource_type, property_name, values): + if resource_type not in RESOURCE_PROPERTY_MODELS: + raise db_exc.BlazarDBResourcePropertiesNotEnabled( + resource_type=resource_type) + + values = values.copy() + session = get_session() + + with session.begin(): + resource_property = _resource_property_get( + session, resource_type, property_name) + + if not resource_property: + raise db_exc.BlazarDBInvalidResourceProperty( + property_name=property_name, + resource_type=resource_type) + + resource_property.update(values) + resource_property.save(session=session) + + return resource_property_get(resource_type, property_name) + + +def _resource_property_get_or_create(session, resource_type, property_name): + if property_name in FORBIDDEN_RESOURCE_PROPERTY_NAMES: + raise db_exc.BlazarDBForbiddenResourceProperty( + property_name=property_name) + + resource_property = _resource_property_get( + session, resource_type, property_name) + + if resource_property: + return resource_property + else: + rp_values = { + 'resource_type': resource_type, + 'property_name': property_name + } + + return resource_property_create(rp_values) + + +def resource_property_get_or_create(resource_type, property_name): + return _resource_property_get_or_create( + get_session(), resource_type, property_name) diff --git a/blazar/db/sqlalchemy/facade_wrapper.py b/blazar/db/sqlalchemy/facade_wrapper.py index 4887f6ed7..a35407b8a 100644 --- a/blazar/db/sqlalchemy/facade_wrapper.py +++ b/blazar/db/sqlalchemy/facade_wrapper.py @@ -38,6 +38,10 @@ def _clear_engine(): def _get_facade(): global _engine_facade if not _engine_facade: - _engine_facade = db_session.EngineFacade.from_config(CONF) + # FIXME(priteau): Remove autocommit=True (and ideally use of + # LegacyEngineFacade) asap since it's not compatible with SQLAlchemy + # 2.0. + _engine_facade = db_session.EngineFacade.from_config(CONF, + autocommit=True) return _engine_facade diff --git a/blazar/db/sqlalchemy/model_base.py b/blazar/db/sqlalchemy/model_base.py index 8990c9b40..93876864e 100644 --- a/blazar/db/sqlalchemy/model_base.py +++ b/blazar/db/sqlalchemy/model_base.py @@ -13,9 +13,30 @@ # See the License for the specific language governing permissions and # limitations under the License. +from oslo_config import cfg from oslo_db.sqlalchemy import models +from oslo_utils import timeutils +from sqlalchemy import Column +from sqlalchemy import DateTime from sqlalchemy.ext import declarative from sqlalchemy.orm import attributes +from sqlalchemy import String + + +class SoftDeleteMixinWithUuid(object): + """Mixin to provide soft delete capabilities. + + We cannot use oslo.db's SoftDeleteMixin as it assumes the `id` column is an + integer, while we use a UUID string. + """ + deleted_at = Column(DateTime) + deleted = Column(String(36), nullable=True) + + def soft_delete(self, session): + """Mark this object as deleted.""" + self.deleted = self.id + self.deleted_at = timeutils.utcnow() + self.save(session=session) class _BlazarBase(models.ModelBase, models.TimestampMixin): @@ -41,6 +62,13 @@ def to_dict(self, include=None): datetime_to_str(d, 'created_at') datetime_to_str(d, 'updated_at') + # Don't show fields created by SoftDeleteMixinWithUuid unless configured + if not cfg.CONF.include_deleted: + if 'deleted' in d: + del d['deleted'] + if 'deleted_at' in d: + del d['deleted_at'] + return d diff --git a/blazar/db/sqlalchemy/models.py b/blazar/db/sqlalchemy/models.py index 3eccc9211..ba7b7dcc9 100644 --- a/blazar/db/sqlalchemy/models.py +++ b/blazar/db/sqlalchemy/models.py @@ -15,15 +15,14 @@ from oslo_utils import uuidutils + +from blazar.db.sqlalchemy import model_base as mb import sqlalchemy as sa from sqlalchemy.dialects.mysql import MEDIUMTEXT -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, validates -from blazar.db.sqlalchemy import model_base as mb # Helpers - - def _generate_unicode_uuid(): return str(uuidutils.generate_uuid()) @@ -40,10 +39,17 @@ def _id_column(): # Main objects: Lease, Reservation, Event -class Lease(mb.BlazarBase): +class Lease(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """Contains all info about lease.""" __tablename__ = 'leases' + __table_args__ = ( + sa.Index('ix_leases_project_id', 'project_id'), + sa.Index('ix_leases_status', 'status'), + sa.Index('ix_leases_end_date', 'end_date'), + sa.Index('ix_leases_project_end_date', 'project_id', 'end_date'), + sa.Index('ix_leases_status_end_date', 'status', 'end_date'), + ) id = _id_column() name = sa.Column(sa.String(80), nullable=False) @@ -67,7 +73,7 @@ def to_dict(self): return d -class Reservation(mb.BlazarBase): +class Reservation(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """Specifies group of nodes within a cluster.""" __tablename__ = 'reservations' @@ -108,6 +114,26 @@ class Reservation(mb.BlazarBase): cascade="all,delete", backref='reservation', lazy='joined') + network_reservation = relationship('NetworkReservation', + uselist=False, + cascade="all,delete", + backref='reservation', + lazy='joined') + network_allocations = relationship('NetworkAllocation', + uselist=True, + cascade="all,delete", + backref='reservation', + lazy='joined') + device_reservation = relationship('DeviceReservation', + uselist=False, + cascade="all,delete", + backref='reservation', + lazy='joined') + device_allocations = relationship('DeviceAllocation', + uselist=True, + cascade="all,delete", + backref='reservation', + lazy='joined') def to_dict(self): d = super(Reservation, self).to_dict() @@ -118,6 +144,7 @@ def to_dict(self): d['hypervisor_properties'] = res['hypervisor_properties'] d['resource_properties'] = res['resource_properties'] d['before_end'] = res['before_end'] + d['on_start'] = res['on_start'] if res['count_range']: try: @@ -137,10 +164,23 @@ def to_dict(self): fip_keys = ['network_id', 'amount'] d.update(self.floatingip_reservation.to_dict(include=fip_keys)) + if self.device_reservation: + dr_keys = ['before_end', 'resource_properties'] + d.update(self.device_reservation.to_dict(include=dr_keys)) + + res = self.device_reservation.to_dict() + if res['count_range']: + try: + minMax = res['count_range'].split('-', 1) + (d['min'], d['max']) = map(int, minMax) + except ValueError: + e = "Invalid count range: {0}".format(res['count_range']) + raise RuntimeError(e) + return d -class Event(mb.BlazarBase): +class Event(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """An events occurring with the lease.""" __tablename__ = 'events' @@ -155,7 +195,26 @@ def to_dict(self): return super(Event, self).to_dict() -class ComputeHostReservation(mb.BlazarBase): +class ResourceProperty(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): + """Defines an resource property by resource type.""" + + __tablename__ = 'resource_properties' + + id = _id_column() + resource_type = sa.Column(sa.String(255), nullable=False) + property_name = sa.Column(sa.String(255), nullable=False) + private = sa.Column(sa.Boolean, nullable=False, + server_default=sa.false()) + is_unique = sa.Column(sa.Boolean, nullable=False, + server_default=sa.false()) + + __table_args__ = (sa.UniqueConstraint('resource_type', 'property_name'),) + + def to_dict(self): + return super(ResourceProperty, self).to_dict() + + +class ComputeHostReservation(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """Description Specifies resources asked by reservation from @@ -171,12 +230,13 @@ class ComputeHostReservation(mb.BlazarBase): count_range = sa.Column(sa.String(36)) hypervisor_properties = sa.Column(MediumText()) before_end = sa.Column(sa.String(36)) + on_start = sa.Column(sa.String(50)) def to_dict(self): return super(ComputeHostReservation, self).to_dict() -class InstanceReservations(mb.BlazarBase): +class InstanceReservations(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """The definition of a flavor of the reservation.""" __tablename__ = 'instance_reservations' @@ -187,21 +247,25 @@ class InstanceReservations(mb.BlazarBase): memory_mb = sa.Column(sa.Integer, nullable=False) disk_gb = sa.Column(sa.Integer, nullable=False) amount = sa.Column(sa.Integer, nullable=False) - affinity = sa.Column(sa.Boolean, nullable=False) + affinity = sa.Column(sa.Boolean, nullable=True) resource_properties = sa.Column(MediumText(), nullable=True) flavor_id = sa.Column(sa.String(36), nullable=True) aggregate_id = sa.Column(sa.Integer, nullable=True) server_group_id = sa.Column(sa.String(36), nullable=True) + before_end = sa.Column(sa.String(36)) -class ComputeHostAllocation(mb.BlazarBase): +class ComputeHostAllocation(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """Mapping between ComputeHost, ComputeHostReservation and Reservation.""" __tablename__ = 'computehost_allocations' + __table_args__ = ( + sa.Index('ix_computehost_allocations_compute_host_id', 'compute_host_id'), + sa.Index('ix_computehost_allocations_reservation_id', 'reservation_id'), + ) id = _id_column() - compute_host_id = sa.Column(sa.String(36), - sa.ForeignKey('computehosts.id')) + compute_host_id = sa.Column(sa.String(36)) reservation_id = sa.Column(sa.String(36), sa.ForeignKey('reservations.id')) @@ -209,7 +273,7 @@ def to_dict(self): return super(ComputeHostAllocation, self).to_dict() -class ComputeHost(mb.BlazarBase): +class ComputeHost(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """Description Specifies resources asked by reservation from @@ -232,16 +296,24 @@ class ComputeHost(mb.BlazarBase): trust_id = sa.Column(sa.String(36), nullable=False) reservable = sa.Column(sa.Boolean, nullable=False, server_default=sa.true()) + disabled = sa.Column(sa.Boolean, nullable=False, + server_default=sa.false()) computehost_extra_capabilities = relationship('ComputeHostExtraCapability', cascade="all,delete", backref='computehost', lazy='joined') + computehost_resource_inventories = relationship( + 'ComputeHostResourceInventory', cascade="all,delete", + backref='computehost', lazy='joined') + computehost_traits = relationship( + 'ComputeHostTrait', cascade="all,delete", + backref='computehost', lazy='joined') def to_dict(self): return super(ComputeHost, self).to_dict() -class ComputeHostExtraCapability(mb.BlazarBase): +class ComputeHostExtraCapability(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """Description Allows to define extra capabilities per administrator request for each @@ -249,18 +321,100 @@ class ComputeHostExtraCapability(mb.BlazarBase): """ __tablename__ = 'computehost_extra_capabilities' + __table_args__ = ( + sa.Index('ix_computehost_extra_capabilities_computehost_id', 'computehost_id'), + ) id = _id_column() computehost_id = sa.Column(sa.String(36), sa.ForeignKey('computehosts.id')) - capability_name = sa.Column(sa.String(64), nullable=False) + property_id = sa.Column(sa.String(36), + sa.ForeignKey('resource_properties.id'), + nullable=False) capability_value = sa.Column(MediumText(), nullable=False) def to_dict(self): return super(ComputeHostExtraCapability, self).to_dict() + @validates('property_id', 'capability_value') + def validate_property(self, key, value): + """Validates the proeprty ID and value before assigning + them to a ComputeHostExtraCapability instance. + + This validation method is called for two scenarios: + 1. When a capability ID is first assigned to a compute host. + both property_id and capability_value are validated in this case + 2. When a capability value is updated for an existing capability ID. + + Args: + key (str): The attribute key being validated + ('property_id' or 'capability_value'). + value (str) + + Raises: + ValueError + + Returns: + str + """ + from blazar.db.sqlalchemy import facade_wrapper + session = facade_wrapper.get_session() + + # Determine the property ID and value to validate based on the key + property_id = value if key == "property_id" else self.property_id + capability_value = value if key == "capability_value" else self.capability_value + + resource_property = session.query(ResourceProperty).filter_by(id=property_id).first() + if resource_property and resource_property.is_unique: + # exclude the current compute_host + # we should allow updating the node_name with current name + existing_capability = ( + session.query(ComputeHostExtraCapability).filter( + ComputeHostExtraCapability.computehost_id!=self.computehost_id, + ComputeHostExtraCapability.property_id==resource_property.id, + ComputeHostExtraCapability.capability_value==capability_value, + ComputeHostExtraCapability.deleted.is_(None) + ) + ).first() + if existing_capability: + raise ValueError( + f"{resource_property.capability_name} must be unique. " + f"Please select unique {resource_property.capability_name} for " + f"{self.computehost_id}" + ) + return value + + +class ComputeHostResourceInventory(mb.BlazarBase): + __tablename__ = 'computehost_resource_inventory' + + id = _id_column() + computehost_id = sa.Column(sa.String(36), sa.ForeignKey('computehosts.id')) + resource_class = sa.Column(sa.String(255), nullable=False) + total = sa.Column(sa.Integer, nullable=False) + reserved = sa.Column(sa.Integer, nullable=False) + min_unit = sa.Column(sa.Integer, nullable=False) + max_unit = sa.Column(sa.Integer, nullable=False) + step_size = sa.Column(sa.Integer, nullable=False) + allocation_ratio = sa.Column(sa.Float, nullable=False) + + def to_dict(self): + return super(ComputeHostResourceInventory, self).to_dict() + + +class ComputeHostTrait(mb.BlazarBase): + __tablename__ = 'computehost_trait' + + id = _id_column() + computehost_id = sa.Column(sa.String(36), sa.ForeignKey('computehosts.id')) + trait = sa.Column(sa.String(255), nullable=False) + + def to_dict(self): + return super(ComputeHostTrait, self).to_dict() + + # Floating IP -class FloatingIPReservation(mb.BlazarBase): +class FloatingIPReservation(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """Description Specifies resources asked by reservation from @@ -285,7 +439,7 @@ def to_dict(self, include=None): return d -class RequiredFloatingIP(mb.BlazarBase): +class RequiredFloatingIP(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """A table for a requested Floating IP. Keeps an user requested floating IP address in a floating IP reservation. @@ -298,7 +452,7 @@ class RequiredFloatingIP(mb.BlazarBase): sa.String(36), sa.ForeignKey('floatingip_reservations.id')) -class FloatingIPAllocation(mb.BlazarBase): +class FloatingIPAllocation(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """Mapping between FloatingIP, FloatingIPReservation and Reservation.""" __tablename__ = 'floatingip_allocations' @@ -310,7 +464,7 @@ class FloatingIPAllocation(mb.BlazarBase): sa.ForeignKey('reservations.id')) -class FloatingIP(mb.BlazarBase): +class FloatingIP(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): """A table for Floating IP resource.""" __tablename__ = 'floatingips' @@ -323,3 +477,167 @@ class FloatingIP(mb.BlazarBase): server_default=sa.true()) __table_args__ = (sa.UniqueConstraint('subnet_id', 'floating_ip_address'),) + + +class NetworkSegment(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): + """Description + + Specifies resources asked by reservation from Network Reservation API. + """ + + __tablename__ = 'network_segments' + + __table_args__ = ( + sa.UniqueConstraint('network_type', 'physical_network', 'segment_id'), + ) + + id = _id_column() + network_type = sa.Column(sa.String(255), nullable=False) + physical_network = sa.Column(sa.String(255), nullable=True) + segment_id = sa.Column(sa.Integer, nullable=False) + + def to_dict(self): + return super(NetworkSegment, self).to_dict() + + +class NetworkReservation(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): + """Description + + Specifies resources asked by reservation from + Network Reservation API. + """ + + __tablename__ = 'network_reservations' + + id = _id_column() + reservation_id = sa.Column(sa.String(36), sa.ForeignKey('reservations.id')) + resource_properties = sa.Column(MediumText()) + network_properties = sa.Column(MediumText()) + before_end = sa.Column(sa.String(36)) + network_name = sa.Column(sa.String(255)) + network_description = sa.Column(sa.String(255)) + network_id = sa.Column(sa.String(255)) + + def to_dict(self): + return super(NetworkReservation, self).to_dict() + + +class NetworkAllocation(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): + """Mapping between NetworkSegment, NetworkReservation and Reservation.""" + + __tablename__ = 'network_allocations' + + id = _id_column() + network_id = sa.Column(sa.String(36), + sa.ForeignKey('network_segments.id')) + reservation_id = sa.Column(sa.String(36), + sa.ForeignKey('reservations.id')) + + def to_dict(self): + return super(NetworkAllocation, self).to_dict() + + +class NetworkSegmentExtraCapability(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): + """Description + + Allows to define extra capabilities per administrator request for each + Network Segment added. + """ + + __tablename__ = 'networksegment_extra_capabilities' + + id = _id_column() + network_id = sa.Column(sa.String(36), sa.ForeignKey('network_segments.id'), + nullable=False) + property_id = sa.Column(sa.String(36), + sa.ForeignKey('resource_properties.id'), + nullable=False) + capability_value = sa.Column(MediumText(), nullable=False) + + def to_dict(self): + return super(NetworkSegmentExtraCapability, self).to_dict() + + +class Device(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): + """Description + + Specifies resources asked by reservation from Device Reservation API. + """ + + __tablename__ = 'devices' + + id = _id_column() + name = sa.Column(sa.String(255), unique=True, nullable=False) + device_type = sa.Column(sa.Enum('container', 'vm', 'shell', + name='allowed_device_types'), + nullable=False) + device_driver = sa.Column( + sa.Enum('zun', 'k8s', name='allowed_device_drivers'), nullable=False) + reservable = sa.Column(sa.Boolean, nullable=False, + server_default=sa.true()) + + @property + def _interfaces(self): + return [i for i in self.interfaces.split(';')] + + @_interfaces.setter + def _interfaces(self, interface): + self._interfaces += ";%s" % interface + + def to_dict(self): + return super(Device, self).to_dict() + + +class DeviceReservation(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): + """Description + + Specifies resources asked by reservation from + Device Reservation API. + """ + + __tablename__ = 'device_reservations' + + id = _id_column() + reservation_id = sa.Column(sa.String(36), sa.ForeignKey('reservations.id')) + count_range = sa.Column(sa.String(36)) + resource_properties = sa.Column(MediumText()) + before_end = sa.Column(sa.String(36)) + + def to_dict(self, include=None): + return super(DeviceReservation, self).to_dict(include=include) + + +class DeviceAllocation(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): + """Mapping between Device, DeviceReservation and Reservation.""" + + __tablename__ = 'device_allocations' + + id = _id_column() + device_id = sa.Column(sa.String(36), + sa.ForeignKey('devices.id')) + reservation_id = sa.Column(sa.String(36), + sa.ForeignKey('reservations.id')) + + def to_dict(self): + return super(DeviceAllocation, self).to_dict() + + +class DeviceExtraCapability(mb.BlazarBase, mb.SoftDeleteMixinWithUuid): + """Description + + Allows to define extra capabilities per administrator request for each + Device added. + """ + + __tablename__ = 'device_extra_capabilities' + + id = _id_column() + device_id = sa.Column(sa.String(36), sa.ForeignKey('devices.id'), + nullable=False) + property_id = sa.Column(sa.String(36), + sa.ForeignKey('resource_properties.id'), + nullable=False) + capability_value = sa.Column(MediumText(), nullable=False) + + def to_dict(self): + return super(DeviceExtraCapability, self).to_dict() diff --git a/blazar/db/sqlalchemy/utils.py b/blazar/db/sqlalchemy/utils.py index 9bf906b7d..fc58a21bf 100644 --- a/blazar/db/sqlalchemy/utils.py +++ b/blazar/db/sqlalchemy/utils.py @@ -14,17 +14,20 @@ # License for the specific language governing permissions and limitations # under the License. -from collections import defaultdict import sys - -import sqlalchemy as sa - +from datetime import datetime, timedelta from blazar.db.sqlalchemy import api from blazar.db.sqlalchemy import facade_wrapper from blazar.db.sqlalchemy import models from blazar.manager import exceptions as mgr_exceptions from blazar.plugins import instances as instance_plugin from blazar.plugins import oshosts as host_plugin +from blazar.plugins import devices as device_plugin +from blazar.plugins import networks as network_plugin +from blazar import status +from collections import defaultdict +import sqlalchemy as sa + get_session = facade_wrapper.get_session @@ -34,13 +37,18 @@ def get_backend(): return sys.modules[__name__] -def _get_leases_from_resource_id(resource_id, start_date, end_date): +def _get_leases_from_network_id(network_id, start_date, end_date): session = get_session() - border0 = start_date <= models.Lease.end_date - border1 = models.Lease.start_date <= end_date - query = (session.query(models.Lease).join(models.Reservation) - .filter(models.Reservation.resource_id == resource_id) - .filter(sa.and_(border0, border1))) + border0 = sa.and_(models.Lease.start_date < start_date, + models.Lease.end_date < start_date) + border1 = sa.and_(models.Lease.start_date > end_date, + models.Lease.end_date > end_date) + query = (api.model_query(models.Lease, session=session) + .join(models.Reservation) + .join(models.NetworkAllocation) + .filter(models.NetworkAllocation.deleted.is_(None)) + .filter(models.NetworkAllocation.network_id == network_id) + .filter(~sa.or_(border0, border1))) for lease in query: yield lease @@ -51,6 +59,7 @@ def _get_leases_from_host_id(host_id, start_date, end_date): border1 = models.Lease.start_date <= end_date query = (session.query(models.Lease).join(models.Reservation) .join(models.ComputeHostAllocation) + .filter(models.ComputeHostAllocation.deleted.is_(None)) .filter(models.ComputeHostAllocation.compute_host_id == host_id) .filter(sa.and_(border0, border1))) for lease in query: @@ -65,18 +74,33 @@ def _get_leases_from_fip_id(fip_id, start_date, end_date): models.Lease.end_date > end_date) query = (session.query(models.Lease).join(models.Reservation) .join(models.FloatingIPAllocation) + .filter(models.FloatingIPAllocation.deleted.is_(None)) .filter(models.FloatingIPAllocation.floatingip_id == fip_id) .filter(~sa.or_(border0, border1))) for lease in query: yield lease +def _get_leases_from_device_id(device_id, start_date, end_date): + session = get_session() + border0 = start_date <= models.Lease.end_date + border1 = models.Lease.start_date <= end_date + query = (session.query(models.Lease).join(models.Reservation) + .join(models.DeviceAllocation) + .filter(models.DeviceAllocation.deleted.is_(None)) + .filter(models.DeviceAllocation.device_id == device_id) + .filter(sa.and_(border0, border1))) + for lease in query: + yield lease + + def get_reservations_by_host_id(host_id, start_date, end_date): session = get_session() border0 = start_date <= models.Lease.end_date border1 = models.Lease.start_date <= end_date query = (session.query(models.Reservation).join(models.Lease) .join(models.ComputeHostAllocation) + .filter(models.ComputeHostAllocation.deleted.is_(None)) .filter(models.ComputeHostAllocation.compute_host_id == host_id) .filter(sa.and_(border0, border1))) return query.all() @@ -88,12 +112,74 @@ def get_reservations_by_host_ids(host_ids, start_date, end_date): border1 = models.Lease.start_date <= end_date query = (session.query(models.Reservation).join(models.Lease) .join(models.ComputeHostAllocation) + .filter(models.ComputeHostAllocation.deleted.is_(None)) .filter(models.ComputeHostAllocation.compute_host_id .in_(host_ids)) .filter(sa.and_(border0, border1))) return query.all() +def get_reservations_by_network_id(network_id, start_date, end_date): + session = get_session() + border0 = sa.and_(models.Lease.start_date < start_date, + models.Lease.end_date < start_date) + border1 = sa.and_(models.Lease.start_date > end_date, + models.Lease.end_date > end_date) + query = (api.model_query(models.Reservation, session=session) + .join(models.Lease) + .join(models.NetworkAllocation) + .filter(models.NetworkAllocation.deleted.is_(None)) + .filter(models.NetworkAllocation.network_id == network_id) + .filter(~sa.or_(border0, border1))) + return query.all() + + +def get_reservations_by_device_id(device_id, start_date, end_date): + session = get_session() + border0 = start_date <= models.Lease.end_date + border1 = models.Lease.start_date <= end_date + query = (session.query(models.Reservation).join(models.Lease) + .join(models.DeviceAllocation) + .filter(models.DeviceAllocation.deleted.is_(None)) + .filter(models.DeviceAllocation.device_id == device_id) + .filter(sa.and_(border0, border1))) + return query.all() + + +def get_reservations_by_device_ids(device_ids, start_date, end_date): + session = get_session() + border0 = start_date <= models.Lease.end_date + border1 = models.Lease.start_date <= end_date + query = (session.query(models.Reservation).join(models.Lease) + .join(models.DeviceAllocation) + .filter(models.DeviceAllocation.deleted.is_(None)) + .filter(models.DeviceAllocation.device_id + .in_(device_ids)) + .filter(sa.and_(border0, border1))) + return query.all() + + +def get_reservations_by_floatingip_ids(floatingip_ids, start_date, end_date): + session = get_session() + border0 = sa.and_(models.Lease.start_date < start_date, + models.Lease.end_date < start_date) + border1 = sa.and_(models.Lease.start_date > end_date, + models.Lease.end_date > end_date) + query = (api.model_query(models.Reservation, session=session) + .join( + models.Lease, + models.Lease.id == models.Reservation.lease_id + ) + .join( + models.FloatingIPAllocation, + models.FloatingIPAllocation.reservation_id == models.Reservation.lease_id + ) + .filter(models.FloatingIPAllocation.deleted.is_(None)) + .filter(models.FloatingIPAllocation.floatingip_id.in_(floatingip_ids)) + .filter(~sa.or_(border0, border1))) + return query.all() + + def get_reservations_for_allocations(session, start_date, end_date, lease_id=None, reservation_id=None): fields = ['id', 'status', 'lease_id', 'start_date', @@ -107,7 +193,8 @@ def get_reservations_for_allocations(session, start_date, end_date, models.Lease.end_date, models.Lease.name, models.Lease.project_id) - .join(models.Lease)) + .join(models.Lease) + .filter(models.Reservation.deleted.is_(None))) if lease_id: reservations_query = reservations_query.filter( @@ -137,6 +224,7 @@ def get_reservation_allocations_by_host_ids(host_ids, start_date, end_date, allocations_query = (session.query( models.ComputeHostAllocation.reservation_id, models.ComputeHostAllocation.compute_host_id) + .filter(models.ComputeHostAllocation.deleted.is_(None)) .filter(models.ComputeHostAllocation.compute_host_id.in_(host_ids)) .filter(models.ComputeHostAllocation.reservation_id.in_( list(set([x['id'] for x in reservations]))))) @@ -146,11 +234,9 @@ def get_reservation_allocations_by_host_ids(host_ids, start_date, end_date, for row in allocations_query.all(): allocations[row[0]].append(row[1]) - allocs = [] for r in reservations: - allocs.append((r['id'], r['lease_id'], allocations[r['id']][0])) - - return allocs + r['host_ids'] = allocations[r['id']] + return reservations def get_reservation_allocations_by_fip_ids(fip_ids, start_date, end_date, @@ -162,6 +248,7 @@ def get_reservation_allocations_by_fip_ids(fip_ids, start_date, end_date, allocations_query = (session.query( models.FloatingIPAllocation.reservation_id, models.FloatingIPAllocation.floatingip_id) + .filter(models.FloatingIPAllocation.deleted.is_(None)) .filter(models.FloatingIPAllocation.floatingip_id.in_(fip_ids)) .filter(models.FloatingIPAllocation.reservation_id.in_( list(set([x['id'] for x in reservations]))))) @@ -177,11 +264,195 @@ def get_reservation_allocations_by_fip_ids(fip_ids, start_date, end_date, return reservations +def get_reservation_allocations_by_network_ids(network_ids, start_date, + end_date, lease_id=None, + reservation_id=None): + session = get_session() + reservations = get_reservations_for_allocations( + session, start_date, end_date, lease_id, reservation_id) + + allocations_query = (session.query( + models.NetworkAllocation.reservation_id, + models.NetworkAllocation.network_id) + .filter(models.NetworkAllocation.deleted.is_(None)) + .filter(models.NetworkAllocation.network_id.in_(network_ids)) + .filter(models.NetworkAllocation.reservation_id.in_( + list(set([x['id'] for x in reservations]))))) + + allocations = defaultdict(list) + + for row in allocations_query.all(): + allocations[row[0]].append(row[1]) + + for r in reservations: + r['network_ids'] = allocations[r['id']] + + return reservations + + +def get_most_recent_reservation_info_by_network_id(network_id): + """returns the recent network reservation which not in + pending status and start_date of the reservation is less than current date + + Args: + network_id (): network id - primary key of NetworkSegment table + """ + session = get_session() + curr_date = datetime.utcnow() + timedelta(seconds=300) + query = ( + session.query( + models.NetworkSegment.id.label('network_id'), + models.NetworkSegment.segment_id, + models.Lease.start_date, + models.Lease.end_date, + models.Reservation.status.label('status'), + models.Reservation.id, + ) + .filter(models.NetworkSegment.id == network_id) + .join( + models.NetworkAllocation, + models.NetworkAllocation.network_id == models.NetworkSegment.id + ) + .join( + models.Reservation, + models.Reservation.id == models.NetworkAllocation.reservation_id + ) + .join( + models.Lease, + models.Lease.id == models.Reservation.lease_id + ) + .filter(models.Lease.start_date < curr_date) + .filter(models.Reservation.status != status.reservation.PENDING) + .order_by(models.Lease.start_date.desc()) + ) + return query.first() + + +def get_reservation_allocations_by_device_ids(device_ids, start_date, end_date, + lease_id=None, + reservation_id=None): + session = get_session() + reservations = get_reservations_for_allocations( + session, start_date, end_date, lease_id, reservation_id) + + allocations_query = (session.query( + models.DeviceAllocation.reservation_id, + models.DeviceAllocation.device_id) + .filter(models.DeviceAllocation.deleted.is_(None)) + .filter(models.DeviceAllocation.device_id.in_(device_ids)) + .filter(models.DeviceAllocation.reservation_id.in_( + list(set([x['id'] for x in reservations]))))) + + allocations = defaultdict(list) + + for row in allocations_query.all(): + allocations[row[0]].append(row[1]) + + for r in reservations: + r['device_ids'] = allocations[r['id']] + + return reservations + + +def get_most_recent_reservation_info_by_host_id(host_id): + """returns the recent host reservation which not in + pending status and start_date of the reservation is less than current date + + Args: + host_id (): Host id - primary key of ComputeHost table + """ + session = get_session() + curr_date = datetime.utcnow() + timedelta(seconds=300) + query = ( + session.query( + models.ComputeHost.id.label('host_id'), + models.ComputeHost.hypervisor_hostname, + models.Reservation.status.label('reservation_status'), + models.Lease.start_date, + models.Lease.end_date, + models.ComputeHostReservation.aggregate_id, + models.ComputeHostReservation.reservation_id, + ) + .filter(models.ComputeHost.id == host_id) + .join( + models.ComputeHostAllocation, + models.ComputeHostAllocation.compute_host_id == models.ComputeHost.id + ) + .join( + models.ComputeHostReservation, + models.ComputeHostReservation.reservation_id == models.ComputeHostAllocation.reservation_id + ) + .join( + models.Reservation, + models.Reservation.id == models.ComputeHostReservation.reservation_id + ) + .join( + models.Lease, + models.Lease.id == models.Reservation.lease_id + ) + .filter(models.Lease.start_date < curr_date) + .filter(models.Reservation.status != status.reservation.PENDING) + .order_by(models.Lease.start_date.desc()) + ) + return query.first() + + +def get_most_recent_reservation_info_by_fip_id(fip_id): + """returns the recent FIP reservation which not in + pending status and start_date of the reservation is less than current date + + Args: + host_id (): Host id - primary key of ComputeHost table + """ + session = get_session() + curr_date = datetime.utcnow() + timedelta(seconds=300) + query = ( + session.query( + models.FloatingIP.id.label('fip_id'), + models.FloatingIP.floating_ip_address, + models.Lease.start_date, + models.Lease.end_date, + models.Reservation.status.label('status'), + models.Reservation.id, + ) + .filter(models.FloatingIP.id == fip_id) + .join( + models.FloatingIPAllocation, + models.FloatingIPAllocation.floatingip_id == models.FloatingIP.id + ) + .join( + models.Reservation, + models.Reservation.id == models.FloatingIPAllocation.reservation_id + ) + .join( + models.Lease, + models.Lease.id == models.Reservation.lease_id + ) + .filter(models.Lease.start_date < curr_date) + .filter(models.Reservation.status != status.reservation.PENDING) + .order_by(models.Lease.start_date.desc()) + ) + return query.first() + + +def get_user_ids_for_lease_ids(lease_ids): + session = get_session() + + leases_query = (session.query(models.Lease.id, models.Lease.user_id) + .filter(models.Lease.id.in_(lease_ids))) + + return leases_query.all() + + def get_plugin_reservation(resource_type, resource_id): if resource_type == host_plugin.RESOURCE_TYPE: return api.host_reservation_get(resource_id) elif resource_type == instance_plugin.RESOURCE_TYPE: return api.instance_reservation_get(resource_id) + elif resource_type == network_plugin.RESOURCE_TYPE: + return api.network_reservation_get(resource_id) + elif resource_type == device_plugin.RESOURCE_TYPE: + return api.device_reservation_get(resource_id) else: raise mgr_exceptions.UnsupportedResourceType( resource_type=resource_type) @@ -218,6 +489,10 @@ def _get_events(resource_id, start_date, end_date, resource_type): leases = _get_leases_from_host_id(resource_id, start_date, end_date) elif resource_type == 'floatingip': leases = _get_leases_from_fip_id(resource_id, start_date, end_date) + elif resource_type == 'network': + leases = _get_leases_from_network_id(resource_id, start_date, end_date) + elif resource_type == 'device': + leases = _get_leases_from_device_id(resource_id, start_date, end_date) else: mgr_exceptions.UnsupportedResourceType(resource_type) diff --git a/blazar/db/utils.py b/blazar/db/utils.py index 19f75b557..531ab6844 100644 --- a/blazar/db/utils.py +++ b/blazar/db/utils.py @@ -111,13 +111,37 @@ def get_reservations_by_host_ids(host_ids, start_date, end_date): return IMPL.get_reservations_by_host_ids(host_ids, start_date, end_date) +def get_reservations_by_network_id(network_id, start_date, end_date): + return IMPL.get_reservations_by_network_id( + network_id, start_date, end_date) + + +def get_reservations_by_device_id(device_id, start_date, end_date): + return IMPL.get_reservations_by_device_id(device_id, start_date, end_date) + + +def get_reservations_by_device_ids(device_ids, start_date, end_date): + return IMPL.get_reservations_by_device_ids(device_ids, start_date, + end_date) + + +def get_reservations_by_floatingip_ids(floatingip_id, start_date, end_date): + return IMPL.get_reservations_by_floatingip_ids( + floatingip_id, start_date, end_date) + + def get_reservation_allocations_by_host_ids(host_ids, start_date, end_date, lease_id=None, reservation_id=None): - return IMPL.get_reservation_allocations_by_host_ids(host_ids, - start_date, end_date, - lease_id, - reservation_id) + return IMPL.get_reservation_allocations_by_host_ids( + host_ids, start_date, end_date, lease_id, reservation_id) + + +def get_reservation_allocations_by_network_ids(network_ids, start_date, + end_date, lease_id=None, + reservation_id=None): + return IMPL.get_reservation_allocations_by_network_ids( + network_ids, start_date, end_date, lease_id, reservation_id) def get_reservation_allocations_by_fip_ids(fip_ids, start_date, end_date, @@ -126,6 +150,25 @@ def get_reservation_allocations_by_fip_ids(fip_ids, start_date, end_date, fip_ids, start_date, end_date, lease_id, reservation_id) +def get_reservation_allocations_by_device_ids(device_ids, start_date, end_date, + lease_id=None, + reservation_id=None): + return IMPL.get_reservation_allocations_by_device_ids( + device_ids, start_date, end_date, lease_id, reservation_id) + + +def get_most_recent_reservation_info_by_host_id(host_id): + return IMPL.get_most_recent_reservation_info_by_host_id(host_id) + + +def get_most_recent_reservation_info_by_fip_id(host_id): + return IMPL.get_most_recent_reservation_info_by_fip_id(host_id) + + +def get_most_recent_reservation_info_by_network_id(host_id): + return IMPL.get_most_recent_reservation_info_by_network_id(host_id) + + def get_plugin_reservation(resource_type, resource_id): return IMPL.get_plugin_reservation(resource_type, resource_id) @@ -142,3 +185,7 @@ def get_reserved_periods(resource_id, start_date, end_date, duration, """Returns a list of reserved periods.""" return IMPL.get_reserved_periods(resource_id, start_date, end_date, duration, resource_type=resource_type) + + +def get_user_ids_for_lease_ids(lease_ids): + return IMPL.get_user_ids_for_lease_ids(lease_ids) diff --git a/blazar/enforcement/enforcement.py b/blazar/enforcement/enforcement.py index 3e0c87d91..c77a9d678 100644 --- a/blazar/enforcement/enforcement.py +++ b/blazar/enforcement/enforcement.py @@ -25,6 +25,10 @@ cfg.ListOpt('enabled_filters', default=[], help='List of enabled usage enforcement filters.'), + cfg.ListOpt('exempt_projects', + default=[], + help=('Allow list of project IDs exempt from enforcement ' + 'constraints.')), ] CONF.register_opts(enforcement_opts, group='enforcement') @@ -37,22 +41,21 @@ def __init__(self): self.load_filters() def load_filters(self): - self.enabled_filters = set() + self.enabled_filters = [] for filter_name in CONF.enforcement.enabled_filters: _filter = getattr(filters, filter_name) if filter_name in filters.all_filters: - self.enabled_filters.add(_filter(conf=CONF)) + self.enabled_filters.append(_filter(conf=CONF)) else: LOG.error("{} not in filters module.".format(filter_name)) - self.enabled_filters = list(self.enabled_filters) - def format_context(self, context, lease_values): ctx = context.to_dict() region_name = CONF.os_region_name auth_url = base.url_for( ctx['service_catalog'], CONF.identity_service, + endpoint_interface="public", os_region_name=region_name) return dict(user_id=lease_values['user_id'], @@ -73,27 +76,36 @@ def format_lease(self, lease_values, reservations, allocations): def check_create(self, context, lease_values, reservations, allocations): context = self.format_context(context, lease_values) + if context['project_id'] in CONF.enforcement.exempt_projects: + return + lease = self.format_lease(lease_values, reservations, allocations) - for _filter in self.enabled_filters: - _filter.check_create(context, lease) + for filter_ in self.enabled_filters: + filter_.check_create(context, lease) def check_update(self, context, current_lease, new_lease, current_allocations, new_allocations, current_reservations, new_reservations): context = self.format_context(context, current_lease) + if context['project_id'] in CONF.enforcement.exempt_projects: + return + current_lease = self.format_lease(current_lease, current_reservations, current_allocations) new_lease = self.format_lease(new_lease, new_reservations, new_allocations) - for _filter in self.enabled_filters: - _filter.check_update(context, current_lease, new_lease) + for filter_ in self.enabled_filters: + filter_.check_update(context, current_lease, new_lease) def on_end(self, context, lease, allocations): context = self.format_context(context, lease) + if context['project_id'] in CONF.enforcement.exempt_projects: + return + lease_values = self.format_lease(lease, lease['reservations'], allocations) - for _filter in self.enabled_filters: - _filter.on_end(context, lease_values) + for filter_ in self.enabled_filters: + filter_.on_end(context, lease_values) diff --git a/blazar/enforcement/exceptions.py b/blazar/enforcement/exceptions.py index b30f355f0..ea1c988de 100644 --- a/blazar/enforcement/exceptions.py +++ b/blazar/enforcement/exceptions.py @@ -21,3 +21,20 @@ class MaxLeaseDurationException(exceptions.NotAuthorized): msg_fmt = _('Lease duration of %(lease_duration)s seconds must be less ' 'than or equal to the maximum lease duration of ' '%(max_duration)s seconds.') + + +class MaxLeaseUpdateWindowException(exceptions.NotAuthorized): + code = 400 + msg_fmt = _('Lease can only be extended within %(extension_window)s ' + 'seconds of the leases current end time.') + + +class ExternalServiceFilterException(exceptions.NotAuthorized): + code = 400 + msg_fmt = _('%(message)s') + + +class ExternalServiceUnsupportedHTTPResponse(exceptions.BlazarException): + code = 400 + msg_fmt = _('External Service Filter returned a %(status)s http response. ' + 'Only 204 and 403 responses are supported.') diff --git a/blazar/enforcement/filters/__init__.py b/blazar/enforcement/filters/__init__.py index b66132f1e..62e53b47f 100644 --- a/blazar/enforcement/filters/__init__.py +++ b/blazar/enforcement/filters/__init__.py @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from blazar.enforcement.filters.external_service_filter import ( + ExternalServiceFilter) from blazar.enforcement.filters.max_lease_duration_filter import ( MaxLeaseDurationFilter) -__all__ = ['MaxLeaseDurationFilter'] +__all__ = ['ExternalServiceFilter', 'MaxLeaseDurationFilter'] all_filters = __all__ diff --git a/blazar/enforcement/filters/base_filter.py b/blazar/enforcement/filters/base_filter.py index 4fd3d5f26..ed14e9942 100644 --- a/blazar/enforcement/filters/base_filter.py +++ b/blazar/enforcement/filters/base_filter.py @@ -22,9 +22,12 @@ class BaseFilter(metaclass=abc.ABCMeta): def __init__(self, conf=None): self.conf = conf + self.register_opts(conf) - for opt in self.enforcement_opts: - self.conf.register_opt(opt, 'enforcement') + @classmethod + def register_opts(cls, conf): + for opt in cls.enforcement_opts: + conf.register_opt(opt, 'enforcement') def __getattr__(self, name): func = getattr(self.conf.enforcement, name) diff --git a/blazar/enforcement/filters/external_service_filter.py b/blazar/enforcement/filters/external_service_filter.py new file mode 100644 index 000000000..ff66a4bcf --- /dev/null +++ b/blazar/enforcement/filters/external_service_filter.py @@ -0,0 +1,176 @@ +# Copyright (c) 2022 University of Chicago. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +import json +import requests +from urllib.parse import urljoin +from urllib.parse import urlparse + +from blazar.enforcement.exceptions import ExternalServiceFilterException +from blazar.enforcement.filters import base_filter +from blazar.exceptions import BlazarException +from blazar.i18n import _ +from blazar.utils.openstack.keystone import BlazarKeystoneClient + +from oslo_config import cfg +from oslo_log import log as logging + +LOG = logging.getLogger(__name__) + + +class ISODateTimeEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, datetime): + return o.isoformat() + + return json.JSONEncoder.default(self, o) + + +GENERIC_DENY_MSG = 'External service enforcement filter denied the request.' + + +class ExternalServiceMisconfigured(BlazarException): + msg_fmt = _('%(message)s') + + +class ExternalServiceFilter(base_filter.BaseFilter): + + enforcement_opts = [ + cfg.StrOpt( + 'external_service_base_endpoint', + default=None, + help='The URL of the external service API.'), + cfg.StrOpt( + 'external_service_check_create_endpoint', + default=None, + help='Overrides check-create endpoint with another URL.'), + cfg.StrOpt( + 'external_service_check_update_endpoint', + default=None, + help='Overrides check-update endpoint with another URL.'), + cfg.StrOpt( + 'external_service_on_end_endpoint', + default=None, + help='Overrides on-end endpoint with another URL.'), + cfg.StrOpt( + 'external_service_token', + default="", + help='Token used for authentication with the external service.') + ] + + def __init__(self, conf=None): + super(ExternalServiceFilter, self).__init__(conf=conf) + + self._validate_url(conf.enforcement.external_service_base_endpoint) + self.base_endpoint = conf.enforcement.external_service_base_endpoint + + self.check_create_endpoint = self._construct_url( + "check-create", + conf.enforcement.external_service_check_create_endpoint) + self.check_update_endpoint = self._construct_url( + "check-update", + conf.enforcement.external_service_check_update_endpoint) + self.on_end_endpoint = self._construct_url( + "on-end", + conf.enforcement.external_service_on_end_endpoint) + + endpoints = ( + self.check_create_endpoint, + self.check_update_endpoint, + self.on_end_endpoint, + ) + + if all(x is None for x in endpoints): + raise ExternalServiceMisconfigured( + message=_("ExternalService has no endpoints set.")) + + self.token = conf.enforcement.external_service_token + + @staticmethod + def _validate_url(url): + if url is None: + return + parsed_url = urlparse(url) + if parsed_url.scheme not in ("http", "https"): + raise ExternalServiceMisconfigured( + message=_("ExternalService URL scheme must be http(s): " + "%s") % url) + if parsed_url.netloc == '': + raise ExternalServiceMisconfigured( + message=_("ExternalService URL must have netloc: " + "%s") % url) + + def _construct_url(self, method, replacement_url): + if replacement_url is None: + if self.base_endpoint is None: + return None + return urljoin(self.base_endpoint, method) + + self._validate_url(replacement_url) + return replacement_url + + def _get_headers(self): + headers = {'Content-Type': 'application/json'} + + if self.token: + headers['X-Auth-Token'] = self.token + else: + client = BlazarKeystoneClient() + headers['X-Auth-Token'] = client.session.get_token() + + + return headers + + def _post(self, url, body): + body = json.dumps(body, cls=ISODateTimeEncoder) + res = requests.post(url, headers=self._get_headers(), data=body) + + if res.status_code == 204: + return True + elif res.status_code == 403: + try: + message = res.json()['message'] + except (requests.JSONDecodeError, KeyError): + # NOTE(yoctozepto): It is more secure not to send the actual + # response to the end user as it may leak something. + # Instead, we log it for debugging. + LOG.debug("The External Service API returned a malformed " + "response (403): %s", res.content) + message = GENERIC_DENY_MSG + else: + # NOTE(yoctozepto): It is more secure not to send the actual + # response to the end user as it may leak something. + # Instead, we log it for debugging. + LOG.debug("The External Service API returned a malformed " + "response (%d): %s", res.status_code, res.content) + message = GENERIC_DENY_MSG + raise ExternalServiceFilterException(message=message) + + def check_create(self, context, lease_values): + if self.check_create_endpoint: + self._post(self.check_create_endpoint, dict( + context=context, lease=lease_values)) + + def check_update(self, context, current_lease_values, new_lease_values): + if self.check_update_endpoint: + self._post(self.check_update_endpoint, dict( + context=context, current_lease=current_lease_values, + lease=new_lease_values)) + + def on_end(self, context, lease_values): + if self.on_end_endpoint: + self._post(self.on_end_endpoint, dict( + context=context, lease=lease_values)) diff --git a/blazar/enforcement/filters/max_lease_duration_filter.py b/blazar/enforcement/filters/max_lease_duration_filter.py index 633079e9f..2dff7c2ef 100644 --- a/blazar/enforcement/filters/max_lease_duration_filter.py +++ b/blazar/enforcement/filters/max_lease_duration_filter.py @@ -13,14 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from blazar.enforcement import exceptions +from blazar.enforcement import exceptions as enforcement_ex from blazar.enforcement.filters import base_filter - +from datetime import datetime +from datetime import timedelta from oslo_config import cfg from oslo_log import log as logging DEFAULT_MAX_LEASE_DURATION = -1 +DEFAULT_RESERVATION_EXTENTSION_WINDOW = -1 LOG = logging.getLogger(__name__) @@ -33,11 +35,19 @@ class MaxLeaseDurationFilter(base_filter.BaseFilter): 'max_lease_duration', default=DEFAULT_MAX_LEASE_DURATION, help='Maximum lease duration in seconds. If this is set to -1, ' - 'there is not limit.'), + 'there is not limit. For active leases being updated, ' + 'the limit applies between now and the new end date.'), + cfg.IntOpt( + 'reservation_extension_window', + default=DEFAULT_RESERVATION_EXTENTSION_WINDOW, + help='Gives users a window towards the end of a reservation to ' + 'extend their lease again for the max lease length. A ' + 'value of -1 will not allow users to extend leases beyond ' + 'the maximum lease length'), cfg.ListOpt( 'max_lease_duration_exempt_project_ids', default=[], - help='Allow list of project ids exempt from filter constraints.'), + help='List of project IDs exempt from max length constraint.'), ] def __init__(self, conf=None): @@ -54,7 +64,7 @@ def check_for_duration_violation(self, start_date, end_date): lease_duration = (end_date - start_date).total_seconds() if lease_duration > self.conf.enforcement.max_lease_duration: - raise exceptions.MaxLeaseDurationException( + raise enforcement_ex.MaxLeaseDurationException( lease_duration=int(lease_duration), max_duration=self.conf.enforcement.max_lease_duration) @@ -71,9 +81,25 @@ def check_update(self, context, current_lease_values, new_lease_values): if self._exempt(context): return - start_date = new_lease_values['start_date'] + start_date = current_lease_values['start_date'] end_date = new_lease_values['end_date'] + # Check if lease is being extended + if (current_lease_values['end_date'] >= end_date and + start_date >= new_lease_values['start_date']): + return + + if self.reservation_extension_window: + min_window = current_lease_values['end_date'] - timedelta( + seconds=self.reservation_extension_window) + update_at = datetime.utcnow() + + if update_at < min_window: + raise enforcement_ex.MaxLeaseUpdateWindowException( + extension_window=(self.reservation_extension_window)) + + start_date = current_lease_values['end_date'] + self.check_for_duration_violation(start_date, end_date) def on_end(self, context, lease_values): diff --git a/blazar/exceptions.py b/blazar/exceptions.py index ee361ef8a..3a84629a1 100644 --- a/blazar/exceptions.py +++ b/blazar/exceptions.py @@ -14,9 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from oslo_log import log as logging - from blazar.i18n import _ +from oslo_log import log as logging LOG = logging.getLogger(__name__) diff --git a/blazar/manager/devices/__init__.py b/blazar/manager/devices/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/blazar/manager/exceptions.py b/blazar/manager/exceptions.py index 9883af637..34e3c65f9 100644 --- a/blazar/manager/exceptions.py +++ b/blazar/manager/exceptions.py @@ -17,6 +17,16 @@ from blazar.i18n import _ +class NotImplemented(exceptions.BlazarException): + code = 409 + msg_fmt = '%(error)s' + + +class ReservationTypeConflict(exceptions.NotFound): + code = 409 + msg_fmt = _("Conflicting reservation types found") + + class NoFreePool(exceptions.NotFound): msg_fmt = _("No Freepool found") @@ -69,8 +79,11 @@ class MultipleHostsFound(exceptions.BlazarException): msg_fmt = _("Multiple Hosts found for pattern '%(host)s'") -class HostHavingServers(exceptions.BlazarException): +class ResourceBusy(exceptions.BlazarException): code = 409 + + +class HostHavingServers(ResourceBusy): msg_fmt = _("Servers [%(servers)s] found for host %(host)s") @@ -115,6 +128,13 @@ class CantAddExtraCapability(exceptions.BlazarException): code = 409 msg_fmt = _("Can't add extracapabilities %(keys)s to Host %(host)s") +class ExtraCapabilityNotFound(exceptions.BlazarException): + code = 404 + msg_fmt = _("Capability %(keys)s not found on resource %(resource)s") + +class ResourceProviderNotFound(exceptions.NotFound): + code = 404 + msg_fmt = _("No resource provider found for host %(host)s") class EndpointsNotFound(exceptions.NotFound): code = 404 @@ -140,10 +160,18 @@ class HypervisorNotFound(exceptions.BlazarException): msg_fmt = _("Aggregate '%(pool)s' not found!") -class NotEnoughHostsAvailable(exceptions.BlazarException): +class NotEnoughResourcesAvailable(exceptions.BlazarException): + pass + + +class NotEnoughHostsAvailable(NotEnoughResourcesAvailable): msg_fmt = _("Not enough hosts available") +class NotEnoughResourcesDefaultProperties(NotEnoughResourcesAvailable): + msg_fmt = _("Not enough resources available with query %(params)s") + + class MalformedParameter(exceptions.BlazarException): code = 400 msg_fmt = _("Malformed parameter %(param)s") @@ -214,7 +242,7 @@ class TooLongFloatingIPs(exceptions.InvalidInput): "required_floatingips.") -class NotEnoughFloatingIPAvailable(exceptions.InvalidInput): +class NotEnoughFloatingIPAvailable(NotEnoughResourcesAvailable): msg_fmt = _("Not enough floating IPs available") @@ -226,3 +254,69 @@ class CantUpdateFloatingIPReservation(exceptions.BlazarException): class NeutronClientError(exceptions.BlazarException): msg_fmt = _("Failed to create Neutron resources for the reservation") + + +# Network plugin related exceptions + +class CantDeleteNetwork(exceptions.BlazarException): + code = 409 + msg_fmt = _("Can't delete network %(network)s. %(msg)s") + + +class NetworkNotFound(exceptions.NotFound): + msg_fmt = _("Network '%(network)s' not found!") + + +class SubnetpoolNotFound(exceptions.NotFound): + msg_fmt = _("Subnetpool '%(subnetpool)s' not found!") + + +class RouterNotFound(exceptions.NotFound): + msg_fmt = _("Router '%(router)s' not found!") + + +class InvalidNetwork(exceptions.NotAuthorized): + msg_fmt = _("Invalid values for network %(network)s") + + +class NotEnoughNetworksAvailable(NotEnoughResourcesAvailable): + msg_fmt = _("Not enough networks available") + + +class NetworkCreationFailed(exceptions.BlazarException): + msg_fmt = _("Failed to create network %(name)s for reservation %(id)s. " + "%(msg)s") + + +class NetworkExtraOnStartFailed(exceptions.BlazarException): + msg_fmt = _("Failed on extra on start steps for reservation %(id)s. " + "%(msg)s") + + +class NetworkDeletionFailed(exceptions.BlazarException): + msg_fmt = _("Failed to delete network %(network_id)s for reservation " + "%(reservation_id)s") + + +# Device plugin related exceptions + +class CantDeleteDevice(exceptions.BlazarException): + code = 409 + msg_fmt = _("Can't delete device %(device)s. %(msg)s") + + +class DeviceNotFound(exceptions.NotFound): + msg_fmt = _("Device '%(device)s' not found!") + + +class NotEnoughDevicesAvailable(NotEnoughResourcesAvailable): + msg_fmt = _("Not enough devices available") + + +class InvalidDevice(exceptions.NotAuthorized): + msg_fmt = _("Invalid values for device %(device)s") + + +class HostHavingContainers(exceptions.BlazarException): + code = 409 + msg_fmt = _("Containers found for host %(host)s") diff --git a/blazar/manager/floatingips/rpcapi.py b/blazar/manager/floatingips/rpcapi.py deleted file mode 100644 index 34322c69a..000000000 --- a/blazar/manager/floatingips/rpcapi.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2019 NTT. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from oslo_config import cfg - -from blazar import manager -from blazar.utils import service - -CONF = cfg.CONF -CONF.import_opt('rpc_topic', 'blazar.manager.service', 'manager') - - -class ManagerRPCAPI(service.RPCClient): - """Client side for the Manager RPC API. - - Used from other services to communicate with blazar-manager service. - """ - BASE_RPC_API_VERSION = '1.0' - - def __init__(self): - """Initiate RPC API client with needed topic and RPC version.""" - super(ManagerRPCAPI, self).__init__(manager.get_target()) - - def get_floatingip(self, floatingip_id): - """Get detailed info about a floatingip.""" - return self.call('virtual:floatingip:get_floatingip', - fip_id=floatingip_id) - - def list_floatingips(self): - """List all floatingips.""" - return self.call('virtual:floatingip:list_floatingip') - - def create_floatingip(self, floatingip_values): - """Create floatingip with specified parameters.""" - return self.call('virtual:floatingip:create_floatingip', - values=floatingip_values) - - def delete_floatingip(self, floatingip_id): - """Delete specified floatingip.""" - return self.call('virtual:floatingip:delete_floatingip', - fip_id=floatingip_id) diff --git a/blazar/manager/leases/rpcapi.py b/blazar/manager/leases/rpcapi.py index f429e640e..862544337 100644 --- a/blazar/manager/leases/rpcapi.py +++ b/blazar/manager/leases/rpcapi.py @@ -45,3 +45,15 @@ def update_lease(self, lease_id, values): def delete_lease(self, lease_id): """Delete specified lease.""" return self.call('delete_lease', lease_id=lease_id) + + def hosts_in_lease(self, lease_id): + """List all hosts in lease by its ID.""" + return self.call('hosts_in_lease', lease_id=lease_id) + + def networks_in_lease(self, lease_id): + """List all networks in lease by its ID.""" + return self.call('networks_in_lease', lease_id=lease_id) + + def devices_in_lease(self, lease_id): + """List all devices in lease by its ID.""" + return self.call('devices_in_lease', lease_id=lease_id) diff --git a/blazar/manager/networks/__init__.py b/blazar/manager/networks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/blazar/manager/oshosts/rpcapi.py b/blazar/manager/oshosts/rpcapi.py index 6aad773cd..00a53fe6a 100644 --- a/blazar/manager/oshosts/rpcapi.py +++ b/blazar/manager/oshosts/rpcapi.py @@ -56,11 +56,26 @@ def delete_computehost(self, host_id): return self.call('physical:host:delete_computehost', host_id=host_id) - def list_allocations(self, query): + def list_allocations(self, query, detail=False): """List all allocations on all computehosts.""" - return self.call('physical:host:list_allocations', query=query) + return self.call('physical:host:list_allocations', query=query, + detail=detail) def get_allocations(self, host_id, query): """List all allocations on a specified computehost.""" return self.call('physical:host:get_allocations', host_id=host_id, query=query) + + def reallocate(self, host_id, data): + """Exchange host from current allocations.""" + return self.call('physical:host:reallocate_computehost', + host_id=host_id, data=data) + + def list_resource_properties(self, query): + """List resource properties and possible values for computehosts.""" + return self.call('physical:host:list_resource_properties', query=query) + + def update_resource_property(self, property_name, values): + """Update resource property for computehost.""" + return self.call('physical:host:update_resource_property', + property_name=property_name, values=values) diff --git a/blazar/manager/service.py b/blazar/manager/service.py index 888bf3e23..a5ed67a5b 100644 --- a/blazar/manager/service.py +++ b/blazar/manager/service.py @@ -13,11 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import defaultdict import datetime -import eventlet +from functools import lru_cache + from oslo_config import cfg -from oslo_log import log as logging +from oslo_utils.excutils import save_and_reraise_exception +from oslo_utils import strutils +from oslo_service import periodic_task from stevedore import enabled from blazar import context @@ -30,8 +34,12 @@ from blazar import monitor from blazar.notification import api as notification_api from blazar import status +from blazar.utils.openstack import placement from blazar.utils import service as service_utils from blazar.utils import trusts +import eventlet +from oslo_log import log as logging + manager_opts = [ cfg.ListOpt('plugins', @@ -48,7 +56,7 @@ default=1, min=0, max=50, - help='Number of times to retry an event action.') + help='Number of times to retry an event action.'), ] CONF = cfg.CONF @@ -60,6 +68,15 @@ EVENT_INTERVAL = 10 +class PeriodicTaskManager(periodic_task.PeriodicTasks): + def __init__(self): + super(PeriodicTaskManager, self).__init__(CONF) + + def periodic_tasks(self, context, raise_on_error=False): + """Tasks to be run at a periodic interval.""" + return self.run_periodic_tasks(context, raise_on_error=raise_on_error) + + class ManagerService(service_utils.RPCServer): """Service class for the blazar-manager service. @@ -70,55 +87,29 @@ class ManagerService(service_utils.RPCServer): def __init__(self): target = manager.get_target() super(ManagerService, self).__init__(target) - self.plugins = self._get_plugins() + self.plugins = get_plugins() self.resource_actions = self._setup_actions() self.monitors = monitor.load_monitors(self.plugins) self.enforcement = enforcement.UsageEnforcement() + self.placement_client = placement.BlazarPlacementClient() + self.periodic_task_manager = PeriodicTaskManager() def start(self): super(ManagerService, self).start() # NOTE(jakecoll): stop_on_exception=False was added because database # exceptions would prevent threads from being scheduled again. # TODO(jakecoll): Find a way to test this. - self.tg.add_timer_args(EVENT_INTERVAL, self._event, + self.tg.add_timer_args(EVENT_INTERVAL, self._process_events, stop_on_exception=False) for m in self.monitors: m.start_monitoring() - def _get_plugins(self): - """Return dict of resource-plugin class pairs.""" - config_plugins = CONF.manager.plugins - plugins = {} - - extension_manager = enabled.EnabledExtensionManager( - check_func=lambda ext: ext.name in config_plugins, - namespace='blazar.resource.plugins', - invoke_on_load=False - ) + for plugin in self.plugins.values(): + if hasattr(plugin, "periodic_tasks"): + for task in plugin.periodic_tasks: + self.periodic_task_manager.add_periodic_task(task) - invalid_plugins = (set(config_plugins) - - set([ext.name for ext - in extension_manager.extensions])) - if invalid_plugins: - raise common_ex.BlazarException('Invalid plugin names are ' - 'specified: %s' % invalid_plugins) - - for ext in extension_manager.extensions: - try: - plugin_obj = ext.plugin() - except Exception as e: - LOG.warning("Could not load {0} plugin " - "for resource type {1} '{2}'".format( - ext.name, ext.plugin.resource_type, e)) - else: - if plugin_obj.resource_type in plugins: - msg = ("You have provided several plugins for " - "one resource type in configuration file. " - "Please set one plugin per resource type.") - raise exceptions.PluginConfigurationError(error=msg) - - plugins[plugin_obj.resource_type] = plugin_obj - return plugins + self.tg.add_dynamic_timer(self.periodic_tasks) def _setup_actions(self): """Setup actions for each resource type supported. @@ -142,25 +133,12 @@ def _setup_actions(self): return actions @service_utils.with_empty_context - def _event(self): - """Tries to commit event. - - If there is an event in Blazar DB to be done, do it and change its - status to 'DONE'. - """ - LOG.debug('Trying to get event from DB.') - events = db_api.event_get_all_sorted_by_filters( - sort_key='time', - sort_dir='asc', - filters={'status': status.event.UNDONE, - 'time': {'op': 'le', - 'border': datetime.datetime.utcnow()}} - ) - + def _process_events_concurrently(self, events): if not events: return LOG.info("Trying to execute events: %s", events) + event_threads = {} for event in events: if not status.LeaseStatus.is_stable(event['lease_id']): LOG.info("Skip event %s because the status of the lease %s " @@ -169,15 +147,93 @@ def _event(self): db_api.event_update(event['id'], {'status': status.event.IN_PROGRESS}) try: - eventlet.spawn_n( + event_thread = eventlet.spawn( service_utils.with_empty_context(self._exec_event), event) + event_threads[event['id']] = event_thread except Exception: db_api.event_update(event['id'], {'status': status.event.ERROR}) - LOG.exception('Error occurred while event %s handling.', + LOG.exception('Error occurred while spawning event %s.', event['id']) + for event_id, event_thread in event_threads.items(): + try: + event_thread.wait() + except Exception: + db_api.event_update(event_id, + {'status': status.event.ERROR}) + LOG.exception('Error occurred while handling event %s.', + event_id) + + def _select_for_execution(self, events): + """Selects the first events that can be safely executed concurrently. + + Events are selected to be executed concurrently if they are of the same + type, while keeping strict time ordering and the following priority of + event types: before_end_lease, end_lease, and start_lease (except for + before_end_lease events where there is a start_lease event for the same + lease at the same time). + + We ensure that: + + - the before_end_lease event of a lease is executed after the + start_lease event and before the end_lease event of the same lease, + - for two reservations using the same hosts back to back, the end_lease + event is executed before the start_lease event. + """ + if not events: + return [] + + events_by_lease = defaultdict(list) + events_by_type = defaultdict(list) + + first_events = [e for e in events if e['time'] == events[0]['time']] + for e in first_events: + events_by_lease[e['lease_id']].append(e) + events_by_type[e['event_type']].append(e) + + # If there is a start_lease event for the same lease, we run it first. + deferred_before_end_events = [] + deferred_end_events = [] + for start_event in events_by_type['start_lease']: + for e in events_by_lease[start_event['lease_id']]: + if e['event_type'] == 'before_end_lease': + events_by_type['before_end_lease'].remove(e) + deferred_before_end_events.append(e) + elif e['event_type'] == 'end_lease': + events_by_type['end_lease'].remove(e) + deferred_end_events.append(e) + + later_events = [e for e in events if e not in first_events] + + return [ + events_by_type['before_end_lease'], + events_by_type['end_lease'], + events_by_type['start_lease'], + deferred_before_end_events, + deferred_end_events, + later_events, + ] + + def _process_events(self): + """Tries to execute events. + + If there is any event in Blazar DB to be executed, do it and change its + status to 'DONE'. Events are executed concurrently if possible. + """ + LOG.debug('Trying to get events from DB.') + events = db_api.event_get_all_sorted_by_filters( + sort_key='time', + sort_dir='asc', + filters={'status': status.event.UNDONE, + 'time': {'op': 'le', + 'border': datetime.datetime.utcnow()}} + ) + + for batch in self._select_for_execution(events): + self._process_events_concurrently(batch) + def _exec_event(self, event): """Execute an event function""" event_fn = getattr(self, event['event_type'], None) @@ -207,9 +263,8 @@ def _exec_event(self, event): 'lease %s.', event['event_type'], event['lease_id']) else: lease = db_api.lease_get(event['lease_id']) - with trusts.create_ctx_from_trust(lease['trust_id']) as ctx: - self._send_notification( - lease, ctx, events=['event.%s' % event['event_type']]) + self._send_notification( + lease, events=['event.%s' % event['event_type']]) def _date_from_string(self, date_string, date_format=LEASE_DATE_FORMAT): try: @@ -230,7 +285,6 @@ def _parse_lease_dates(self, start_date, end_date): start_date = now else: start_date = self._date_from_string(start_date) - if end_date == 'now': end_date = now else: @@ -254,7 +308,7 @@ def _check_for_invalid_date_inputs(self, lease, values, now): 'Terminated leases can only be renamed') if (values['end_date'] < now or - values['end_date'] < values['start_date']): + values['end_date'] < values['start_date']): raise common_ex.InvalidInput( 'End date must be later than current and start date') @@ -268,8 +322,32 @@ def validate_params(self, values, required_params): def get_lease(self, lease_id): return db_api.lease_get(lease_id) + def hosts_in_lease(self, lease_id): + return db_api.hosts_in_lease(lease_id) + + def networks_in_lease(self, lease_id): + return db_api.networks_in_lease(lease_id) + + def devices_in_lease(self, lease_id): + return db_api.devices_in_lease(lease_id) + def list_leases(self, project_id=None, query=None): - return db_api.lease_list(project_id) + limit = query.get("limit") + if strutils.is_int_like(limit): + limit = int(limit) + elif limit is not None: + raise common_ex.InvalidInput( + f'limit must be integer, got {limit}') + return db_api.lease_list( + project_id, + status=query.get("status"), + lease_id=query.get("lease_id"), + lease_name=query.get("lease_name"), + marker=query.get("marker"), + limit=limit, + sort_key=query.get("sort_key", "end_date"), + sort_dir=query.get("sort_dir", "desc"), + ) def create_lease(self, lease_values): """Create a lease with reservations. @@ -313,12 +391,6 @@ def create_lease(self, lease_values): allocations = self._allocation_candidates( lease_values, reservations) - try: - self.enforcement.check_create( - context.current(), lease_values, reservations, allocations) - except common_ex.NotAuthorized as e: - LOG.error("Enforcement checks failed. %s", str(e)) - raise common_ex.NotAuthorized(e) events.append({'event_type': 'start_lease', 'time': start_date, @@ -336,7 +408,7 @@ def create_lease(self, lease_values): self._check_date_within_lease_limits(before_end_date, lease_values) except common_ex.BlazarException as e: - LOG.error("Invalid before_end_date param. %s", str(e)) + LOG.warning("Invalid before_end_date param. %s", str(e)) raise e elif CONF.manager.minutes_before_end_lease > 0: delta = datetime.timedelta( @@ -360,21 +432,36 @@ def create_lease(self, lease_values): raise exceptions.LeaseNameAlreadyExists( name=lease_values['name']) except db_ex.BlazarDBException: - LOG.exception('Cannot create a lease') - raise + with save_and_reraise_exception(): + LOG.exception('Cannot create a lease') else: try: for reservation in reservations: reservation['lease_id'] = lease['id'] reservation['start_date'] = lease['start_date'] reservation['end_date'] = lease['end_date'] - self._create_reservation(reservation) + reservation['project_id'] = lease['project_id'] + reservation["id"] = self._create_reservation(reservation) except Exception: - LOG.exception("Failed to create reservation for a lease. " - "Rollback the lease and associated " - "reservations") + with save_and_reraise_exception(): + LOG.exception("Failed to create reservation for a " + "lease. Rollback the lease and " + "associated reservations") + db_api.lease_destroy(lease_id) + + try: + self.enforcement.check_create( + context.current(), lease_values, reservations, allocations) + except common_ex.NotAuthorized as e: + LOG.warning("Enforcement checks failed. %s", str(e)) db_api.lease_destroy(lease_id) - raise + raise common_ex.NotAuthorized(e) + except Exception: + with save_and_reraise_exception(): + LOG.exception("Unexpected enforcement error. " + "Destroying lease and associated " + "reservations") + db_api.lease_destroy(lease_id) try: for event in events: @@ -382,22 +469,50 @@ def create_lease(self, lease_values): db_api.event_create(event) except (exceptions.UnsupportedResourceType, common_ex.BlazarException): - LOG.exception("Failed to create event for a lease. " - "Rollback the lease and associated " - "reservations") - db_api.lease_destroy(lease_id) - raise + with save_and_reraise_exception(): + LOG.exception("Failed to create event for a lease. " + "Rollback the lease and associated " + "reservations") + db_api.lease_destroy(lease_id) else: db_api.lease_update( lease_id, {'status': status.lease.PENDING}) lease = db_api.lease_get(lease_id) - self._send_notification(lease, ctx, events=['create']) + self._send_notification(lease, events=['create']) return lease + def _add_resource_type(self, reservations, existing_reservations): + rsvns_by_id = {} + + for r in existing_reservations: + rsvns_by_id[r['id']] = r + for r in reservations: + if 'resource_type' not in r: + r['resource_type'] = rsvns_by_id[r['id']]['resource_type'] + + return reservations + @status.lease.lease_status( - transition=status.lease.UPDATING, result_in=status.lease.STABLE) + transition=status.lease.UPDATING, + result_in=status.lease.STABLE, + non_fatal_exceptions=[ + common_ex.InvalidInput, + exceptions.InvalidRange, + exceptions.MissingParameter, + exceptions.MalformedRequirements, + exceptions.MalformedParameter, + exceptions.NotEnoughResourcesAvailable, + exceptions.InvalidDate, + exceptions.CantUpdateParameter, + exceptions.InvalidPeriod, + enforcement.exceptions.MaxLeaseDurationException, + enforcement.exceptions.MaxLeaseUpdateWindowException, + enforcement.exceptions.ExternalServiceUnsupportedHTTPResponse, + enforcement.exceptions.ExternalServiceFilterException, + ] + ) def update_lease(self, lease_id, values): if not values: return db_api.lease_get(lease_id) @@ -417,83 +532,87 @@ def update_lease(self, lease_id, values): start_date, end_date, now = self._parse_lease_dates(start_date, end_date) - values['start_date'] = start_date values['end_date'] = end_date - self._check_for_invalid_date_inputs(lease, values, now) - with trusts.create_ctx_from_trust(lease['trust_id']): - if before_end_date: - try: - before_end_date = self._date_from_string(before_end_date) - self._check_date_within_lease_limits(before_end_date, - values) - except common_ex.BlazarException as e: - LOG.error("Invalid before_end_date param. %s", str(e)) - raise e + if before_end_date: + try: + before_end_date = self._date_from_string(before_end_date) + self._check_date_within_lease_limits(before_end_date, + values) + except common_ex.BlazarException as e: + LOG.warning("Invalid before_end_date param. %s", str(e)) + raise e + + # TODO(frossigneux) rollback if an exception is raised + reservations = values.get('reservations', []) + existing_reservations = ( + db_api.reservation_get_all_by_lease_id(lease_id)) + try: + invalid_ids = set([r['id'] for r in reservations]).difference( + [r['id'] for r in existing_reservations]) + except KeyError: + raise exceptions.MissingParameter(param='reservation ID') - reservations = values.get('reservations', []) - existing_reservations = ( - db_api.reservation_get_all_by_lease_id(lease_id)) + if invalid_ids: + raise common_ex.InvalidInput( + 'Please enter valid reservation IDs. Invalid reservation ' + 'IDs are: %s' % ','.join([str(id) for id in invalid_ids])) - try: - invalid_ids = set([r['id'] for r in reservations]).difference( - [r['id'] for r in existing_reservations]) - except KeyError: - raise exceptions.MissingParameter(param='reservation ID') + # Populate resource_type if missing from data passed by client + reservations = self._add_resource_type( + reservations, existing_reservations) - if invalid_ids: - raise common_ex.InvalidInput( - 'Please enter valid reservation IDs. Invalid reservation ' - 'IDs are: %s' % ','.join([str(id) for id in invalid_ids])) + try: + [ + self.plugins[r['resource_type']] for r + in (reservations + existing_reservations)] + except KeyError: + raise exceptions.CantUpdateParameter(param='resource_type') - try: - [ - self.plugins[r['resource_type']] for r - in (reservations + existing_reservations)] - except KeyError: - raise exceptions.CantUpdateParameter(param='resource_type') - - existing_allocs = self._existing_allocations(existing_reservations) - - if reservations: - new_reservations = reservations - new_allocs = self._allocation_candidates(values, - existing_reservations) - else: - # User is not updating reservation parameters, e.g., is only - # adjusting lease start/end dates. - new_reservations = existing_reservations - new_allocs = existing_allocs + existing_allocs = self._existing_allocations(existing_reservations) + if reservations: + new_reservations = reservations + values["project_id"] = lease["project_id"] + new_allocs = self._allocation_candidates(values, + new_reservations) + else: + # User is not updating reservation parameters, e.g., is only + # adjusting lease start/end dates. + new_reservations = existing_reservations + new_allocs = existing_allocs + + try: + self.enforcement.check_update(context.current(), lease, values, + existing_allocs, new_allocs, + existing_reservations, + new_reservations) + except common_ex.NotAuthorized as e: + LOG.error("Enforcement checks failed. %s", str(e)) + # raise the same error so the decorator can capture it + # and handle non fatal exceptions + raise e + + # TODO(frossigneux) rollback if an exception is raised + for reservation in (existing_reservations): + v = {} + v['start_date'] = values['start_date'] + v['end_date'] = values['end_date'] try: - self.enforcement.check_update(context.current(), lease, values, - existing_allocs, new_allocs, - existing_reservations, - new_reservations) - except common_ex.NotAuthorized as e: - LOG.error("Enforcement checks failed. %s", str(e)) - raise common_ex.NotAuthorized(e) - - # TODO(frossigneux) rollback if an exception is raised - for reservation in existing_reservations: - v = {} - v['start_date'] = values['start_date'] - v['end_date'] = values['end_date'] - try: - v.update([r for r in reservations - if r['id'] == reservation['id']].pop()) - except IndexError: - pass - resource_type = v.get('resource_type', - reservation['resource_type']) - - if resource_type != reservation['resource_type']: - raise exceptions.CantUpdateParameter( - param='resource_type') - self.plugins[resource_type].update_reservation( - reservation['id'], v) + v.update([r for r in reservations + if r['id'] == reservation['id']].pop()) + except IndexError: + pass + resource_type = v.get('resource_type', + reservation['resource_type']) + + if resource_type != reservation['resource_type']: + raise exceptions.CantUpdateParameter( + param='resource_type') + self.plugins[resource_type].update_reservation( + reservation['id'], v) event = db_api.event_get_first_sorted_by_filters( 'lease_id', @@ -532,8 +651,7 @@ def update_lease(self, lease_id, values): db_api.lease_update(lease_id, values) lease = db_api.lease_get(lease_id) - with trusts.create_ctx_from_trust(lease['trust_id']) as ctx: - self._send_notification(lease, ctx, events=notifications) + self._send_notification(lease, events=notifications) return lease @@ -577,62 +695,68 @@ def delete_lease(self, lease_id): db_api.event_update(end_event['id'], {'status': status.event.IN_PROGRESS}) - with trusts.create_ctx_from_trust(lease['trust_id']) as ctx: - reservations = lease['reservations'] + reservations = self._reservations_execution_ordered(lease) - if lease_not_started or lease_not_ended: - # Only run the on_end enforcement if we're explicitly - # ending the lease for the first time OR if we're terminating - # it before the lease ever started. It's important to run - # on_end in the second case to inform enforcement that the - # lease is no longer in play. - allocations = self._existing_allocations(reservations) + if lease_not_started or lease_not_ended: + # Only run the on_end enforcement if we're explicitly + # ending the lease for the first time OR if we're terminating + # it before the lease ever started. It's important to run + # on_end in the second case to inform enforcement that the + # lease is no longer in play. + allocations = self._existing_allocations(reservations) + try: + self.enforcement.on_end(context.current(), lease, allocations) + except Exception as e: + LOG.error(e) + + unclean_end = False + for reservation in self._reservations_execution_ordered(lease): + if reservation['status'] != status.reservation.DELETED: + plugin = self.plugins[reservation['resource_type']] try: - self.enforcement.on_end(ctx, lease, allocations) - except Exception as e: - LOG.error(e) + plugin.on_end(reservation['resource_id'], lease=lease) + except (db_ex.BlazarDBException, RuntimeError): + LOG.exception("Failed to delete reservation %s", + reservation['id']) + unclean_end = True + if unclean_end: + raise exceptions.EventError( + error="Failed to cleanly end lease %(lease_id)s", + lease_id=lease['id']) - for reservation in reservations: - if reservation['status'] != status.reservation.DELETED: - plugin = self.plugins[reservation['resource_type']] - try: - plugin.on_end(reservation['resource_id']) - except (db_ex.BlazarDBException, RuntimeError): - LOG.exception("Failed to delete a reservation " - "for a lease.") - raise - db_api.lease_destroy(lease_id) - self._send_notification(lease, ctx, events=['delete']) + if end_lease: + db_api.event_update(end_event['id'], + {'status': status.event.DONE}) + db_api.lease_destroy(lease_id) + self._send_notification(lease, events=['delete']) @status.lease.lease_status( transition=status.lease.STARTING, result_in=(status.lease.ACTIVE, status.lease.ERROR)) def start_lease(self, lease_id, event_id): - lease = self.get_lease(lease_id) - with trusts.create_ctx_from_trust(lease['trust_id']): - self._basic_action(lease_id, event_id, 'on_start', - status.reservation.ACTIVE) + self._basic_action(lease_id, event_id, 'on_start', + status.reservation.ACTIVE) @status.lease.lease_status( transition=status.lease.TERMINATING, result_in=(status.lease.TERMINATED, status.lease.ERROR)) def end_lease(self, lease_id, event_id): lease = self.get_lease(lease_id) + allocations = self._existing_allocations(lease['reservations']) + try: + # no rpc call with authentication context, i.e. + # context.current() doesn't work here. + # so need to get context from the lease trust. + self.enforcement.on_end(trusts.create_ctx_from_trust( + lease['trust_id']), lease, allocations) + except Exception as e: + LOG.error(e) - with trusts.create_ctx_from_trust(lease['trust_id']) as ctx: - allocations = self._existing_allocations(lease['reservations']) - try: - self.enforcement.on_end(ctx, lease, allocations) - except Exception as e: - LOG.error(e) - - self._basic_action(lease_id, event_id, 'on_end', - status.reservation.DELETED) + self._basic_action(lease_id, event_id, 'on_end', + status.reservation.DELETED) def before_end_lease(self, lease_id, event_id): - lease = self.get_lease(lease_id) - with trusts.create_ctx_from_trust(lease['trust_id']): - self._basic_action(lease_id, event_id, 'before_end') + self._basic_action(lease_id, event_id, 'before_end') def _basic_action(self, lease_id, event_id, action_time, reservation_status=None): @@ -640,17 +764,21 @@ def _basic_action(self, lease_id, event_id, action_time, lease = self.get_lease(lease_id) event_status = status.event.DONE - for reservation in lease['reservations']: + for reservation in self._reservations_execution_ordered(lease): resource_type = reservation['resource_type'] try: if reservation_status is not None: if not status.reservation.is_valid_transition( reservation['status'], reservation_status): raise common_ex.InvalidStatus - self.resource_actions[resource_type][action_time]( - reservation['resource_id'] - ) - except common_ex.BlazarException: + action_fn = self.resource_actions[resource_type][action_time] + action_fn(reservation['resource_id'], lease=lease) + except Exception as exc: + if not isinstance(exc, common_ex.BlazarException): + LOG.warning(( + "An unexpected exception type was generated. This " + "indicates that some exception is not being wrapped " + "properly in a BlazarException.")) LOG.exception("Failed to execute action %(action)s " "for lease %(lease)s", {'action': action_time, @@ -668,6 +796,25 @@ def _basic_action(self, lease_id, event_id, action_time, return event_status + def _reservations_execution_ordered(self, lease): + """Sort reservations in order of desired execution. + + This is currently hard-coded such that network reservations always + execute last, because it is harder to tear down a network reservation + cleanly if there are still running instances related to a physical + host or instance reservation. + """ + execution_order = { + 'default': 0, + 'network': 1, + } + + def _sort_key(res): + return execution_order.get( + res['resource_type'], execution_order['default']) + + return sorted(lease['reservations'], key=_sort_key) + def _create_reservation(self, values): resource_type = values['resource_type'] if resource_type not in self.plugins: @@ -685,6 +832,7 @@ def _create_reservation(self, values): ) db_api.reservation_update(reservation['id'], {'resource_id': resource_id}) + return reservation["id"] def _allocation_candidates(self, lease, reservations): """Returns dict by resource type of reservation candidates.""" @@ -695,6 +843,7 @@ def _allocation_candidates(self, lease, reservations): resource_type = reservation['resource_type'] res['start_date'] = lease['start_date'] res['end_date'] = lease['end_date'] + res['project_id'] = lease['project_id'] if resource_type not in self.plugins: raise exceptions.UnsupportedResourceType( @@ -706,7 +855,38 @@ def _allocation_candidates(self, lease, reservations): raise common_ex.BlazarException( 'Invalid plugin names are specified: %s' % resource_type) - candidate_ids = plugin.allocation_candidates(res) + original_res = res.copy() + try: + plugin.update_default_parameters(res) + candidate_ids = plugin.allocation_candidates(res) + except exceptions.NotEnoughResourcesAvailable: + candidate_ids = None + # Retry this function if allowed + if hasattr( + CONF[plugin.resource_type], + "retry_allocation_without_defaults" + ) and CONF[plugin.resource_type]\ + .retry_allocation_without_defaults: + LOG.info("Not enough resources with default properties. " + "Retrying with defaults removed.") + try: + candidate_ids = plugin.allocation_candidates( + original_res) + except exceptions.NotEnoughResourcesAvailable: + pass + + # If the retry didn't get candidate IDs, raise an exception + if candidate_ids is None: + if hasattr( + CONF[plugin.resource_type], + "display_default_resource_properties" + ) and CONF[plugin.resource_type]\ + .display_default_resource_properties: + raise exceptions.\ + NotEnoughResourcesDefaultProperties( + params=str(res)) + else: + raise allocations[resource_type] = [ plugin.get(cid) for cid in candidate_ids] @@ -739,11 +919,11 @@ def _existing_allocations(self, reservations): return allocations - def _send_notification(self, lease, ctx, events=[]): + def _send_notification(self, lease, events=[]): payload = notification_api.format_lease_payload(lease) for event in events: - notification_api.send_lease_notification(ctx, payload, + notification_api.send_lease_notification({}, payload, 'lease.%s' % event) def _check_date_within_lease_limits(self, date, lease): @@ -789,27 +969,46 @@ def _update_before_end_event(self, old_lease, new_lease, db_api.event_update(event['id'], update_values) - def __getattr__(self, name): - """RPC Dispatcher for plugins methods.""" + def periodic_tasks(self, raise_on_error=False): + """Tasks to be run at a periodic interval.""" + ctxt = context.admin() + return self.periodic_task_manager.periodic_tasks( + ctxt, raise_on_error=raise_on_error + ) - fn = None - try: - resource_type, method = name.rsplit(':', 1) - except ValueError: - # NOTE(sbauza) : the dispatcher needs to know which plugin to use, - # raising error if consequently not - raise AttributeError(name) + +@lru_cache(maxsize=None) +def get_plugins(): + """Return dict of resource-plugin class pairs.""" + config_plugins = CONF.manager.plugins + plugins = {} + + extension_manager = enabled.EnabledExtensionManager( + check_func=lambda ext: ext.name in config_plugins, + namespace='blazar.resource.plugins', + invoke_on_load=False + ) + + invalid_plugins = (set(config_plugins) - + set([ext.name for ext + in extension_manager.extensions])) + if invalid_plugins: + raise common_ex.BlazarException('Invalid plugin names are ' + 'specified: %s' % invalid_plugins) + + for ext in extension_manager.extensions: try: - try: - fn = getattr(self.plugins[resource_type], method) - except KeyError: - LOG.error("Plugin with resource type %s not found", - resource_type) - raise exceptions.UnsupportedResourceType( - resource_type=resource_type) - except AttributeError: - LOG.error("Plugin %s doesn't include method %s", - self.plugins[resource_type], method) - if fn is not None: - return fn - raise AttributeError(name) + plugin_obj = ext.plugin() + except Exception as e: + LOG.warning("Could not load {0} plugin " + "for resource type {1} '{2}'".format( + ext.name, ext.plugin.resource_type, e)) + else: + if plugin_obj.resource_type in plugins: + msg = ("You have provided several plugins for " + "one resource type in configuration file. " + "Please set one plugin per resource type.") + raise exceptions.PluginConfigurationError(error=msg) + + plugins[plugin_obj.resource_type] = plugin_obj + return plugins diff --git a/blazar/opts.py b/blazar/opts.py index 518481b16..82c09969d 100644 --- a/blazar/opts.py +++ b/blazar/opts.py @@ -45,6 +45,8 @@ def list_opts(): ('manager', itertools.chain(blazar.manager.opts, blazar.manager.service.manager_opts)), ('enforcement', itertools.chain( + blazar.enforcement.filters.external_service_filter + .ExternalServiceFilter.enforcement_opts, blazar.enforcement.filters.max_lease_duration_filter.MaxLeaseDurationFilter.enforcement_opts, # noqa blazar.enforcement.enforcement.enforcement_opts)), ('notifications', blazar.notification.notifier.notification_opts), diff --git a/blazar/plugins/base.py b/blazar/plugins/base.py index 6ee753e02..245d08ee8 100644 --- a/blazar/plugins/base.py +++ b/blazar/plugins/base.py @@ -14,7 +14,14 @@ # limitations under the License. import abc - +import collections +import datetime + +from blazar import context +from blazar import policy +from blazar.db import api as db_api +from blazar.db import utils as db_utils +from blazar.utils.openstack import keystone from oslo_config import cfg from oslo_log import log as logging @@ -89,16 +96,57 @@ def update_reservation(self, reservation_id, values): pass @abc.abstractmethod - def on_end(self, resource_id): + def on_end(self, resource_id, lease=None): """Delete resource.""" pass @abc.abstractmethod - def on_start(self, resource_id): + def on_start(self, resource_id, lease=None): """Wake up resource.""" pass - def before_end(self, resource_id): + def list_resource_properties(self, query): + detail = False if not query else query.get('detail', False) + all_properties = False if not query else query.get('all', False) + resource_properties = collections.defaultdict(list) + is_property_unique = {} + + include_private = all_properties and policy.enforce( + context.current(), 'admin', {}, do_raise=False) + + for name, private, value, is_unique in db_api.resource_properties_list( + self.resource_type): + + if include_private or not private: + resource_properties[name].append(value) + is_property_unique[name] = is_unique + + if detail: + return [ + dict( + property=k, private=False, values=v, + is_unique=is_property_unique[k] + ) + for k, v in resource_properties.items()] + else: + return [dict(property=k) for k, v in resource_properties.items()] + + def update_default_parameters(self, values): + """Update values with any defaults""" + pass + + def add_default_resource_properties(self, values): + if not values.get('resource_properties', ''): + values['resource_properties'] = CONF[ + self.resource_type + ].default_resource_properties + return values + + def update_resource_property(self, property_name, values): + return db_api.resource_property_update( + self.resource_type, property_name, values) + + def before_end(self, resource_id, lease=None): """Take actions before the end of a lease""" pass @@ -124,6 +172,45 @@ def get_query_options(self, params, index_type): unsupported) return options + def is_project_allowed(self, project_id, resource): + # If this resource has the extra capability "authorized_projects" + if "authorized_projects" in resource and \ + isinstance(resource["authorized_projects"], str): + # Parse the field as a CSV, and check the resulting list + authorized_projects = resource["authorized_projects"].split(",") + return project_id in authorized_projects + return True + + def add_extra_allocation_info(self, resource_allocations): + """Add extra information to allocations (to show in calendar)""" + extras = CONF.api.allocation_extras + for allocs in resource_allocations.values(): + for alloc in allocs: + alloc["extras"] = {} + if "user_name" in extras: + ids = [] + for allocations in resource_allocations.values(): + for alloc in allocations: + ids.append(alloc["lease_id"]) + items = db_utils.get_user_ids_for_lease_ids(ids) + keystoneclient = keystone.BlazarKeystoneClient() + users = keystoneclient.users.list() + user_map = {user.id: user for user in users} + lease_to_name = dict() + for lease_id, user_id in items: + user = user_map[user_id] + lease_to_name[lease_id] = user.name + + for allocations in resource_allocations.values(): + for alloc in allocations: + alloc["extras"]["user_name"] = \ + lease_to_name[alloc["lease_id"]] + + def add_allocation_cleaning_time(self, resource_allocations, cleaning_time): + for allocs in resource_allocations.values(): + for alloc in allocs: + alloc["start_date"] = alloc["start_date"] - datetime.timedelta(minutes=cleaning_time) + alloc["end_date"] = alloc["end_date"] + datetime.timedelta(minutes=cleaning_time) class BaseMonitorPlugin(metaclass=abc.ABCMeta): """Base class of monitor plugin.""" diff --git a/blazar/plugins/devices/__init__.py b/blazar/plugins/devices/__init__.py new file mode 100644 index 000000000..ebdfd7126 --- /dev/null +++ b/blazar/plugins/devices/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2014 Bull. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RESOURCE_TYPE = u'device' diff --git a/blazar/plugins/devices/device_plugin.py b/blazar/plugins/devices/device_plugin.py new file mode 100644 index 000000000..0ed209171 --- /dev/null +++ b/blazar/plugins/devices/device_plugin.py @@ -0,0 +1,759 @@ +# -*- coding: utf-8 -*- +# +# Author: Pierre Riteau +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from collections import defaultdict +import datetime + +from oslo_config import cfg +from oslo_utils import strutils +from stevedore import named + +from blazar.db import api as db_api +from blazar.db import exceptions as db_ex +from blazar.db import utils as db_utils +from blazar.manager import exceptions as manager_ex +from blazar.plugins import base +from blazar.plugins import devices as plugin +from blazar.plugins import monitor +from blazar import status +from blazar.utils import plugins as plugins_utils +from oslo_log import log as logging +import random + + +plugin_opts = [ + cfg.StrOpt('before_end', + default='', + help='Actions which we will be taken before the end of ' + 'the lease'), + cfg.ListOpt('plugins', + default=['zun.plugin', 'k8s.plugin'], + help='All plugins to use (one for every device driver to ' + 'support.)'), + cfg.IntOpt('cleaning_time', + default=0, + min=0, + help='The minimum interval [minutes] between the end of a ' + 'lease and the start of the next lease for the same ' + 'device. This interval is used for cleanup.'), + cfg.StrOpt('default_resource_properties', + default='', + help='Default resource_properties when creating a lease of ' + 'this type.'), + cfg.BoolOpt('display_default_resource_properties', + default=True, + help='Display default resource_properties if allocation fails ' + 'due to not enough resources'), + cfg.BoolOpt('retry_allocation_without_defaults', + default=True, + help='Whether an allocation should be retried on failure ' + 'without the default properties'), +] + +plugin_opts.extend(monitor.monitor_opts) + +CONF = cfg.CONF +CONF.register_opts(plugin_opts, group=plugin.RESOURCE_TYPE) +LOG = logging.getLogger(__name__) + +before_end_options = ['', 'default', 'email'] + +QUERY_TYPE_ALLOCATION = 'allocation' + +MONITOR_ARGS = {"resource_type": plugin.RESOURCE_TYPE} + + +def _get_plugins(): + """Return dict of resource-plugin class pairs.""" + plugins = {} + + extension_manager = named.NamedExtensionManager( + namespace='blazar.device.driver.plugins', + names=CONF.device.plugins, + invoke_on_load=False + ) + + for ext in extension_manager.extensions: + try: + plugin_obj = ext.plugin() + except Exception as e: + LOG.warning("Could not load {0} plugin " + "for resource type {1} '{2}'".format( + ext.name, ext.plugin.device_driver, e)) + LOG.exception(e) + else: + if plugin_obj.device_driver in plugins: + msg = ("You have provided several plugins for " + "one device driver in configuration file. " + "Please set one plugin per device driver.") + raise manager_ex.PluginConfigurationError(error=msg) + + plugins[plugin_obj.device_driver] = plugin_obj + return plugins + + +class DevicePlugin(base.BasePlugin): + """Plugin for device resource.""" + resource_type = plugin.RESOURCE_TYPE + title = 'Device Plugin' + description = 'This plugin creates and deletes devices.' + query_options = { + QUERY_TYPE_ALLOCATION: ['lease_id', 'reservation_id'] + } + + def __init__(self): + super(DevicePlugin, self).__init__() + self.plugins = _get_plugins() + self.monitor = DeviceMonitorPlugin(**MONITOR_ARGS) + self.monitor.register_reallocater(self._reallocate) + + def reserve_resource(self, reservation_id, values): + """Create reservation.""" + device_ids = self.allocation_candidates(values) + + if not device_ids: + raise manager_ex.NotEnoughDevicesAvailable() + + device_rsrv_values = { + 'reservation_id': reservation_id, + 'resource_properties': values['resource_properties'], + 'count_range': values['count_range'], + 'status': 'pending', + 'before_end': values['before_end'], + } + device_reservation = db_api.device_reservation_create( + device_rsrv_values) + for device_id in device_ids: + db_api.device_allocation_create({'device_id': device_id, + 'reservation_id': reservation_id}) + return device_reservation['id'] + + def update_reservation(self, reservation_id, values): + """Update reservation.""" + reservation = db_api.reservation_get(reservation_id) + lease = db_api.lease_get(reservation['lease_id']) + + if (not [x for x in values.keys() if x in ['min', 'max', + 'resource_properties']] + and values['start_date'] >= lease['start_date'] + and values['end_date'] <= lease['end_date']): + # Nothing to update + return + + dates_before = {'start_date': lease['start_date'], + 'end_date': lease['end_date']} + dates_after = {'start_date': values['start_date'], + 'end_date': values['end_date']} + device_reservation = db_api.device_reservation_get( + reservation['resource_id']) + self._update_allocations(dates_before, dates_after, reservation_id, + reservation['status'], device_reservation, + values, lease) + + updates = {} + if 'min' in values or 'max' in values: + count_range = str(values.get( + 'min', device_reservation['count_range'].split('-')[0]) + ) + '-' + str(values.get( + 'max', device_reservation['count_range'].split('-')[1]) + ) + updates['count_range'] = count_range + if 'resource_properties' in values: + updates['resource_properties'] = values.get( + 'resource_properties') + if updates: + db_api.device_reservation_update(device_reservation['id'], updates) + + def on_start(self, resource_id, lease=None): + device_reservation = db_api.device_reservation_get(resource_id) + + devices = defaultdict(list) + for allocation in db_api.device_allocation_get_all_by_values( + reservation_id=device_reservation['reservation_id']): + device = db_api.device_get(allocation['device_id']) + devices[device["device_driver"]].append(device) + + for device_driver, devices_list in devices.items(): + self.plugins[device_driver].allocate( + device_reservation, lease, devices_list) + + def before_end(self, resource_id, lease=None): + """Take an action before the end of a lease.""" + device_reservation = db_api.device_reservation_get(resource_id) + + action = device_reservation['before_end'] + if action == 'default': + action = CONF[plugin.RESOURCE_TYPE].before_end + + if action == 'email': + plugins_utils.send_lease_extension_reminder( + lease, CONF.os_region_name) + + def on_end(self, resource_id, lease=None): + device_reservation = db_api.device_reservation_get(resource_id) + db_api.device_reservation_update(device_reservation['id'], + {'status': 'completed'}) + + devices = defaultdict(list) + allocations = db_api.device_allocation_get_all_by_values( + reservation_id=device_reservation['reservation_id']) + for allocation in allocations: + device = db_api.device_get(allocation['device_id']) + devices[device["device_driver"]].append( + db_api.device_get(allocation['device_id'])) + db_api.device_allocation_destroy(allocation['id']) + + for device_driver, devices_list in devices.items(): + self.plugins[device_driver].deallocate( + device_reservation, lease, devices_list) + + def _get_extra_capabilities(self, device_id): + extra_capabilities = {} + raw_extra_capabilities = ( + db_api.device_extra_capability_get_all_per_device(device_id)) + for capability, capability_name in raw_extra_capabilities: + key = capability_name + extra_capabilities[key] = capability.capability_value + return extra_capabilities + + def get(self, device_id): + return self.get_device(device_id) + + def get_device(self, device_id): + device = db_api.device_get(device_id) + if device is None: + return device + return self.get_device_with_extra_capabilities(device) + + def get_device_with_extra_capabilities(self, device): + extra_capabilities = self._get_extra_capabilities(device["id"]) + if extra_capabilities: + res = device.copy() + res.update(extra_capabilities) + return res + else: + return device + + def list_devices(self): + raw_device_list = db_api.device_list() + device_list = [] + for device in raw_device_list: + device_list.append(self.get_device(device['id'])) + return device_list + + def create_device(self, values): + if 'trust_id' in values: + del values['trust_id'] + device_id = self.plugins[values.get( + 'device_driver')].create_device(values) + return self.get_device(device_id) + + def is_updatable_extra_capability(self, capability, capability_name): + reservations = db_utils.get_reservations_by_device_id( + capability['device_id'], datetime.datetime.utcnow(), + datetime.date.max) + + for r in reservations: + plugin_reservation = db_utils.get_plugin_reservation( + r['resource_type'], r['resource_id']) + + requirements_queries = plugins_utils.convert_requirements( + plugin_reservation['resource_properties']) + + for requirement in requirements_queries: + if requirement.split(" ")[0] == capability_name: + return False + return True + + def update_device(self, device_id, values): + # nothing to update + if not values: + return self.get_device(device_id) + + device_property_names = ['device_type', 'device_driver'] + device_properties = {} + for prop_key in list(values.keys()): + if prop_key in device_property_names: + device_properties[prop_key] = values.pop(prop_key) + if device_properties: + db_api.device_update(device_id, device_properties) + + cant_update_extra_capability = [] + cant_delete_extra_capability = [] + previous_capabilities = self._get_extra_capabilities(device_id) + updated_keys = set(values.keys()) & set(previous_capabilities.keys()) + new_keys = set(values.keys()) - set(previous_capabilities.keys()) + + for key in updated_keys: + raw_capability, cap_name = next(iter( + db_api.device_extra_capability_get_all_per_name( + device_id, key))) + + if self.is_updatable_extra_capability(raw_capability, cap_name): + if values[key] is not None: + try: + capability = {'capability_value': values[key]} + db_api.device_extra_capability_update( + raw_capability['id'], capability) + except (db_ex.BlazarDBException, RuntimeError): + cant_update_extra_capability.append(cap_name) + else: + try: + db_api.device_extra_capability_destroy( + raw_capability['id']) + except db_ex.BlazarDBException: + cant_delete_extra_capability.append(cap_name) + else: + LOG.info("Capability %s can't be updated because " + "existing reservations require it.", + cap_name) + cant_update_extra_capability.append(cap_name) + + for key in new_keys: + new_capability = { + 'device_id': device_id, + 'capability_name': key, + 'capability_value': values[key], + } + try: + db_api.device_extra_capability_create(new_capability) + except (db_ex.BlazarDBException, RuntimeError): + cant_update_extra_capability.append(key) + + if cant_update_extra_capability: + raise manager_ex.CantAddExtraCapability( + host=device_id, keys=cant_update_extra_capability) + + if cant_delete_extra_capability: + raise manager_ex.ExtraCapabilityNotFound( + resource=device_id, keys=cant_delete_extra_capability) + + LOG.info('Extra capabilities on device %s updated with %s', + device_id, values) + return self.get_device(device_id) + + def delete_device(self, device_id): + device = db_api.device_get(device_id) + if not device: + raise manager_ex.DeviceNotFound(device=device_id) + + if db_api.device_allocation_get_all_by_values( + device_id=device_id): + raise manager_ex.CantDeleteDevice( + device=device_id, + msg='The device is reserved.' + ) + + try: + db_api.device_destroy(device_id) + self.plugins[device["device_driver"]].after_destroy(device) + except db_ex.BlazarDBException as e: + raise manager_ex.CantDeleteDevice(device=device_id, msg=str(e)) + + def reallocate_device(self, device_id, data): + allocations = self.get_allocations(device_id, data, detail=True) + + for alloc in allocations['reservations']: + reservation_flags = {} + device_allocation = db_api.device_allocation_get_all_by_values( + device_id=device_id, + reservation_id=alloc['id'])[0] + + if self._reallocate(device_allocation): + if alloc['status'] == status.reservation.ACTIVE: + reservation_flags.update(dict(resources_changed=True)) + db_api.lease_update(alloc['lease_id'], dict(degraded=True)) + else: + reservation_flags.update(dict(missing_resources=True)) + db_api.lease_update(alloc['lease_id'], dict(degraded=True)) + + db_api.reservation_update(alloc['id'], reservation_flags) + + return self.get_allocations(device_id, data) + + def _reallocate(self, allocation): + """Allocate an alternative device. + + :param allocation: allocation to change. + :return: True if an alternative device was successfully allocated. + """ + reservation = db_api.reservation_get(allocation['reservation_id']) + device_reservation = db_api.device_reservation_get( + reservation['resource_id']) + lease = db_api.lease_get(reservation['lease_id']) + + # Remove the old device from the trait. + if reservation['status'] == status.reservation.ACTIVE: + device = db_api.device_get(allocation['device_id']) + self.plugins[device["device_driver"]].remove_active_device( + device, device_reservation, lease) + + # Allocate an alternative device. + start_date = max(datetime.datetime.utcnow(), lease['start_date']) + new_deviceids = self._matching_devices( + device_reservation['resource_properties'], + '1-1', start_date, lease['end_date'], lease['project_id'] + ) + if not new_deviceids: + db_api.device_allocation_destroy(allocation['id']) + LOG.warn('Could not find alternative device for reservation %s ' + '(lease: %s).', reservation['id'], lease['name']) + return False + else: + new_deviceid = new_deviceids.pop() + db_api.device_allocation_update(allocation['id'], + {'device_id': new_deviceid}) + LOG.warn('Resource changed for reservation %s (lease: %s).', + reservation['id'], lease['name']) + if reservation['status'] == status.reservation.ACTIVE: + new_device = db_api.device_get(new_deviceid) + self.plugins[device["device_driver"]].add_active_device( + new_device, device_reservation, lease) + + return True + + def list_allocations(self, query, detail=False): + devices_id_list = [d['id'] for d in db_api.device_list()] + options = self.get_query_options(query, QUERY_TYPE_ALLOCATION) + options['detail'] = detail + devices_allocations = self.query_device_allocations(devices_id_list, + **options) + self.add_extra_allocation_info(devices_allocations) + self.add_allocation_cleaning_time(devices_allocations, CONF.cleaning_time) + return [{"resource_id": device, "reservations": allocs} + for device, allocs in devices_allocations.items()] + + def get_allocations(self, device_id, query, detail=False): + options = self.get_query_options(query, QUERY_TYPE_ALLOCATION) + options['detail'] = detail + device_allocations = self.query_device_allocations( + [device_id], **options) + allocs = device_allocations.get(device_id, []) + return {"resource_id": device_id, "reservations": allocs} + + def query_allocations(self, devices, lease_id=None, reservation_id=None): + return self.query_device_allocations(devices, lease_id=lease_id, + reservation_id=reservation_id) + + def query_device_allocations(self, devices, lease_id=None, + reservation_id=None, detail=False): + """Return dict of device and its allocations. + + The list element forms + { + 'device-id': [ + { + 'lease_id': lease_id, + 'id': reservation_id, + 'start_date': lease_start_date, + 'end_date': lease_end_date + }, + ] + }. + """ + start = datetime.datetime.utcnow() + end = datetime.date.max + + reservations = db_utils.get_reservation_allocations_by_device_ids( + devices, start, end, lease_id, reservation_id) + device_allocations = {d: [] for d in devices} + + for reservation in reservations: + if not detail: + del reservation['project_id'] + del reservation['lease_name'] + del reservation['status'] + + for device_id in reservation['device_ids']: + if device_id in device_allocations.keys(): + device_allocations[device_id].append({ + k: v for k, v in reservation.items() + if k != 'device_ids'}) + + return device_allocations + + def update_default_parameters(self, values): + self.add_default_resource_properties(values) + + def allocation_candidates(self, values): + self._check_params(values) + + device_ids = self._matching_devices( + values['resource_properties'], + values['count_range'], + values['start_date'], + values['end_date'], + values['project_id'] + ) + + min_devices, _ = [int(n) for n in values['count_range'].split('-')] + + if len(device_ids) < min_devices: + raise manager_ex.NotEnoughDevicesAvailable() + + return device_ids + + def _convert_int_param(self, param, name): + """Checks that the parameter is present and can be converted to int.""" + if param is None: + raise manager_ex.MissingParameter(param=name) + if strutils.is_int_like(param): + param = int(param) + else: + raise manager_ex.MalformedParameter(param=name) + return param + + def _validate_min_max_range(self, values, min_devices, max_devices): + min_devices = self._convert_int_param(min_devices, 'min') + max_devices = self._convert_int_param(max_devices, 'max') + if min_devices <= 0 or max_devices <= 0: + raise manager_ex.MalformedParameter( + param='min and max (must be greater than or equal to 1)') + if max_devices < min_devices: + raise manager_ex.InvalidRange() + values['count_range'] = str(min_devices) + '-' + str(max_devices) + + def _check_params(self, values): + self._validate_min_max_range(values, values.get('min'), + values.get('max')) + + if 'resource_properties' not in values: + raise manager_ex.MissingParameter(param='resource_properties') + + if 'before_end' not in values: + values['before_end'] = 'default' + if values['before_end'] not in before_end_options: + raise manager_ex.MalformedParameter(param='before_end') + + if 'on_start' not in values: + values['on_start'] = 'default' + + def _matching_devices(self, resource_properties, count_range, + start_date, end_date, project_id): + """Return the matching devices (preferably not allocated)""" + count_range = count_range.split('-') + min_device = count_range[0] + max_device = count_range[1] + allocated_device_ids = [] + not_allocated_device_ids = [] + filter_array = [] + start_date_with_margin = start_date - datetime.timedelta( + minutes=CONF.device.cleaning_time) + end_date_with_margin = end_date + datetime.timedelta( + minutes=CONF.device.cleaning_time) + + if resource_properties: + filter_array += plugins_utils.convert_requirements( + resource_properties) + for device in db_api.reservable_device_get_all_by_queries( + filter_array): + device = self.get_device_with_extra_capabilities(device) + if not self.is_project_allowed(project_id, device): + continue + if not db_api.device_allocation_get_all_by_values( + device_id=device['id']): + not_allocated_device_ids.append(device['id']) + elif db_utils.get_free_periods( + device['id'], + start_date_with_margin, + end_date_with_margin, + end_date_with_margin - start_date_with_margin, + resource_type='device' + ) == [ + (start_date_with_margin, end_date_with_margin), + ]: + allocated_device_ids.append(device['id']) + if len(not_allocated_device_ids) >= int(min_device): + random.shuffle(not_allocated_device_ids) + return not_allocated_device_ids[:int(max_device)] + all_device_ids = allocated_device_ids + not_allocated_device_ids + if len(all_device_ids) >= int(min_device): + random.shuffle(all_device_ids) + return all_device_ids[:int(max_device)] + else: + return [] + + def _update_allocations(self, dates_before, dates_after, reservation_id, + reservation_status, device_reservation, values, + lease): + min_devices = values.get('min', int( + device_reservation['count_range'].split('-')[0])) + max_devices = values.get( + 'max', int(device_reservation['count_range'].split('-')[1])) + self._validate_min_max_range(values, min_devices, max_devices) + resource_properties = values.get( + 'resource_properties', + device_reservation['resource_properties']) + allocs = db_api.device_allocation_get_all_by_values( + reservation_id=reservation_id) + + allocs_to_remove = self._allocations_to_remove( + dates_before, dates_after, max_devices, + resource_properties, allocs) + + if (allocs_to_remove and + reservation_status == status.reservation.ACTIVE): + raise manager_ex.NotEnoughDevicesAvailable() + + kept_devices = len(allocs) - len(allocs_to_remove) + if kept_devices < max_devices: + min_devices = min_devices - kept_devices \ + if (min_devices - kept_devices) > 0 else 0 + max_devices = max_devices - kept_devices + device_ids = self._matching_devices( + resource_properties, + str(min_devices) + '-' + str(max_devices), + dates_after['start_date'], dates_after['end_date'], + lease['project_id']) + if len(device_ids) >= min_devices: + for device_id in device_ids: + db_api.device_allocation_create( + {'device_id': device_id, + 'reservation_id': reservation_id}) + new_device = db_api.device_get(device_id) + if reservation_status == status.reservation.ACTIVE: + # Add new device into the trait. + + self.plugins[new_device["device_driver"]].\ + add_active_device( + new_device, device_reservation, lease) + else: + raise manager_ex.NotEnoughDevicesAvailable() + + for allocation in allocs_to_remove: + db_api.device_allocation_destroy(allocation['id']) + + def _allocations_to_remove(self, dates_before, dates_after, max_devices, + resource_properties, allocs): + allocs_to_remove = [] + requested_device_ids = [device['id'] for device in + self._filter_devices_by_properties( + resource_properties + )] + for alloc in allocs: + if alloc['device_id'] not in requested_device_ids: + allocs_to_remove.append(alloc) + continue + if (dates_before['start_date'] > dates_after['start_date'] or + dates_before['end_date'] < dates_after['end_date']): + reserved_periods = db_utils.get_reserved_periods( + alloc['device_id'], + dates_after['start_date'], + dates_after['end_date'], + datetime.timedelta(seconds=1)) + + max_start = max(dates_before['start_date'], + dates_after['start_date']) + min_end = min(dates_before['end_date'], + dates_after['end_date']) + + if not (len(reserved_periods) == 0 or + (len(reserved_periods) == 1 and + reserved_periods[0][0] == max_start and + reserved_periods[0][1] == min_end)): + allocs_to_remove.append(alloc) + kept_devices = len(allocs) - len(allocs_to_remove) + if kept_devices > max_devices: + allocs_to_remove.extend( + [allocation for allocation in allocs + if allocation not in allocs_to_remove + ][:(kept_devices - max_devices)] + ) + + return allocs_to_remove + + def _filter_devices_by_properties(self, resource_properties): + filter = [] + if resource_properties: + filter += plugins_utils.convert_requirements(resource_properties) + if filter: + return db_api.device_get_all_by_queries(filter) + else: + return db_api.device_list() + + +class DeviceMonitorPlugin(monitor.GeneralMonitorPlugin): + """Monitor plugin for device resource.""" + + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = \ + super(DeviceMonitorPlugin, cls).__new__(cls, *args, **kwargs) + cls._instance.plugins = _get_plugins() + return cls._instance + + def filter_allocations(self, reservation, device_ids): + return [alloc for alloc + in reservation['device_allocations'] + if alloc['device_id'] in device_ids] + + def get_reservations_by_resource_ids(self, device_ids, + interval_begin, interval_end): + return db_utils.get_reservations_by_device_ids(device_ids, + interval_begin, + interval_end) + + def get_unreservable_resourses(self): + return db_api.unreservable_device_get_all_by_queries([]) + + def get_notification_event_types(self): + """Get event types of notification messages to handle.""" + return [] + + def notification_callback(self, event_type, payload): + """Handle a notification message. + + It is used as a callback of a notification-based resource monitor. + :param event_type: an event type of a notification. + :param payload: a payload of a notification. + :return: a dictionary of {reservation id: flags to update} + e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': + {'missing_resources': True}} + """ + return {} + + def set_reservable(self, resource, is_reservable): + db_api.device_update(resource["id"], {"reservable": is_reservable}) + LOG.warn('%s %s.', resource["name"], + "recovered" if is_reservable else "failed") + + def poll_resource_failures(self): + """Check health of devices by calling driver service API. + + :return: a list of failed devices, a list of recovered devices. + """ + devices = db_api.device_get_all_by_filters({}) + + device_partition = defaultdict(list) + for device in devices: + device_partition[device["device_driver"]].append(device) + + failed_devices = [] + recovered_devices = [] + + for device_driver in self.plugins.keys(): + try: + driver_failed_devices, driver_recovered_devices = \ + self.plugins[device_driver].poll_resource_failures( + device_partition[device_driver]) + failed_devices.extend(driver_failed_devices) + recovered_devices.extend(driver_recovered_devices) + except AttributeError as e: + LOG.warning('poll_resource_failures is not implemented for {}' + .format(device_driver)) + raise e + + return failed_devices, recovered_devices diff --git a/blazar/plugins/devices/k8s_plugin.py b/blazar/plugins/devices/k8s_plugin.py new file mode 100644 index 000000000..1e588c4c9 --- /dev/null +++ b/blazar/plugins/devices/k8s_plugin.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# +# Author: Chameleon Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +from blazar.db import api as db_api +from blazar.db import exceptions as db_ex +from blazar.manager import exceptions as manager_ex +from kubernetes import client +from kubernetes import config +from oslo_log import log as logging + +opts = [ + cfg.StrOpt( + 'kubeconfig_file', help='Kubeconfig file to use for calls to k8s'), +] + +CONF = cfg.CONF +CONF.register_opts(opts, group="k8s") +LOG = logging.getLogger(__name__) + +LABEL_NAMESPACE = "blazar.openstack.org" +LABELS = { + "reservation_id": f"{LABEL_NAMESPACE}/reservation_id", + "project_id": f"{LABEL_NAMESPACE}/project_id", + "device": f"{LABEL_NAMESPACE}/device", +} + + +class K8sPlugin(): + device_driver = 'k8s' + + def __init__(self): + config.load_kube_config(config_file=CONF.k8s.kubeconfig_file) + self.core_v1 = client.CoreV1Api() + self.apps_v1 = client.AppsV1Api() + + def has_label(self, node, label, value): + '''Get if the node has label=value, or if value is none, any value''' + return (label in node.metadata.labels and + (value is None or node.metadata.labels.get(label) == value)) + + def set_label(self, name, label, value): + body = { + "metadata": { + "labels": { + label: value, + }, + }, + } + return self.core_v1.patch_node(name, body) + + def set_res_id_label(self, name, reservation_id): + return self.set_label( + name, LABELS["reservation_id"], reservation_id) + + def set_project_id_label(self, name, project_id): + return self.set_label( + name, LABELS["project_id"], project_id) + + def set_device(self, name, value): + return self.set_label(name, LABELS["device"], value) + + def create_device(self, device_values): + device_name = device_values.get('name') + + if device_name is None: + raise manager_ex.InvalidHost(host=device_values) + + try: + self.core_v1.read_node(device_name) + except client.ApiException as exc: + if exc.status != 404: + LOG.exception("Error fetching node from k8s") + raise manager_ex.DeviceNotFound(device=device_name) + + device_properties = { + 'name': device_name, + 'device_type': 'container', + 'device_driver': K8sPlugin.device_driver + } + + to_store = set(device_values.keys()) - set(device_properties.keys()) + extra_capabilities_keys = to_store + extra_capabilities = dict( + (key, device_values[key]) for key in extra_capabilities_keys + ) + + if any([len(key) > 64 for key in extra_capabilities_keys]): + raise manager_ex.ExtraCapabilityTooLong() + + device = None + cantaddextracapability = [] + try: + device = db_api.device_create(device_properties) + except db_ex.BlazarDBException: + raise + for key in extra_capabilities: + values = {'device_id': device['id'], + 'capability_name': key, + 'capability_value': extra_capabilities[key], + } + try: + db_api.device_extra_capability_create(values) + except db_ex.BlazarDBException: + cantaddextracapability.append(key) + if cantaddextracapability: + raise manager_ex.CantAddExtraCapability( + keys=cantaddextracapability, + host=device['id']) + self.set_device(device_name, device['id']) + return device['id'] + + def is_active(self, node): + conditions = node.status.conditions + for i in conditions: + if i.type == "Ready": + return i.status == "True" + return False + + def poll_resource_failures(self, devices): + failed_devices = [] + recovered_devices = [] + + all_nodes = self.core_v1.list_node().items + for device in devices: + for node in all_nodes: + if node.metadata.name == device["name"]: + if not self.is_active(node) and device["reservable"]: + failed_devices.append(device) + elif self.is_active(node) and not device["reservable"]: + recovered_devices.append(device) + # Handle case when node is rebuilt; it will not have the + # "device" label anymore. + if node.metadata.labels.get(LABELS["device"]) != device["id"]: + self.set_device(device["name"], device["id"]) + break + else: + # Node is completely missing from k8s + failed_devices.append(device) + + return failed_devices, recovered_devices + + def allocate(self, device_reservation, lease, devices): + project_id = lease["project_id"] + for device in devices: + self.set_res_id_label( + device["name"], device_reservation["reservation_id"]) + self.set_project_id_label(device["name"], project_id) + + def deallocate(self, device_reservation, lease, devices): + namespace = lease["project_id"] + for device in devices: + self.set_res_id_label(device["name"], None) + self.set_project_id_label(device["name"], None) + + for deployment in self.apps_v1.list_namespaced_deployment( + namespace).items: + if self.has_label( + deployment, + LABELS["reservation_id"], + device_reservation["reservation_id"]): + self.apps_v1.delete_namespaced_deployment( + deployment.metadata.name, namespace) + + def after_destroy(self, device): + self.set_device(device["name"], None) + + def remove_active_device(self, device, device_reservation, lease): + self.set_res_id_label(device["name"], None) + self.set_project_id_label(device["name"], None) + + def add_active_device(self, device, device_reservation, lease): + project_id = lease["project_id"] + self.set_res_id_label( + device["name"], device_reservation["reservation_id"]) + self.set_project_id_label(device["name"], project_id) diff --git a/blazar/plugins/devices/zun_plugin.py b/blazar/plugins/devices/zun_plugin.py new file mode 100644 index 000000000..dae9acafb --- /dev/null +++ b/blazar/plugins/devices/zun_plugin.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- +# +# Author: Pierre Riteau +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +from blazar.db import api as db_api +from blazar.db import exceptions as db_ex +from blazar.manager import exceptions as manager_ex +from blazar.utils.openstack import placement +from blazar.utils.openstack import zun +from oslo_log import log as logging +from zunclient import exceptions as zun_ex + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +class ZunPlugin(zun.ZunClientWrapper): + """Plugin for zun device driver.""" + device_driver = 'zun' + + def __init__(self): + self.placement_client = placement.BlazarPlacementClient() + for blazar_device in db_api.device_list(): + if not blazar_device['reservable']: + continue + name = blazar_device['name'] + parent_rp = self.placement_client.get_resource_provider( + name) + reservation_rp = self.placement_client.get_reservation_provider( + name) + if not parent_rp: + LOG.warning("No resource provider found " + "for blazar device {}".format(name)) + elif not reservation_rp: + LOG.warning("No reservation provider found for blazar " + "device {}. Auto-creating one. ".format(name)) + rrp = self.placement_client.create_reservation_provider(name) + LOG.info( + "Reservation provider {} has created.".format(rrp['name'])) + + def create_device(self, device_values): + device_id = device_values.get('id') + device_name = device_values.get('name') + device_ref = device_id or device_name + + if device_ref is None: + raise manager_ex.InvalidHost(host=device_values) + + inventory = zun.ZunInventory() + zun_compute_node = inventory.get_host_details(device_ref) + if len(zun_compute_node['containers']) > 0: + raise manager_ex.HostHavingContainers(host=device_ref) + + device_properties = {'id': zun_compute_node['id'], + 'name': zun_compute_node['name'], + 'device_type': 'container', + 'device_driver': 'zun'} + + to_store = set(device_values.keys()) - set(device_properties.keys()) + extra_capabilities_keys = to_store + extra_capabilities = dict( + (key, device_values[key]) for key in extra_capabilities_keys + ) + + if any([len(key) > 64 for key in extra_capabilities_keys]): + raise manager_ex.ExtraCapabilityTooLong() + + self.placement_client.create_reservation_provider( + host_name=zun_compute_node['name']) + + device = None + cantaddextracapability = [] + try: + device = db_api.device_create(device_properties) + except db_ex.BlazarDBException: + self.placement_client.delete_reservation_provider( + host_name=zun_compute_node['name']) + raise + for key in extra_capabilities: + values = {'device_id': device['id'], + 'capability_name': key, + 'capability_value': extra_capabilities[key], + } + try: + db_api.device_extra_capability_create(values) + except db_ex.BlazarDBException: + cantaddextracapability.append(key) + if cantaddextracapability: + raise manager_ex.CantAddExtraCapability( + keys=cantaddextracapability, + host=device['id']) + return device['id'] + + def cleanup_device(self, device): + try: + # TODO(jason): zunclient is broken when passing both all_projects + # and 'host' as a keyword argument; the parameters are encoded + # like /v1/containers/?all_projects=1?host=..., which is malformed. + # Passing in 'host' to the list() function would however probably + # be more efficient. + host_containers = [ + container for container in + self.zun.containers.list(all_projects=True) + if container.host == device['name'] + ] + except zun_ex.ClientException as exc: + LOG.error(( + 'During lease teardown, failed to enumerate containers. ' + 'Containers may need to be manually cleaned up on %s.' + 'Error: %s' + ), device['name'], exc) + host_containers = [] + + for container in host_containers: + try: + self.zun.containers.delete( + container.uuid, force=True, stop=True) + except zun_ex.NotFound: + LOG.info('Could not find container %s, may have been deleted ' + 'concurrently.', container.name) + except Exception as e: + LOG.exception('Failed to delete %s: %s.', + container.name, str(e)) + + def poll_resource_failures(self, devices): + failed_devices = [] + recovered_devices = [] + + zun_compute_services = {s.host: s for s in self.zun.services.list()} + zun_devices = {d["name"]: d for d in devices + if d.get("device_driver") == self.device_driver} + + for device_name, device in zun_devices.items(): + is_reservable = device.get("reservable") + cs = zun_compute_services.get(device_name) + if is_reservable and cs and \ + cs.state == 'down' or cs.disabled: + failed_devices.append(device) + if not is_reservable and cs and \ + cs.state == 'up' and not cs.disabled: + recovered_devices.append(device) + + return failed_devices, recovered_devices + + def allocate(self, device_reservation, lease, devices): + self.placement_client.create_reservation_trait( + device_reservation['reservation_id'], lease['project_id']) + for device in devices: + rp = self.placement_client.get_reservation_provider(device['name']) + self.placement_client. \ + associate_reservation_trait_with_resource_provider( + rp['uuid'], + device_reservation['reservation_id'], + lease['project_id']) + + def remove_active_device(self, device, device_reservation, lease): + rp = self.placement_client.get_reservation_provider(device['name']) + self.placement_client. \ + dissociate_reservation_trait_with_resource_provider( + rp['uuid'], + device_reservation['reservation_id'], + lease['project_id']) + + def add_active_device(self, device, device_reservation, lease): + rp = self.placement_client.get_reservation_provider( + device['name']) + self.placement_client. \ + associate_reservation_trait_with_resource_provider( + rp['uuid'], + device_reservation['reservation_id'], + lease['project_id']) + + def deallocate(self, device_reservation, lease, devices): + # If a device lease fails to start, the reservation trait is never + # added to the parent resource provider. If that lease is deleted, + # deleting the trait fails because it does not exist. This case + # will be handled by logging a message rather than failing. + reservation_id = device_reservation['reservation_id'] + project_id = lease['project_id'] + if not self.placement_client.reservation_trait_exists( + reservation_id, project_id): + LOG.warning("Reservation trait doesn't exist for reservation {0} " + "and project {1}".format(reservation_id, project_id)) + return + resource_providers = self.placement_client. \ + get_reservation_trait_resource_providers(reservation_id, + project_id) + for rp in resource_providers: + self.placement_client. \ + dissociate_reservation_trait_with_resource_provider( + rp['uuid'], + reservation_id, + project_id) + device = None + for d in devices: + if d["id"] == rp['parent_provider_uuid']: + device = d + break + if device: + self.cleanup_device(device) + else: + LOG.warning( + 'Failed to retrieve device from resource provider %s', + rp['parent_provider_uuid'] + ) + self.placement_client.delete_reservation_trait( + reservation_id, project_id) + + def after_destroy(self, device): + self.placement_client.delete_reservation_provider(device['name']) diff --git a/blazar/plugins/dummy_vm_plugin.py b/blazar/plugins/dummy_vm_plugin.py index 4abc8e14b..74c70ba37 100644 --- a/blazar/plugins/dummy_vm_plugin.py +++ b/blazar/plugins/dummy_vm_plugin.py @@ -42,10 +42,10 @@ def allocation_candidates(self, lease_values): def update_reservation(self, reservation_id, values): return None - def on_start(self, resource_id): + def on_start(self, resource_id, lease=None): """Dummy VM plugin does nothing.""" return 'VM %s should be waked up this moment.' % resource_id - def on_end(self, resource_id): + def on_end(self, resource_id, lease=None): """Dummy VM plugin does nothing.""" return 'VM %s should be deleted this moment.' % resource_id diff --git a/blazar/plugins/flavor/__init__.py b/blazar/plugins/flavor/__init__.py new file mode 100644 index 000000000..837b97a08 --- /dev/null +++ b/blazar/plugins/flavor/__init__.py @@ -0,0 +1,14 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RESOURCE_TYPE = 'flavor:instance' diff --git a/blazar/plugins/flavor/flavor_plugin.py b/blazar/plugins/flavor/flavor_plugin.py new file mode 100644 index 000000000..d2feefbd5 --- /dev/null +++ b/blazar/plugins/flavor/flavor_plugin.py @@ -0,0 +1,518 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import collections +import datetime +import json + +from novaclient import exceptions as nova_exceptions +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import strutils + +from blazar import context +from blazar.db import api as db_api +from blazar.manager import exceptions as mgr_exceptions +from blazar.plugins import base +from blazar.plugins import flavor as plugin +from blazar.plugins.instances import instance_plugin +from blazar.plugins.oshosts import host_plugin +from blazar.utils.openstack import nova +from blazar.utils import plugins as plugins_utils +from blazar.utils.openstack import placement + +plugin_opts = [ + cfg.StrOpt('before_end', + default='', + help='Actions which we will be taken before the end of ' + 'the lease'), + cfg.BoolOpt('filter_ironic_hosts', + default=True, + help='Filter out ironic (baremetal) hosts from flavor ' + 'reservation candidates.'), +] + +CONF = cfg.CONF +CONF.register_opts(plugin_opts, group=plugin.RESOURCE_TYPE) + +LOG = logging.getLogger(__name__) + + +before_end_options = ['', 'snapshot', 'default', 'email'] + +QUERY_TYPE_ALLOCATION = 'allocation' + + +class FlavorPlugin(base.BasePlugin): + """Plugin for Nova flavor-based server reservations.""" + + resource_type = plugin.RESOURCE_TYPE + title = 'Plugin for Nova flavor-based server reservations' + description = 'Reserve compute resources modeled by Nova flavors.' + + query_options = { + QUERY_TYPE_ALLOCATION: ['lease_id', 'reservation_id'] + } + + def __init__(self): + super().__init__() + self._placement_client = placement.BlazarPlacementClient() + self._host_plugin = host_plugin.PhysicalHostPlugin() + self._instance_plugin = instance_plugin.VirtualInstancePlugin() + + def get(self, host_id): + return self._host_plugin.get(host_id) + + def list_allocations(self, query): + return self._host_plugin.list_allocations(query) + + def query_allocations(self, hosts, lease_id=None, reservation_id=None): + return self._host_plugin.query_allocations( + hosts, lease_id, reservation_id) + + def allocation_candidates(self, reservation): + """Return a list of candidate host_ids.""" + host_ids, _ = self._pick_hosts(reservation) + return host_ids + + def _pick_hosts(self, reservation): + self._validate_reservation_params(reservation) + + flavor_id = reservation['flavor_id'] + resource_request, resource_traits, source_flavor = \ + self._get_flavor_details(flavor_id) + + affinity = strutils.bool_from_string( + reservation['affinity'], default=None) + start_date = reservation['start_date'] + end_date = reservation['end_date'] + candidates = self._query_available_hosts( + start_date, end_date, resource_request, resource_traits, + reservation['project_id']) + + # Fail if we have fewer candidates than amount requested + req_amount = reservation['amount'] + if len(candidates) < req_amount: + raise mgr_exceptions.NotEnoughHostsAvailable() + + if affinity is not None: + raise mgr_exceptions.NotImplemented( + error="Affinity not supported yet") + + # return just enough hosts to satisfy the request + while len(candidates) > req_amount: + candidates.pop() + host_ids = [host['id'] for host in candidates] + return host_ids, source_flavor + + def _validate_reservation_params(self, values): + marshall_attributes = set([ + 'amount', 'flavor_id', 'affinity', 'start_date', 'end_date', + ]) + missing_attr = marshall_attributes - set(values.keys()) + if missing_attr: + raise mgr_exceptions.MissingParameter(param=','.join(missing_attr)) + + try: + values['amount'] = strutils.validate_integer( + values['amount'], "amount", 1, db_api.DB_MAX_INT) + except ValueError as e: + raise mgr_exceptions.MalformedParameter(str(e)) + + def _query_available_hosts(self, start_date, end_date, + resource_request, resource_traits, project_id, + excludes=[]): + # TODO(johngarbutt): offload more of this to the db + # we should be able to exclude hosts that don't match the + # resource requests, e.g. baremetal vs virtual + # or missing traits + if resource_traits: + raise mgr_exceptions.NotImplemented( + error="Resource traits not supported yet") + hosts = db_api.reservable_host_get_all_by_queries([]) + + if CONF[self.resource_type].filter_ironic_hosts: + # Flavor reservations schedule Nova instances, which cannot run on + # ironic (baremetal) hosts; exclude them so they stay available to + # the physical:host plugin. + hosts = [h for h in hosts if h['hypervisor_type'] != 'ironic'] + + # Honor per-host project restrictions. authorized_projects is an extra + # capability, which get_computehost joins to the host for us. + hosts = [ + h for h in hosts + if self.is_project_allowed( + project_id, self._host_plugin.get_computehost(h['id'])) + ] + + # find reservations for each host in our time period + free_hosts, reserved_hosts = \ + self._instance_plugin.filter_hosts_by_reservation( + hosts, + start_date - datetime.timedelta(minutes=CONF.cleaning_time), + end_date + datetime.timedelta(minutes=CONF.cleaning_time), + excludes) + + available_hosts = [] + for host_info in (reserved_hosts + free_hosts): + # check how many instances can fit on this host + hosts_list = self._get_hosts_list(host_info, resource_request) + available_hosts.extend(hosts_list) + return available_hosts + + def _get_hosts_list(self, host_info, resource_request): + """For given host, work out how many instances can fit on it.""" + + # For each host, look how many slots are available, + # given the current list of reservations within the + # target time window for this host + + # get high water mark of usage during all reservations + max_usage = self._max_usages(host_info['reservations']) + LOG.debug(f"Max usage {host_info['host']['hypervisor_hostname']} " + f"is {max_usage}") + + host = host_info['host'] + host_crs = db_api.host_resource_inventory_get_all_per_host(host['id']) + host_inventory = {cr['resource_class']: cr for cr in host_crs} + if not host_inventory: + LOG.warning("host added before inventory set in DB!") + return [] + LOG.debug(f"Inventory for {host_info['host']['hypervisor_hostname']} " + f"is {host_inventory}") + + # see how much room for slots we have + hosts_list = [] + current_usage = max_usage.copy() + + def has_free_slot(): + for rc, requested in resource_request.items(): + if not requested: + # skip things like requests for 0 vcpus + continue + + host_details = host_inventory.get(rc) + if not host_details: + # host doesn't have this sort of resource + LOG.debug(f"Resource {rc} not found for " + f"{host_info['host']['hypervisor_hostname']}") + return False + usage = current_usage[rc] + + if requested > host_details["max_unit"]: + # requested more than the max allowed by this host + LOG.debug(f"Requested {requested} {rc} for " + f"{host_info['host']['hypervisor_hostname']} " + f"but maximum is {host_details['max_unit']}") + return False + + capacity = ((host_details["total"] - host_details["reserved"]) + * host_details["allocation_ratio"]) + LOG.debug(f"Capacity is {capacity} for {rc} for " + f"{host_info['host']['hypervisor_hostname']}") + if (usage + requested) > capacity: + LOG.debug("Current usage is %d, requested %d", + usage, requested) + return False + + # We have enough resources for all resource requests + return True + + while (has_free_slot()): + hosts_list.append(host) + for rc, requested in resource_request.items(): + current_usage[rc] += requested + + LOG.debug(f"For host {host_info['host']['hypervisor_hostname']} " + f"we have {len(hosts_list)} slots.") + return hosts_list + + def _get_cached_flavor(self, instance_reservation): + source_flavor = instance_reservation["resource_properties"] + if source_flavor and "OS-FLV-EXT-DATA:ephemeral" in source_flavor: + return json.loads(source_flavor) + + def _max_usages(self, reservations): + """For reservation list for a host, find resource high watermark.""" + def resource_usage_by_event(event): + instance_reservation = event['reservation']['instance_reservation'] + request_count = instance_reservation["amount"] + source_flavor = self._get_cached_flavor(instance_reservation) + if source_flavor: + flavor_resource_inventory, _ = \ + self._estimate_flavor_resources(source_flavor) + return { + rc: amount * request_count + for rc, amount in flavor_resource_inventory.items() + if amount > 0 + } + raise mgr_exceptions.ReservationTypeConflict() + + # Get sorted list of events for all reservations + # that exist in the target time window + events_list = [] + for r in reservations: + fetched_events = db_api.event_get_all_sorted_by_filters( + sort_key='time', sort_dir='asc', + filters={'lease_id': r['lease_id']}) + events_list.extend([{'event': e, 'reservation': r} + for e in fetched_events]) + events_list.sort(key=lambda x: x['event']['time']) + + current_usage = collections.defaultdict(int) + max_usage = collections.defaultdict(int) + for event in events_list: + usage = resource_usage_by_event(event) + + if event['event']['event_type'] == 'start_lease': + LOG.debug(f"found start{event} with {usage}") + for rc, usage_amount in usage.items(): + current_usage[rc] += usage_amount + # TODO(johngarbutt) what if the max usage is + # actually outside the target time window? + if max_usage[rc] < current_usage[rc]: + max_usage[rc] = current_usage[rc] + + elif event['event']['event_type'] == 'end_lease': + for rc, usage_amount in usage.items(): + current_usage[rc] -= usage_amount + + LOG.debug(f"after {event}\nusage is: {current_usage}\n" + f"max is: {max_usage}") + return max_usage + + def _get_flavor_details(self, flavor_id): + # access nova using the user token, + # to ensure we can only see flavors they can see + user_client = nova.NovaClientWrapper() + flavor = user_client.nova.nova.flavors.get(flavor_id) + source_flavor = flavor.to_dict() + # TODO(johngarbutt): use newer api to get this above + source_flavor["extra_specs"] = flavor.get_keys() + if 'links' in source_flavor.keys(): + del source_flavor['links'] + + # NOTE(johngarbutt): we are only partially reproducing all the + # options that are available in a flavor. + resource_request, resource_traits = \ + self._estimate_flavor_resources(source_flavor) + + return (resource_request, resource_traits, source_flavor) + + def _estimate_flavor_resources(self, source_flavor): + resource_request = {} + resource_traits = {} + + # add default resource requests + resource_request["VCPU"] = int(source_flavor['vcpus']) + resource_request["MEMORY_MB"] = int(source_flavor['ram']) + # NOTE(priteau): This reserves resources for the root disk even if the + # instance will boot from volume. + resource_request["DISK_GB"] = ( + int(source_flavor['disk']) + + int(source_flavor['OS-FLV-EXT-DATA:ephemeral'])) + + # Check for PCPUs + hw_cpu_policy = source_flavor['extra_specs'].get("hw:cpu_policy") + if hw_cpu_policy == "dedicated": + resource_request["PCPU"] = source_flavor['vcpus'] + resource_request["VCPU"] = 0 + + # Check for traits and extra resources + for key, value in source_flavor['extra_specs'].items(): + if key.startswith("trait:"): + trait = key.split(":")[1] + if value == "required": + resource_traits[trait] = "required" + elif value == "forbidden": + resource_traits[trait] = "forbidden" + else: + raise mgr_exceptions.MalformedParameter( + "Invalid value for trait %s" % trait) + + if key.startswith("resources:"): + rc = key.split(":")[1] + resource_request[rc] = int(value) + + # TODO(johngarbutt): look for other extra specs that we + # don't support and error out the reservation if we find any + + return (resource_request, resource_traits) + + def reserve_resource(self, reservation_id, values): + host_ids, source_flavor = self._pick_hosts(values) + + if 'before_end' not in values: + values['before_end'] = 'default' + if values['before_end'] not in before_end_options: + raise mgr_exceptions.MalformedParameter(param='before_end') + + instance_reservation_val = { + 'reservation_id': reservation_id, + # use flavor display values, + # even though we probably reserve different resources + 'vcpus': source_flavor["vcpus"], + 'memory_mb': source_flavor["ram"], + 'disk_gb': source_flavor["disk"], + 'amount': values['amount'], + 'affinity': None, + 'resource_properties': json.dumps(source_flavor), + 'before_end': values['before_end'], + } + instance_reservation = db_api.instance_reservation_create( + instance_reservation_val) + + for host_id in host_ids: + db_api.host_allocation_create({'compute_host_id': host_id, + 'reservation_id': reservation_id}) + + try: + flavor_id, aggregate_id = \ + self._create_resources(instance_reservation) + except nova_exceptions.ClientException: + LOG.exception("Failed to create Nova resources " + "for reservation %s", reservation_id) + self._cleanup_resources(instance_reservation) + raise mgr_exceptions.NovaClientError() + + db_api.instance_reservation_update(instance_reservation['id'], + {'flavor_id': flavor_id, + 'aggregate_id': aggregate_id}) + + return instance_reservation['id'] + + def _create_resources(self, instance_reservation): + reservation_id = instance_reservation['reservation_id'] + # TODO(johngarbutt) we ignore affinity for now + # user_client = nova.NovaClientWrapper() + # reserved_group = user_client.nova.server_groups.create( + # instance_plugin.RESERVATION_PREFIX + ':' + reservation_id, + # 'affinity' if inst_reservation['affinity'] else 'anti-affinity' + # ) + + reserved_flavor = self._create_flavor(instance_reservation) + + ctx = context.current() + pool = nova.ReservationPool() + pool_metadata = { + instance_plugin.RESERVATION_PREFIX: reservation_id, + # this is added to work with Nova configuration options + # [scheduler]limit_tenants_to_placement_aggregate=True and + # [scheduler]placement_aggregate_required_for_tenants=True + 'filter_tenant_id': ctx.project_id, + } + agg = pool.create(name=reservation_id, metadata=pool_metadata) + + # TODO(johngarbutt) maybe add inventory here, but mark + # then inventory as reserved to start with? + self._placement_client.create_reservation_class(reservation_id) + + return reserved_flavor.id, agg.id + + def _create_flavor(self, instance_reservation): + source_flavor = self._get_cached_flavor(instance_reservation) + if not source_flavor: + raise mgr_exceptions.ReservationTypeConflict() + + reservation_id = instance_reservation['reservation_id'] + flavor_details = { + 'flavorid': reservation_id, + 'name': instance_plugin.RESERVATION_PREFIX + ":" + reservation_id, + 'vcpus': source_flavor['vcpus'], + 'ram': source_flavor['ram'], + 'disk': source_flavor['disk'], + 'is_public': False + } + # create flavor using admin access + reserved_flavor = self._instance_plugin.nova.nova.flavors.create( + **flavor_details) + + # Set extra specs to the flavor + rsv_id_rc_format = reservation_id.upper().replace("-", "_") + reservation_rc = "resources:CUSTOM_RESERVATION_" + rsv_id_rc_format + extra_specs = source_flavor["extra_specs"].copy() + extra_specs[instance_plugin.FLAVOR_EXTRA_SPEC] = reservation_id + extra_specs[reservation_rc] = "1" + reserved_flavor.set_keys(extra_specs) + + return reserved_flavor + + def _cleanup_resources(self, instance_reservation): + self._instance_plugin.cleanup_resources(instance_reservation) + + def update_reservation(self, reservation_id, values): + """Only supports updating lease start and end date.""" + + reservation = db_api.reservation_get(reservation_id) + instance_reservation = db_api.instance_reservation_get( + reservation['resource_id']) + + if ('flavor_id' in values and + values.get('flavor_id') != instance_reservation['flavor_id']): + raise mgr_exceptions.CantUpdateParameter(param="flavor_id") + if ('amount' in values and + values.get('amount') != instance_reservation['amount']): + raise mgr_exceptions.CantUpdateParameter(param="amount") + + flavor_id = instance_reservation['flavor_id'] + resource_request, resource_traits, _ = self._get_flavor_details( + flavor_id) + + # The Nova flavor includes this reservation as a custom resource. As it + # isn't in the host inventory DB, querying for it won't match anything, + # so pop the reservation resource before querying available hosts. + rsv_id_rc_format = reservation_id.upper().replace("-", "_") + reservation_rc = "CUSTOM_RESERVATION_" + rsv_id_rc_format + if reservation_rc in resource_request: + resource_request.pop(reservation_rc) + + existing_allocations = db_api.host_allocation_get_all_by_values( + reservation_id=reservation_id) + lease = db_api.lease_get(reservation['lease_id']) + candidates = self._query_available_hosts( + values['start_date'], + values['end_date'], + resource_request, resource_traits, + lease['project_id'], + [reservation_id] + ) + + # Ensure that every existing allocation still has a candidate host, so + # the reservation can stay where it is over the new time window. + alloc_count = collections.Counter( + [alloc["compute_host_id"] for alloc in existing_allocations] + ) + candidate_count = collections.Counter( + [can["id"] for can in candidates] + ) + # TODO(Mark): We could support changing "amount" pretty easy here. + if not all(alloc_count[key] <= candidate_count.get(key, 0) + for key in alloc_count): + raise mgr_exceptions.NotEnoughHostsAvailable() + + def on_start(self, resource_id, lease=None): + self._instance_plugin.on_start(resource_id, lease) + + def on_end(self, resource_id, lease=None): + self._instance_plugin.on_end(resource_id, lease) + + def before_end(self, resource_id, lease=None): + """Take an action before the end of a lease.""" + instance_reservation = db_api.instance_reservation_get(resource_id) + + action = instance_reservation['before_end'] + if action == 'default': + action = CONF[plugin.RESOURCE_TYPE].before_end + + if action == 'email': + plugins_utils.send_lease_extension_reminder( + lease, CONF.os_region_name) diff --git a/blazar/plugins/floatingips/floatingip_plugin.py b/blazar/plugins/floatingips/floatingip_plugin.py index 8bc734854..a266fb3cc 100644 --- a/blazar/plugins/floatingips/floatingip_plugin.py +++ b/blazar/plugins/floatingips/floatingip_plugin.py @@ -13,30 +13,41 @@ # under the License. import datetime +import concurrent.futures from oslo_config import cfg from oslo_log import log as logging +from oslo_utils.excutils import save_and_reraise_exception from oslo_utils import netutils from oslo_utils import strutils -from blazar import context from blazar.db import api as db_api from blazar.db import exceptions as db_ex from blazar.db import utils as db_utils from blazar import exceptions from blazar.manager import exceptions as manager_ex -from blazar.plugins import base +from blazar.plugins import base, monitor from blazar.plugins import floatingips as plugin from blazar import status from blazar.utils.openstack import neutron from blazar.utils import plugins as plugins_utils +from blazar.utils.openstack import exceptions as utils_exceptions +plugin_opts = [ + cfg.BoolOpt('retry_allocation_without_defaults', + default=False, + help='Whether an allocation should be retried on failure ' + 'without the default properties'), +] + +plugin_opts.extend(monitor.monitor_opts) CONF = cfg.CONF +CONF.register_opts(plugin_opts, group=plugin.RESOURCE_TYPE) LOG = logging.getLogger(__name__) QUERY_TYPE_ALLOCATION = 'allocation' - +MONITOR_ARGS = {"resource_type": plugin.RESOURCE_TYPE} class FloatingIpPlugin(base.BasePlugin): """Plugin for floating IP resource.""" @@ -48,6 +59,10 @@ class FloatingIpPlugin(base.BasePlugin): QUERY_TYPE_ALLOCATION: ['lease_id', 'reservation_id'] } + def __init__(self): + super(FloatingIpPlugin, self).__init__() + self.monitor = FloatingIpMonitorPlugin(**MONITOR_ARGS) + def check_params(self, values): if 'network_id' not in values: raise manager_ex.MissingParameter(param='network_id') @@ -68,18 +83,14 @@ def check_params(self, values): raise manager_ex.InvalidIPFormat(ip=ip) def _update_allocations(self, dates_before, dates_after, reservation_id, - reservation_status, fip_reservation, values): + reservation_status, fip_reservation, lease, + values): amount = int(values.get('amount', fip_reservation['amount'])) fip_allocations = db_api.fip_allocation_get_all_by_values( reservation_id=reservation_id) allocs_to_remove = self._allocations_to_remove( dates_before, dates_after, fip_allocations, amount) - if (allocs_to_remove and - reservation_status == status.reservation.ACTIVE): - raise manager_ex.CantUpdateFloatingIPReservation( - msg="Cannot remove allocations from an active reservation") - kept_fips = len(fip_allocations) - len(allocs_to_remove) fip_ids_to_add = [] @@ -98,7 +109,6 @@ def _update_allocations(self, dates_before, dates_after, reservation_id, # Create new floating IPs if reservation is active created_fips = [] if reservation_status == status.reservation.ACTIVE: - ctx = context.current() fip_pool = neutron.FloatingIPPool(fip_reservation['network_id']) for fip_id in fip_ids_to_add: try: @@ -108,7 +118,7 @@ def _update_allocations(self, dates_before, dates_after, reservation_id, fip['floating_ip_address'], reservation_id)) fip_pool.create_reserved_floatingip( fip['subnet_id'], fip['floating_ip_address'], - ctx.project_id, reservation_id) + lease['project_id'], reservation_id) created_fips.append(fip['floating_ip_address']) except Exception as e: for fip_address in created_fips: @@ -122,11 +132,8 @@ def _update_allocations(self, dates_before, dates_after, reservation_id, db_api.fip_allocation_create({ 'floatingip_id': fip_id, 'reservation_id': reservation_id}) - - for allocation in allocs_to_remove: - LOG.debug('Removing floating IP {} from reservation {}'.format( - allocation['floatingip_id'], reservation_id)) - db_api.fip_allocation_destroy(allocation['id']) + if allocs_to_remove: + self.deallocate(fip_reservation, allocs_to_remove) def _allocations_to_remove(self, dates_before, dates_after, allocs, amount): @@ -225,7 +232,7 @@ def update_reservation(self, reservation_id, values): "with an empty list") self._update_allocations(dates_before, dates_after, reservation_id, - reservation['status'], fip_reservation, + reservation['status'], fip_reservation, lease, values) updates = {} if 'amount' in values: @@ -244,28 +251,34 @@ def update_reservation(self, reservation_id, values): } db_api.required_fip_create(fip_address_values) - def on_start(self, resource_id): + def on_start(self, resource_id, lease=None): fip_reservation = db_api.fip_reservation_get(resource_id) allocations = db_api.fip_allocation_get_all_by_values( reservation_id=fip_reservation['reservation_id']) - ctx = context.current() fip_pool = neutron.FloatingIPPool(fip_reservation['network_id']) for alloc in allocations: fip = db_api.floatingip_get(alloc['floatingip_id']) fip_pool.create_reserved_floatingip( fip['subnet_id'], fip['floating_ip_address'], - ctx.project_id, fip_reservation['reservation_id']) + lease['project_id'], fip_reservation['reservation_id']) - def on_end(self, resource_id): + def on_end(self, resource_id, lease=None): fip_reservation = db_api.fip_reservation_get(resource_id) allocations = db_api.fip_allocation_get_all_by_values( reservation_id=fip_reservation['reservation_id']) - fip_pool = neutron.FloatingIPPool(fip_reservation['network_id']) + self.deallocate(fip_reservation, allocations) + + def deallocate(self, fip_reservation, allocations): + reservation = db_api.reservation_get(fip_reservation["reservation_id"]) + if reservation["status"] == status.reservation.ACTIVE: + fip_pool = neutron.FloatingIPPool(fip_reservation['network_id']) + for alloc in allocations: + fip = db_api.floatingip_get(alloc['floatingip_id']) + fip_pool.delete_reserved_floatingip(fip['floating_ip_address']) for alloc in allocations: - fip = db_api.floatingip_get(alloc['floatingip_id']) - fip_pool.delete_reserved_floatingip(fip['floating_ip_address']) + db_api.fip_allocation_destroy(alloc['id']) def allocation_candidates(self, values): self.check_params(values) @@ -348,9 +361,10 @@ def create_floatingip(self, values): try: subnet = pool.fetch_subnet(floatingip_address) except exceptions.BlazarException: - LOG.info("Floating IP %s in network %s can't be used " - "for Blazar's resource.", floatingip_address, network_id) - raise + with save_and_reraise_exception(): + LOG.info("Floating IP %s in network %s can't be used " + "for Blazar's resource.", floatingip_address, + network_id) floatingip_values = { 'floating_network_id': network_id, @@ -441,3 +455,110 @@ def query_fip_allocations(self, fips, detail=None, lease_id=None, if k != 'floatingip_ids'}) return fip_allocations + + +class FloatingIpMonitorPlugin(monitor.GeneralMonitorPlugin, neutron.NeutronClientWrapper): + """ + Monitors reserved floating IPs which fail to clean up. + Deletes any floating IPs without an associated active reservation. + """ + + def __new__(cls, *args, **kwargs): + return super(FloatingIpMonitorPlugin, cls).__new__(cls, *args, **kwargs) + + def filter_allocations(self, reservation, resource_ids): + return [alloc for alloc + in reservation["floatingip_allocations"] + if alloc["floating_ip_id"] in resource_ids] + + def get_reservations_by_resource_ids(self, resource_ids, + interval_begin, interval_end): + return db_utils.get_reservations_by_floatingip_ids(resource_ids, + interval_begin, + interval_end) + + def get_unreservable_resourses(self): + return db_api.unreservable_fip_get_all_by_queries([]) + + def get_notification_event_types(self): + """Get event types of notification messages to handle.""" + return ['service.update'] + + def notification_callback(self, event_type, payload): + return {} + + def set_reservable(self, resource, is_reservable): + db_api.floatingip_update(resource["id"], {"reservable": is_reservable}) + LOG.warn( + f"{resource['floating_ip_address']} " + f"{'recovered' if is_reservable else 'failed'} - setting reservable True" + ) + + def poll_resource_failures(self): + failed = [] + recovered = [] + dry_run = CONF[plugin.RESOURCE_TYPE].enable_polling_monitor_dry_run + fips = db_api.floatingip_list() + + def process_fip(fip): + fip_address = fip["floating_ip_address"] + fip_curr_reservation = db_utils.get_most_recent_reservation_info_by_fip_id(fip['id']) + if fip_curr_reservation and fip_curr_reservation['status'] == status.reservation.ACTIVE: + # This means that FIP works fine and no need to recover + LOG.debug(f"FIP {fip_address} is in active reservation {fip_curr_reservation['id']} - skipping") + return + fip_pool = neutron.FloatingIPPool(fip['floating_network_id']) + # check if the FIP is in neutron subnet allocation pools + try: + subnet = fip_pool.fetch_subnet(fip_address) + # if the FIP is in neutron allocation pools, make the FIP not reservable + except utils_exceptions.NeutronUsesFloatingIP as e: + LOG.warning(f"Failed: Floating ip {fip_address} is in use by subnet pools") + failed.append(fip) + return + # if the FIP is not found in any subnet, it can be reserved + except utils_exceptions.FloatingIPSubnetNotFound as e: + LOG.warning(f"Floating ip {fip_address} is not found in any subnet, recovering...") + return + # get the floating IP reservation ID from neutron + try: + fip_info_from_neutron = fip_pool.show_floatingip(fip["floating_ip_address"]) + except manager_ex.FloatingIPNotFound as e: + # If the FIP is not created in neutron then the IP is in 'pending' state and should be reservable + LOG.debug("Error getting Floating IP from neutron - {e}") + return + # get the reservation ID from neutron tags + fip_tags = fip_info_from_neutron.get('tags') + if fip_tags: + try: + reservation_tag = next((fip_tags[i + 1] for i, tag in enumerate(fip_tags) if tag == 'blazar'), None) + reservation_id_from_neutron = reservation_tag.replace("reservation:", "") + except Exception as e: + LOG.debug("Floating IP does not have 'blazar' tag in neutron - skipping", exc_info=e) + return + reservation = db_api.reservation_get(reservation_id_from_neutron) + if not reservation or reservation["status"] in [status.reservation.DELETED, status.reservation.ERROR]: + LOG.warning( + f"Found floating IP {fip['id']} stuck in " + f"deleted or errored reservation {reservation_id_from_neutron}. Recovering..." + ) + LOG.warning(f"Deleting {fip_address} from neutron.") + if not dry_run: + try: + fip_pool.delete_reserved_floatingip(fip_address) + recovered.append(fip) + except Exception as e: + LOG.warning(f"Cannot delete FIP {fip_address} from neutron - {e}") + failed.append(fip) + else: + LOG.debug(f"FIP {fip_address} in an active reservation {reservation['id']} - skipping") + else: + LOG.warning(f"{fip_address} does not have reservation tags in neutron") + + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = executor.map(process_fip, fips) + try: + results = list(futures) + except Exception as e: + LOG.exception('Skipping health check. %s', str(e)) + return failed, recovered diff --git a/blazar/plugins/instances/instance_plugin.py b/blazar/plugins/instances/instance_plugin.py index cece130fa..2c4dfd4c6 100644 --- a/blazar/plugins/instances/instance_plugin.py +++ b/blazar/plugins/instances/instance_plugin.py @@ -12,13 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -import collections import datetime -import retrying from novaclient import exceptions as nova_exceptions from oslo_config import cfg -from oslo_log import log as logging from oslo_utils import strutils from oslo_utils.strutils import bool_from_string @@ -34,6 +31,10 @@ from blazar.utils.openstack import nova from blazar.utils.openstack import placement from blazar.utils import plugins as plugins_utils +import collections +from oslo_log import log as logging +import retrying + CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -45,6 +46,8 @@ NONE_VALUES = ('None', 'none', None) QUERY_TYPE_ALLOCATION = 'allocation' +MONITOR_ARGS = {"resource_type": plugin.RESOURCE_TYPE} + class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): """Plugin for virtual instance resources.""" @@ -56,16 +59,15 @@ class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): } def __init__(self): - super(VirtualInstancePlugin, self).__init__( - username=CONF.os_admin_username, - password=CONF.os_admin_password, - user_domain_name=CONF.os_admin_user_domain_name, - project_name=CONF.os_admin_project_name, - project_domain_name=CONF.os_admin_project_domain_name) - + super(VirtualInstancePlugin, self).__init__() self.freepool_name = CONF.nova.aggregate_freepool_name - self.monitor = oshosts.host_plugin.PhysicalHostMonitorPlugin() + self.monitor = oshosts.host_plugin.PhysicalHostMonitorPlugin( + **MONITOR_ARGS + ) self.monitor.register_healing_handler(self.heal_reservations) + self.monitor.register_reallocater( + oshosts.host_plugin.PhysicalHostPlugin._reallocate + ) self.placement_client = placement.BlazarPlacementClient() def filter_hosts_by_reservation(self, hosts, start_date, end_date, @@ -311,7 +313,8 @@ def pickup_hosts(self, reservation_id, values): return {'added': added_host_ids, 'removed': removed_host_ids} - def _create_flavor(self, reservation_id, vcpus, memory, disk, group_id): + def _create_flavor(self, reservation_id, vcpus, memory, disk, + group_id=None): flavor_details = { 'flavorid': reservation_id, 'name': RESERVATION_PREFIX + ":" + reservation_id, @@ -327,37 +330,46 @@ def _create_flavor(self, reservation_id, vcpus, memory, disk, group_id): reservation_rc = "resources:CUSTOM_RESERVATION_" + rsv_id_rc_format extra_specs = { FLAVOR_EXTRA_SPEC: reservation_id, - "affinity_id": group_id, reservation_rc: "1" } + if group_id is not None: + extra_specs["affinity_id"] = group_id reserved_flavor.set_keys(extra_specs) return reserved_flavor - def _create_resources(self, inst_reservation): + def _create_resources(self, ctx, inst_reservation): reservation_id = inst_reservation['reservation_id'] - ctx = context.current() user_client = nova.NovaClientWrapper() - reserved_group = user_client.nova.server_groups.create( - RESERVATION_PREFIX + ':' + reservation_id, - 'affinity' if inst_reservation['affinity'] else 'anti-affinity' - ) - - reserved_flavor = self._create_flavor(reservation_id, - inst_reservation['vcpus'], - inst_reservation['memory_mb'], - inst_reservation['disk_gb'], - reserved_group.id) + flavor_args = { + 'reservation_id': reservation_id, + 'vcpus': inst_reservation['vcpus'], + 'memory': inst_reservation['memory_mb'], + 'disk': inst_reservation['disk_gb'] + } - pool = nova.ReservationPool() pool_metadata = { RESERVATION_PREFIX: reservation_id, 'filter_tenant_id': ctx.project_id, - 'affinity_id': reserved_group.id } - agg = pool.create(name=reservation_id, metadata=pool_metadata) + + if inst_reservation['affinity'] is not None: + reserved_group = user_client.nova.server_groups.create( + RESERVATION_PREFIX + ':' + reservation_id, + 'affinity' if inst_reservation['affinity'] else 'anti-affinity' + ) + flavor_args['group_id'] = reserved_group.id + pool_metadata['affinity_id'] = reserved_group.id + else: + reserved_group = None + + reserved_flavor = self._create_flavor(**flavor_args) + + pool = nova.ReservationPool() + agg = pool.create(name=reservation_id, project_id=ctx.project_id, + metadata=pool_metadata) self.placement_client.create_reservation_class(reservation_id) @@ -372,8 +384,10 @@ def check_and_delete_resource(client, id): reservation_id = instance_reservation['reservation_id'] - check_and_delete_resource(self.nova.nova.server_groups, - instance_reservation['server_group_id']) + server_group_id = instance_reservation['server_group_id'] + if server_group_id: + check_and_delete_resource(self.nova.nova.server_groups, + server_group_id) check_and_delete_resource(self.nova.nova.flavors, reservation_id) check_and_delete_resource(nova.ReservationPool(), reservation_id) @@ -448,6 +462,7 @@ def _validate_reservation_params(self, values): param='affinity (must be a bool value or None)') def reserve_resource(self, reservation_id, values): + ctx = context.current() self._check_missing_reservation_params(values) self._validate_reservation_params(values) @@ -470,16 +485,18 @@ def reserve_resource(self, reservation_id, values): 'reservation_id': reservation_id}) try: - flavor, group, pool = self._create_resources(instance_reservation) + flavor, group, pool = self._create_resources( + ctx, instance_reservation) except nova_exceptions.ClientException: LOG.exception("Failed to create Nova resources " "for reservation %s", reservation_id) self.cleanup_resources(instance_reservation) raise mgr_exceptions.NovaClientError() + server_group_id = group.id if group is not None else None db_api.instance_reservation_update(instance_reservation['id'], {'flavor_id': flavor.id, - 'server_group_id': group.id, + 'server_group_id': server_group_id, 'aggregate_id': pool.id}) return instance_reservation['id'] @@ -562,25 +579,22 @@ def update_reservation(self, reservation_id, new_values): self.update_host_allocations(changed_hosts['added'], changed_hosts['removed'], reservation_id) + self.update_resources(reservation_id) - try: - self.update_resources(reservation_id) - except mgr_exceptions.NovaClientError: - raise - - def on_start(self, resource_id): - ctx = context.current() + def on_start(self, resource_id, lease=None): instance_reservation = db_api.instance_reservation_get(resource_id) reservation_id = instance_reservation['reservation_id'] + # TODO(johngarbutt): create flavor after updating placement? + # else we will race with automation looking for the flavor here try: self.nova.flavor_access.add_tenant_access(reservation_id, - ctx.project_id) + lease['project_id']) except nova_exceptions.ClientException: LOG.info('Failed to associate flavor %(reservation_id)s ' 'to project %(project_id)s', {'reservation_id': reservation_id, - 'project_id': ctx.project_id}) + 'project_id': lease['project_id']}) raise mgr_exceptions.EventError() pool = nova.ReservationPool() @@ -600,14 +614,13 @@ def on_start(self, resource_id): self.placement_client.update_reservation_inventory( host['hypervisor_hostname'], reservation_id, num) - def on_end(self, resource_id): + def on_end(self, resource_id, lease=None): instance_reservation = db_api.instance_reservation_get(resource_id) reservation_id = instance_reservation['reservation_id'] - ctx = context.current() try: self.nova.flavor_access.remove_tenant_access( - reservation_id, ctx.project_id) + reservation_id, lease['project_id']) except nova_exceptions.NotFound: pass @@ -803,9 +816,9 @@ def _get_extra_capabilities(self, host_id): extra_capabilities = {} raw_extra_capabilities = ( db_api.host_extra_capability_get_all_per_host(host_id)) - for capability in raw_extra_capabilities: - key = capability['capability_name'] - extra_capabilities[key] = capability['capability_value'] + for capability, capability_name in raw_extra_capabilities: + key = capability_name + extra_capabilities[key] = capability.capability_value return extra_capabilities def get(self, host_id): diff --git a/blazar/plugins/monitor.py b/blazar/plugins/monitor.py new file mode 100644 index 000000000..3e99db937 --- /dev/null +++ b/blazar/plugins/monitor.py @@ -0,0 +1,234 @@ +# -*- coding: utf-8 -*- +# +# Author: François Rossigneux +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +from oslo_config import cfg + +import abc +from blazar.manager import exceptions as manager_ex +from blazar.plugins import base +from blazar import status +from oslo_log import log as logging + + +monitor_opts = [ + cfg.BoolOpt('enable_notification_monitor', + default=False, + help='Enable notification-based resource monitoring. ' + 'If it is enabled, the blazar-manager monitors states of ' + 'resource by subscribing to notifications of ' + 'the corresponding service.'), + cfg.ListOpt('notification_topics', + default=['notifications', 'versioned_notifications'], + help='Notification topics to subscribe to.'), + cfg.BoolOpt('enable_polling_monitor', + default=False, + help='Enable polling-based resource monitoring. ' + 'If it is enabled, the blazar-manager monitors states ' + 'of resource by polling the service API.'), + cfg.BoolOpt('enable_polling_monitor_dry_run', + default=False, + help='If it is enabled, resource monitor plugin only warns ' + 'about a resource failure but not heal/fix it.'), + cfg.IntOpt('polling_interval', + default=60, + min=1, + help='Interval (seconds) of polling for health checking.'), + cfg.IntOpt('healing_interval', + default=60, + min=0, + help='Interval (minutes) of reservation healing. ' + 'If 0 is specified, the interval is infinite and all the ' + 'reservations in the future is healed at one time.'), +] + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +class GeneralMonitorPlugin(base.BaseMonitorPlugin, metaclass=abc.ABCMeta): + """Monitor plugin for resource.""" + + # Singleton design pattern + _instance = None + + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = \ + super(GeneralMonitorPlugin, cls).__new__(cls) + cls._instance.resource_type = kwargs.get("resource_type") + super(GeneralMonitorPlugin, cls._instance).__init__() + return cls._instance + + def __init__(self, *args, **kwargs): + """Do nothing. + + This class uses the Singleton design pattern and an instance of this + class is generated and initialized in __new__(). + """ + pass + + def register_healing_handler(self, handler): + self.heal_reservations = handler + + def register_reallocater(self, reallocator): + self._reallocate = reallocator + + @abc.abstractmethod + def filter_allocations(self, reservation, resource_ids): + """Filter allocations of a reservation by resource ids + + :param reservation: a reservation dict + :param resource_ids: a list of resource ids + :return: a list of allocations that contain resources + """ + pass + + @abc.abstractmethod + def get_reservations_by_resource_ids(self, resource_ids, + interval_begin, interval_end): + """Get reservations by resource ids. + + :param resource ids: a list of resource ids. + :param interval_begin: start date of the searching period. + :param interval_end: end date of the searching period. + :return: a list of reservation dict + """ + pass + + @abc.abstractmethod + def get_unreservable_resourses(self): + """Get all unreservable resources + + """ + pass + + @abc.abstractmethod + def poll_resource_failures(self): + """Get a list of failed resources and recovered resources + + """ + pass + + @abc.abstractmethod + def set_reservable(self, resource_id, is_reservable): + """Set resource as reservable or not reservable + + """ + pass + + def heal_reservations(self, failed_resources, interval_begin, + interval_end): + """Heal reservations which suffer from resource failures. + + :param failed_resources: a list of failed resources. + :param interval_begin: start date of the period to heal. + :param interval_end: end date of the period to heal. + :return: a dictionary of {reservation id: flags to update} + e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': + {'missing_resources': True}} + """ + reservation_flags = {} + + resource_ids = [h['id'] for h in failed_resources] + reservations = self.get_reservations_by_resource_ids(resource_ids, + interval_begin, + interval_end) + + for reservation in reservations: + active_reservation = reservation['status'] == status.reservation.ACTIVE + if reservation['resource_type'] != self.resource_type or active_reservation: + continue + + reservation_id = reservation["id"] + + for allocation in self.filter_allocations(reservation, + resource_ids): + try: + if not self._reallocate(allocation): + if reservation_id not in reservation_flags: + reservation_flags[reservation_id] = {} + reservation_flags[reservation_id].update( + {'missing_resources': True}) + except manager_ex.ResourceBusy: + LOG.info( + "Cannot heal reservation %s, found servers", + reservation["id"] + ) + + return reservation_flags + + def is_notification_enabled(self): + """Check if the notification monitor is enabled.""" + return CONF[self.resource_type].enable_notification_monitor + + def get_notification_topics(self): + """Get topics of notification to subscribe to.""" + return CONF[self.resource_type].notification_topics + + def is_polling_enabled(self): + """Check if the polling monitor is enabled.""" + return CONF[self.resource_type].enable_polling_monitor + + def get_polling_interval(self): + """Get interval of polling.""" + return CONF[self.resource_type].polling_interval + + def poll(self): + """Detect and handle resource failures. + + :return: a dictionary of {reservation id: flags to update} + e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': + {'missing_resources': True}} + """ + LOG.trace('Poll...') + + failed_resources, recovered_resources = self.poll_resource_failures() + if failed_resources: + for resource in failed_resources: + self.set_reservable(resource, False) + if recovered_resources: + for resource in recovered_resources: + self.set_reservable(resource, True) + + return self.heal() + + def get_healing_interval(self): + """Get interval of reservation healing in minutes.""" + return CONF[self.resource_type].healing_interval + + def heal(self): + """Heal suffering reservations in the next healing interval. + + :return: a dictionary of {reservation id: flags to update} + """ + reservation_flags = {} + resources = self.get_unreservable_resourses() + + interval_begin = datetime.datetime.utcnow() + interval = self.get_healing_interval() + if interval == 0: + interval_end = datetime.date.max + else: + interval_end = interval_begin + datetime.timedelta( + minutes=interval) + + reservation_flags.update(self.heal_reservations(resources, + interval_begin, + interval_end)) + + return reservation_flags diff --git a/blazar/plugins/networks/__init__.py b/blazar/plugins/networks/__init__.py new file mode 100644 index 000000000..882305bd4 --- /dev/null +++ b/blazar/plugins/networks/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2018 StackHPC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RESOURCE_TYPE = u'network' diff --git a/blazar/plugins/networks/network_plugin.py b/blazar/plugins/networks/network_plugin.py new file mode 100644 index 000000000..bd3e48bd3 --- /dev/null +++ b/blazar/plugins/networks/network_plugin.py @@ -0,0 +1,953 @@ +# -*- coding: utf-8 -*- +# +# Author: Pierre Riteau +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import concurrent.futures +from random import shuffle +import ipaddress + +from keystoneauth1 import exceptions as keystone_excptions +from neutronclient.common import exceptions as neutron_ex +from oslo_config import cfg +from oslo_log import log as logging +from stevedore import named + +from blazar.db import api as db_api +from blazar.db import exceptions as db_ex +from blazar.db import utils as db_utils +from blazar.manager import exceptions as manager_ex +from blazar.plugins import base +from blazar.plugins import monitor +from blazar.plugins import networks as plugin +from blazar import status +from blazar.utils.openstack import ironic +from blazar.utils.openstack import neutron +from blazar.utils import plugins as plugins_utils + + +plugin_opts = [ + cfg.StrOpt('default_resource_properties', + default='', + help='Default resource_properties when creating a lease of ' + 'this type.'), + cfg.BoolOpt('display_default_resource_properties', + default=True, + help='Display default resource_properties if allocation fails ' + 'due to not enough resources'), + cfg.BoolOpt('retry_allocation_without_defaults', + default=True, + help='Whether an allocation should be retried on failure ' + 'without the default properties'), + cfg.ListOpt('usage_type_plugins', + default=[], + help='All plugins to use'), +] + +plugin_opts.extend(monitor.monitor_opts) + +CONF = cfg.CONF +CONF.register_opts(plugin_opts, group=plugin.RESOURCE_TYPE) +LOG = logging.getLogger(__name__) + +before_end_options = ['', 'snapshot', 'default', 'email'] + +QUERY_TYPE_ALLOCATION = 'allocation' +MONITOR_ARGS = {"resource_type": plugin.RESOURCE_TYPE} + +def _get_plugins(): + """Return dict of resource-plugin class pairs.""" + plugins = {} + + extension_manager = named.NamedExtensionManager( + namespace='blazar.network.usage.type.plugins', + names=CONF.network.usage_type_plugins, + invoke_on_load=False + ) + + for ext in extension_manager.extensions: + try: + plugin_obj = ext.plugin() + except Exception as e: + LOG.warning("Could not load {0} plugin " + "for resource type {1} '{2}'".format( + ext.name, ext.plugin.usage_type, e)) + else: + if plugin_obj.usage_type in plugins: + msg = ("You have provided several plugins for " + "one usage type in configuration file. " + "Please set one plugin per usage type.") + raise manager_ex.PluginConfigurationError(error=msg) + + plugins[plugin_obj.usage_type] = plugin_obj + return plugins + + +class NetworkPlugin(base.BasePlugin): + """Plugin for network resource.""" + resource_type = plugin.RESOURCE_TYPE + title = 'Network Plugin' + description = 'This plugin creates and deletes networks.' + query_options = { + QUERY_TYPE_ALLOCATION: ['lease_id', 'reservation_id'] + } + + def __init__(self): + super(NetworkPlugin, self).__init__() + self.plugins = _get_plugins() + self.periodic_tasks = [] + self.monitor = NetworkMonitorPlugin(**MONITOR_ARGS) + for p in self.plugins.values(): + if hasattr(p, "periodic_tasks"): + self.periodic_tasks.extend(p.periodic_tasks) + + def filter_networks_by_reservation(self, networks, start_date, end_date): + free = [] + non_free = [] + + for network in networks: + reservations = db_utils.get_reservations_by_network_id( + network['id'], start_date, end_date) + + if reservations == []: + free.append({'network': network, 'reservations': None}) + elif [r for r in reservations + if r['resource_type'] == self.resource_type]: + non_free.append( + {'network': network, 'reservations': reservations}) + + return free, non_free + + def reserve_resource(self, reservation_id, values): + """Create reservation.""" + network_ids = self.allocation_candidates(values) + + network_id = next(iter( + self._matching_networks( + values['network_properties'], + values['resource_properties'], + values['start_date'], + values['end_date'], + ) + ), None) + + if not network_id: + raise manager_ex.NotEnoughNetworksAvailable() + + network_id = network_ids[0] + network_rsrv_values = { + 'reservation_id': reservation_id, + 'network_properties': values['network_properties'], + 'resource_properties': values['resource_properties'], + 'status': 'pending', + 'before_end': values['before_end'], + 'network_name': values['network_name'], + 'network_description': values.get('network_description'), + } + network_reservation = db_api.network_reservation_create( + network_rsrv_values) + db_api.network_allocation_create({ + 'network_id': network_id, 'reservation_id': reservation_id}) + return network_reservation['id'] + + def update_reservation(self, reservation_id, values): + """Update reservation.""" + reservation = db_api.reservation_get(reservation_id) + lease = db_api.lease_get(reservation['lease_id']) + + if (not [x for x in values.keys() if x in ['network_properties', + 'resource_properties']] + and values['start_date'] >= lease['start_date'] + and values['end_date'] <= lease['end_date']): + # Nothing to update + return + + dates_before = {'start_date': lease['start_date'], + 'end_date': lease['end_date']} + dates_after = {'start_date': values['start_date'], + 'end_date': values['end_date']} + network_reservation = db_api.network_reservation_get( + reservation['resource_id']) + self._update_allocations(dates_before, dates_after, reservation_id, + reservation['status'], network_reservation, + values) + + updates = {} + if 'network_properties' in values: + updates['network_properties'] = values.get( + 'network_properties') + if 'resource_properties' in values: + updates['resource_properties'] = values.get( + 'resource_properties') + if updates: + db_api.network_reservation_update( + network_reservation['id'], updates) + + def on_start(self, resource_id, lease=None): + """Creates a Neutron network using the allocated segment.""" + network_reservation = db_api.network_reservation_get(resource_id) + network_name = network_reservation['network_name'] + network_description = network_reservation['network_description'] + reservation_id = network_reservation['reservation_id'] + + # We need the lease to get to the project_id + reservation = db_api.reservation_get(reservation_id) + lease = db_api.lease_get(reservation['lease_id']) + + for allocation in db_api.network_allocation_get_all_by_values( + reservation_id=reservation_id): + network_segment = self.get_network(allocation['network_id']) + network_type = network_segment['network_type'] + physical_network = network_segment['physical_network'] + segment_id = network_segment['segment_id'] + neutron_client = neutron.BlazarNeutronClient() + network_body = { + "network": { + "name": network_name, + "provider:network_type": network_type, + "provider:segmentation_id": segment_id, + "project_id": lease['project_id'] + } + } + + if physical_network: + network_body['network']['provider:physical_network'] = ( + physical_network) + + if network_description: + network_body['network']['description'] = network_description + + try: + network = neutron_client.create_network(body=network_body) + network_dict = network['network'] + network_id = network_dict['id'] + db_api.network_reservation_update(network_reservation['id'], + {'network_id': network_id}) + except Exception as e: + LOG.error("create_network failed: %s", e) + raise manager_ex.NetworkCreationFailed(name=network_name, + id=reservation_id, + msg=str(e)) + + # extra steps for the usage type + usage_type = network_segment.get("usage_type", None) + if usage_type: + try: + self.plugins[usage_type].perform_extra_on_start_steps( + network_segment, network + ) + except Exception as e: + LOG.error("Extra on start steps failed: %s", e) + raise manager_ex.NetworkExtraOnStartFailed( + id=reservation_id, + msg=str(e) + ) + + def delete_subnet(self, neutron_client, subnet_id): + neutron_client.delete_subnet(subnet_id) + + def delete_router(self, neutron_client, router_id): + neutron_client.remove_gateway_router(router_id) + neutron_client.delete_router(router_id) + + def delete_neutron_network(self, network_id, reservation_id, + trust_id=None): + if network_id is None: + LOG.info("Not deleting network for reservation %s as no network " + "ID was recorded", + reservation_id) + return + + neutron_client = neutron.BlazarNeutronClient(trust_id=trust_id) + ironic_client = None + try: + ironic_client = ironic.BlazarIronicClient() + except keystone_excptions.catalog.EndpointNotFound: + LOG.exception("The endpoint for Ironic not found") + + try: + neutron_client.show_network(network_id) + except neutron_ex.NetworkNotFoundClient: + LOG.info("Not deleting network %s as it could not be found", + network_id) + return + + try: + ports = neutron_client.list_ports(network_id=network_id) + instance_ports = neutron_client.list_ports( + device_owner='compute:nova', network_id=network_id) + for instance_port in instance_ports['ports']: + neutron_client.delete_port(instance_port['id']) + + subnets = neutron_client.list_subnets(network_id=network_id) + subnet_ids = [s['id'] for s in subnets['subnets']] + + router_ids = [port['device_id'] for port in ports['ports'] if + port['device_owner'] == 'network:router_interface'] + for router_id in router_ids: + router_ports = neutron_client.list_ports(device_id=router_id) + + # Remove static routes + # (zhenz) comment out removing static routes, as it breaks + # current design with storage vlan + # neutron_client.update_router( + # router_id, body={'router': {'routes': []}}) + + # Remove subnets + subnets = set() + for router_port in router_ports['ports']: + if router_port['device_owner'] != 'network:router_gateway': + for fixed_ip in router_port['fixed_ips']: + sid = fixed_ip['subnet_id'] + if sid in subnet_ids: + subnets.update([sid]) + for subnet_id in subnets: + body = {} + body['subnet_id'] = subnet_id + neutron_client.remove_interface_router(router_id, + body=body) + + # Delete external gateway and router + # (zhenz) comment out deleting router, as it breaks + # current design with storage vlan + # self.delete_router(neutron_client, router_id) + + for subnet_id in subnet_ids: + self.delete_subnet(neutron_client, subnet_id) + + neutron_client.delete_network(network_id) + except Exception: + LOG.exception("Failed to delete network %s", network_id) + raise manager_ex.NetworkDeletionFailed( + network_id=network_id, reservation_id=reservation_id) + + def on_end(self, resource_id, lease=None): + """Delete the Neutron network created when the lease started. + + We first need to delete associated Neutron resources. + """ + + network_reservation = db_api.network_reservation_get(resource_id) + reservation_id = network_reservation['reservation_id'] + + db_api.network_reservation_update(network_reservation['id'], + {'status': 'completed'}) + allocations = db_api.network_allocation_get_all_by_values( + reservation_id=reservation_id) + for allocation in allocations: + db_api.network_allocation_destroy(allocation['id']) + network_id = network_reservation['network_id'] + + # The call to delete must be done without trust_id so the admin role is + # used + self.delete_neutron_network(network_id, reservation_id) + + def _get_extra_capabilities(self, network_id): + extra_capabilities = {} + raw_extra_capabilities = ( + db_api.network_extra_capability_get_all_per_network(network_id)) + for capability, capability_name in raw_extra_capabilities: + key = capability_name + extra_capabilities[key] = capability.capability_value + return extra_capabilities + + def get(self, network_id): + return self.get_network(network_id) + + def get_network(self, network_id): + network = db_api.network_get(network_id) + extra_capabilities = self._get_extra_capabilities(network_id) + if network is not None and extra_capabilities: + res = network.copy() + res.update(extra_capabilities) + return res + else: + return network + + def list_networks(self): + raw_network_list = db_api.network_list() + network_list = [] + for network in raw_network_list: + network_list.append(self.get_network(network['id'])) + return network_list + + def validate_network_param(self, values): + marshall_attributes = set(['network_type', 'physical_network', + 'segment_id']) + missing_attr = marshall_attributes - set(values.keys()) + if missing_attr: + raise manager_ex.MissingParameter(param=','.join(missing_attr)) + + def create_network(self, values): + if 'trust_id' in values: + del values['trust_id'] + + # TODO(priteau): check that no network is using this segmentation_id + self.validate_network_param(values) + network_type = values.get('network_type') + physical_network = values.get('physical_network') + segment_id = values.get('segment_id') + if network_type != 'vlan' and network_type != 'vxlan': + raise manager_ex.MalformedParameter(param=network_type) + + # Check that VLAN segmentation ID is valid + try: + segment_id = int(segment_id) + except ValueError: + raise manager_ex.MalformedParameter(param=segment_id) + if segment_id < 1 or segment_id > 4094: + raise manager_ex.MalformedParameter(param=segment_id) + + network_values = { + 'network_type': network_type, + 'physical_network': physical_network, + 'segment_id': segment_id + } + network = db_api.network_create(network_values) + + to_store = set(values.keys()) - set(network.keys()) + extra_capabilities_keys = to_store + extra_capabilities = dict( + (key, values[key]) for key in extra_capabilities_keys + ) + if any([len(key) > 64 for key in extra_capabilities_keys]): + raise manager_ex.ExtraCapabilityTooLong() + + cantaddextracapability = [] + for key in extra_capabilities: + values = {'network_id': network['id'], + 'capability_name': key, + 'capability_value': extra_capabilities[key], + } + try: + db_api.network_extra_capability_create(values) + except db_ex.BlazarDBException: + cantaddextracapability.append(key) + if cantaddextracapability: + raise manager_ex.CantAddExtraCapability( + keys=cantaddextracapability, + host=network['id']) + return self.get_network(network['id']) + + def is_updatable_extra_capability(self, capability, capability_name): + reservations = db_utils.get_reservations_by_network_id( + capability['network_id'], datetime.datetime.utcnow(), + datetime.date.max) + + for r in reservations: + plugin_reservation = db_utils.get_plugin_reservation( + r['resource_type'], r['resource_id']) + + requirements_queries = plugins_utils.convert_requirements( + plugin_reservation['resource_properties']) + + # TODO(masahito): If all the reservations using the + # extra_capability can be re-allocated it's okay to update + # the extra_capability. + for requirement in requirements_queries: + # A requirement is of the form "key op value" as string + if requirement.split(" ")[0] == capability_name: + return False + return True + + def update_network(self, network_id, values): + # nothing to update + if not values: + return self.get_network(network_id) + + network = db_api.network_get(network_id) + if not network: + raise manager_ex.NetworkNotFound(network=network_id) + + updatable = ['network_type', 'physical_network', 'segment_id'] + + network_type = values.get('network_type') + if network_type == 'vlan': + segment_id = values.get('segment_id') + if segment_id is not None: + try: + segment_id = int(segment_id) + except ValueError: + raise manager_ex.MalformedParameter(param=segment_id) + if segment_id < 1 or segment_id > 4094: + raise manager_ex.MalformedParameter(param=segment_id) + + new_values = {} + for key in updatable: + if key in values and values[key] is not None: + new_values[key] = values[key] + db_api.network_update(network_id, new_values) + + cant_update_extra_capability = [] + cant_delete_extra_capability = [] + previous_capabilities = self._get_extra_capabilities(network_id) + updated_keys = set(values.keys()) & set(previous_capabilities.keys()) + new_keys = set(values.keys()) - set(previous_capabilities.keys()) + + for key in updated_keys: + raw_capability, cap_name = next(iter( + db_api.network_extra_capability_get_all_per_name( + network_id, key))) + capability = { + 'capability_name': key, + 'capability_value': values[key], + } + if self.is_updatable_extra_capability(raw_capability, cap_name): + if values[key] is not None: + try: + db_api.network_extra_capability_update( + raw_capability['id'], capability) + except (db_ex.BlazarDBException, RuntimeError): + cant_update_extra_capability.append(cap_name) + else: + try: + db_api.network_extra_capability_destroy( + raw_capability['id']) + except db_ex.BlazarDBException: + cant_delete_extra_capability.append(cap_name) + else: + LOG.info("Capability %s can't be updated because " + "existing reservations require it.", + cap_name) + cant_update_extra_capability.append(cap_name) + + for key in new_keys: + new_capability = { + 'network_id': network_id, + 'capability_name': key, + 'capability_value': values[key], + } + try: + db_api.network_extra_capability_create(new_capability) + except (db_ex.BlazarDBException, RuntimeError): + cant_update_extra_capability.append(key) + + if cant_update_extra_capability: + raise manager_ex.CantAddExtraCapability( + network=network_id, keys=cant_update_extra_capability) + + if cant_delete_extra_capability: + raise manager_ex.ExtraCapabilityNotFound( + resource=network_id, keys=cant_delete_extra_capability) + + LOG.info('Extra capabilities on network %s updated with %s', + network_id, values) + + def delete_network(self, network_id): + network = db_api.network_get(network_id) + if not network: + raise manager_ex.NetworkNotFound(network=network_id) + + if db_api.network_allocation_get_all_by_values( + network_id=network_id): + raise manager_ex.CantDeleteNetwork( + network=network_id, + msg='The network is reserved.' + ) + + try: + db_api.network_destroy(network_id) + except db_ex.BlazarDBException as e: + # Nothing so bad, but we need to alert admins + # they have to rerun + raise manager_ex.CantDeleteNetwork(network=network_id, msg=str(e)) + + def list_allocations(self, query, detail=False): + network_id_list = [n['id'] for n in db_api.network_list()] + options = self.get_query_options(query, QUERY_TYPE_ALLOCATION) + options['detail'] = detail + + network_allocations = self.query_network_allocations(network_id_list, + **options) + self.add_extra_allocation_info(network_allocations) + self.add_allocation_cleaning_time(network_allocations, CONF.cleaning_time) + return [{"resource_id": network, "reservations": allocs} + for network, allocs in network_allocations.items()] + + def get_allocations(self, network_id, query): + options = self.get_query_options(query, QUERY_TYPE_ALLOCATION) + network_allocations = self.query_network_allocations([network_id], + **options) + allocs = network_allocations.get(network_id, []) + return {"resource_id": network_id, "reservations": allocs} + + def query_allocations(self, networks, lease_id=None, reservation_id=None): + return self.query_network_allocations(networks, lease_id=lease_id, + reservation_id=reservation_id) + + def query_network_allocations(self, networks, lease_id=None, + reservation_id=None, detail=False): + """Return dict of network and its allocations + + The list element forms + { + 'network-id': [ + { + 'lease_id': lease_id, + 'id': reservation_id, + 'start_date': lease_start_date, + 'end_date': lease_end_date + } + ] + }. + """ + start = datetime.datetime.utcnow() + end = datetime.date.max + + reservations = db_utils.get_reservation_allocations_by_network_ids( + networks, start, end, lease_id, reservation_id) + network_allocations = {n: [] for n in networks} + + for reservation in reservations: + if not detail: + del reservation['project_id'] + del reservation['lease_name'] + del reservation['status'] + + for network_id in reservation['network_ids']: + if network_id in network_allocations.keys(): + network_allocations[network_id].append({ + k: v for k, v in reservation.items() + if k != 'network_ids'}) + + return network_allocations + + def update_default_parameters(self, values): + self.add_default_resource_properties(values) + + def allocation_candidates(self, values): + self._check_params(values) + + network_ids = self._matching_networks( + values['network_properties'], + values['resource_properties'], + values['start_date'], + values['end_date'] + ) + + if len(network_ids) < 1: + raise manager_ex.NotEnoughNetworksAvailable() + + return network_ids[:1] + + def _matching_networks(self, network_properties, resource_properties, + start_date, end_date): + """Return the matching networks (preferably not allocated)""" + allocated_network_ids = [] + not_allocated_network_ids = [] + filter_array = [] + start_date_with_margin = start_date - datetime.timedelta( + minutes=CONF.cleaning_time) + end_date_with_margin = end_date + datetime.timedelta( + minutes=CONF.cleaning_time) + + # TODO(frossigneux) support "or" operator + if network_properties: + filter_array = plugins_utils.convert_requirements( + network_properties) + if resource_properties: + filter_array += plugins_utils.convert_requirements( + resource_properties) + for network in db_api.network_get_all_by_queries( + filter_array): + if not db_api.network_allocation_get_all_by_values( + network_id=network['id']): + not_allocated_network_ids.append(network['id']) + elif db_utils.get_free_periods( + network['id'], + start_date_with_margin, + end_date_with_margin, + end_date_with_margin - start_date_with_margin, + resource_type='network' + ) == [ + (start_date_with_margin, end_date_with_margin), + ]: + allocated_network_ids.append(network['id']) + + if len(not_allocated_network_ids): + shuffle(not_allocated_network_ids) + return not_allocated_network_ids + + all_network_ids = allocated_network_ids + not_allocated_network_ids + if len(all_network_ids): + shuffle(all_network_ids) + return all_network_ids + else: + return [] + + def _check_params(self, values): + required_values = ['network_name', 'network_properties', + 'resource_properties'] + for value in required_values: + if value not in values: + raise manager_ex.MissingParameter(param=value) + + if 'network_description' in values: + values['network_description'] = str(values['network_description']) + + if 'before_end' not in values: + values['before_end'] = 'default' + if values['before_end'] not in before_end_options: + raise manager_ex.MalformedParameter(param='before_end') + + def _update_allocations(self, dates_before, dates_after, reservation_id, + reservation_status, network_reservation, values): + network_properties = values.get( + 'network_properties', + network_reservation['network_properties']) + resource_properties = values.get( + 'resource_properties', + network_reservation['resource_properties']) + alloc = next(iter(db_api.network_allocation_get_all_by_values( + reservation_id=reservation_id))) + change_allocation = self._allocation_needs_change( + dates_before, dates_after, network_properties, + resource_properties, alloc) + + if not change_allocation: + return + + if (reservation_status == status.reservation.ACTIVE): + # To support removing a reserved network from an active reservation + # we need support for deallocating a network, which requires + # deleting the network and its subnets, removing ports etc, which + # is not currenty implemented. + # TODO(jasonanderson): should have a more descriptive error here. + raise manager_ex.NotEnoughNetworksAvailable(( + "Operation requires removing a network from its reservation, " + "which is not supported.")) + + new_network_id = next(iter( + self._matching_networks( + network_properties, resource_properties, + dates_after['start_date'], dates_after['end_date']) + ), None) + + if not new_network_id: + raise manager_ex.NotEnoughNetworksAvailable() + + LOG.debug('Adding network %s to reservation %s', + new_network_id, reservation_id) + db_api.network_allocation_create({ + 'network_id': new_network_id, + 'reservation_id': reservation_id, + }) + + LOG.debug('Removing network %s from reservation %s', + alloc['network_id'], reservation_id) + db_api.network_allocation_destroy(alloc['id']) + + def _allocation_needs_change(self, dates_before, dates_after, + network_properties, resource_properties, + alloc): + """Determines if a network allocation needs to be exchanged.""" + requested_network_ids = [network['id'] for network in + self._filter_networks_by_properties( + network_properties, resource_properties)] + + if alloc['network_id'] not in requested_network_ids: + return True + + starting_earlier = ( + dates_after['start_date'] < dates_before['start_date']) + ending_later = dates_after['end_date'] > dates_before['end_date'] + + if (starting_earlier or ending_later): + max_start = max(dates_before['start_date'], + dates_after['start_date']) + min_end = min(dates_before['end_date'], + dates_after['end_date']) + + reserved_periods = db_utils.get_reserved_periods( + alloc['network_id'], + dates_after['start_date'], + dates_after['end_date'], + datetime.timedelta(minutes=CONF.cleaning_time), + resource_type='network') + reserved_by_others = [ + p for p in reserved_periods + if not (p[0] == max_start and p[1] == min_end) + ] + return len(reserved_by_others) > 0 + + return False + + def _filter_networks_by_properties(self, network_properties, + resource_properties): + filter = [] + if network_properties: + filter += plugins_utils.convert_requirements(network_properties) + if resource_properties: + filter += plugins_utils.convert_requirements(resource_properties) + if filter: + return db_api.network_get_all_by_queries(filter) + else: + return db_api.network_list() + + +def remove_subnet_route_from_router(router_id, subnet_id, dry_run): + """Removes the route in the router pointing to the subnet + + Args: + router_id (str): The router in which routes need to be checked and removed + subnet_id (str): the subnet to which routes should be removed + """ + neutron_client = neutron.BlazarNeutronClient() + def is_route_pointing_to_subnet(route): + if 'gateway' in route: + next_hop_ip = ipaddress.ip_address(route['gateway']) + elif 'nexthop' in route: + next_hop_ip = ipaddress.ip_address(route['nexthop']) + subnet_cidr = ipaddress.ip_network(subnet['cidr']) + return next_hop_ip in subnet_cidr + try: + subnet = neutron_client.show_subnet(subnet_id).get('subnet', {}) + except neutron_ex.NotFound as e: + LOG.exception(f"Could not get subnet {subnet_id} - {e}") + raise e + try: + router = neutron_client.show_router(router_id).get('router', {}) + except neutron_ex.NotFound as e: + LOG.exception(f"Could not get router {router_id} - {e}") + raise e + # Check if any of the routes in router are pointing to subnet + routes = router.get('routes', []) + subnet_routes = [route for route in routes if is_route_pointing_to_subnet(route)] + for route in subnet_routes: + LOG.warning( + f"Removing the route {route} from router {router['id']} " + f"as it is pointing to {subnet['id']}" + ) + # remove this subnet route + if not dry_run: + neutron_client.remove_extra_routes_from_router( + router_id, + {'router': {'routes': [route]}} + ) + + +class NetworkMonitorPlugin(monitor.GeneralMonitorPlugin, neutron.NeutronClientWrapper): + """ + Monitor plugin for network resources + """ + + def __new__(cls, *args, **kwargs): + return super(NetworkMonitorPlugin, cls).__new__(cls, *args, **kwargs) + + def filter_allocations(self, reservation, resource_ids): + return [alloc for alloc + in reservation["network_allocations"] + if alloc["network_id"] in resource_ids] + + def get_reservations_by_resource_ids(self, resource_ids, + interval_begin, interval_end): + return db_utils.get_reservations_by_floatingip_ids(resource_ids, + interval_begin, + interval_end) + + def get_notification_event_types(self): + """Get event types of notification messages to handle.""" + return ['service.update'] + + def notification_callback(self, event_type, payload): + return {} + + def set_reservable(self, resource, is_reservable): + # network segments does not have reservable flag + LOG.debug("set_reservable() for network monitor is not implemented") + pass + + def get_unreservable_resourses(self): + # because network segments does not have reservable flag + # there are no unreservable resources + return [] + + def poll_resource_failures(self): + failed = [] + recovered = [] + dry_run = CONF[plugin.RESOURCE_TYPE].enable_polling_monitor_dry_run + network_segments = db_api.network_list() + + def process_network(network): + """ + For each blazar network, check if the network is not in use by a reservation + and get the neutron network based on segment_id + Tear down the network, if neutron network is using a vlan when there is no active + reservation is blazar + + 1. Detach the network's subnets from the router. + 2. Delete the subnet(s) + 3. Delete the network + + Args: + network (dict): blazar's network_segment row + """ + network_id_from_blazar = network["id"] + segment_id = network["segment_id"] + # gets the most recent reservation for the blazar network + network_curr_reservation = db_utils.get_most_recent_reservation_info_by_network_id(network_id_from_blazar) + if network_curr_reservation and network_curr_reservation['status'] == status.reservation.ACTIVE: + # This means that network works fine as the reservation started fine + LOG.debug(f"Network {network_id_from_blazar} - VLAN {segment_id} is in active reservation {network_curr_reservation['id']} - skipping") + return + neutron_client = neutron.BlazarNeutronClient() + networks = neutron_client.list_networks()['networks'] + network_from_neutron = next(( + network + for network in networks + if network.get('provider:segmentation_id') == segment_id + ), None) + if not network_from_neutron: + LOG.debug(f"Blazar network - {network_id_from_blazar} is not found in neutron - skipping") + return + network_id_from_neutron = network_from_neutron['id'] + ports_from_network = neutron_client.list_ports(network_id=network_id_from_neutron)["ports"] + router_ports = [ + port for port in ports_from_network + if port["device_owner"] == "network:router_interface" + ] + for port in router_ports: + for fixed_ip in port["fixed_ips"]: + router_id = port["device_id"] + subnet_id = fixed_ip["subnet_id"] + try: + remove_subnet_route_from_router(router_id, subnet_id, dry_run) + except Exception as e: + LOG.exception(f"Cannot remove subnet {subnet_id} route in router {router_id} - {e}") + LOG.warning(f"Detaching router {router_id} from Network subnet {subnet_id} - VLAN {segment_id}") + if not dry_run: + neutron_client.remove_interface_router(router_id, {'subnet_id': subnet_id}) + for port in ports_from_network: + if not dry_run: + try: + neutron_client.delete_port(port['id']) + except neutron_ex.PortNotFoundClient: + LOG.debug(f"Port {port['id']} not found in neutron") + subnets = neutron_client.list_subnets(network_id=network_id_from_neutron)['subnets'] + for subnet in subnets: + LOG.warning(f"Deleting subnet {subnet['id']} - VLAN {segment_id}") + if not dry_run: + neutron_client.delete_subnet(subnet['id']) + LOG.warning(f"Deleting network {network_id_from_neutron} - VLAN {segment_id}") + if not dry_run: + neutron_client.delete_network(network_id_from_neutron) + + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = executor.map(process_network, network_segments) + try: + results = list(futures) + except Exception as e: + LOG.exception('Skipping health check. %s', str(e)) + return failed, recovered diff --git a/blazar/plugins/networks/storage_plugin.py b/blazar/plugins/networks/storage_plugin.py new file mode 100644 index 000000000..e99a4d2b6 --- /dev/null +++ b/blazar/plugins/networks/storage_plugin.py @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- +# +# Author: Chameleon Cloud +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from blazar.manager import exceptions as manager_ex +from blazar.utils.openstack import manila +from blazar.utils.openstack import neutron +import collections +from oslo_config import cfg +from oslo_log import log as logging +from oslo_service import periodic_task + +opts = [ + cfg.IntOpt('set_manila_share_access_rules_interval', + default=5*60, + help='Set access rules for all manila shares every N seconds.' + 'If this number is negative the periodic task will be ' + 'disabled.'), + cfg.StrOpt('ceph_nfs_share_type', + default='default_share_type', + help='The Ceph NFS share type'), + cfg.StrOpt('storage_subnetpool', + default='filesystem-subnet-pool', + help='The storage subnet pool name'), + cfg.StrOpt('storage_router', + default='filesystem-ganesha-router', + help='The storage router name'), +] + +CONF = cfg.CONF +CONF.register_opts(opts, group="network_storage") +LOG = logging.getLogger(__name__) + +STORAGE_ROUTER_NAME = "storage_router_{network_segment_id}" + + +class StoragePlugin(): + """Plugin for storage usage type.""" + usage_type = "storage" + + def __init__(self): + super(StoragePlugin, self).__init__() + self.neutron_client = neutron.BlazarNeutronClient() + # get ganesha subnetpool by name + ganesha_subnetpool = self.neutron_client.list_subnetpools( + name=CONF.network_storage.storage_subnetpool + ).get("subnetpools") + self.ganesha_subnetpool = next( + iter(ganesha_subnetpool), None + ) + if not self.ganesha_subnetpool: + raise manager_ex.SubnetpoolNotFound( + network=CONF.network_storage.storage_subnetpool + ) + # get ganesha router by name + ganesha_router = self.neutron_client.list_routers( + name=CONF.network_storage.storage_router + ).get("routers") + self.ganesha_router = next( + iter(ganesha_router), None + ) + if not self.ganesha_router: + raise manager_ex.RouterNotFound( + network=CONF.network_storage.storage_router + ) + # collect periodic tasks + self.periodic_tasks = [self._set_manila_share_access_rules] + + def perform_extra_on_start_steps(self, network_segment, neutron_network): + neutron_network = neutron_network["network"] + try: + # create a subnet with the reserved network and subnetpool + subnet_body = { + "subnet": { + "name": f"{neutron_network['name']}-subnet", + "subnetpool_id": self.ganesha_subnetpool["id"], + "network_id": neutron_network["id"], + "ip_version": 4, + "project_id": neutron_network["project_id"], + } + } + subet = self.neutron_client.create_subnet(body=subnet_body) + # share the network with service project + rbac_policy_body = { + "rbac_policy": { + "object_type": "network", + "action": "access_as_shared", + "target_tenant": CONF.os_admin_project_name, + "object_id": neutron_network["id"], + } + } + self.neutron_client.create_rbac_policy( + rbac_policy_body + ) + # add the subnet to ganesha router + interface_body = { + 'subnet_id': subet["subnet"]["id"], + } + self.neutron_client.add_interface_router( + router=self.ganesha_router["id"], body=interface_body + ) + except Exception as e: + self.neutron_client.delete_network(neutron_network["id"]) + raise e + + def _get_ganesha_router_interfaces(self): + ports = self.neutron_client.list_ports( + device_id=self.ganesha_router["id"] + )["ports"] + result = collections.defaultdict(list) + for p in ports: + for fixed_ip in p["fixed_ips"]: + subnet = self.neutron_client.show_subnet( + fixed_ip["subnet_id"] + )["subnet"] + result[subnet["tenant_id"]].append(subnet["cidr"]) + + return result + + @periodic_task.periodic_task( + spacing=CONF.network_storage.set_manila_share_access_rules_interval, + run_immediately=True + ) + def _set_manila_share_access_rules(self, manager_obj, context): + manila_client = manila.BlazarManilaClient() + # get all available shares + shares = manila_client.shares.list( + search_opts={ + "all_tenants": 1, + "share_type": CONF.network_storage.ceph_nfs_share_type, + "status": "available", + } + ) + + ganesha_router_project_cidrs = self._get_ganesha_router_interfaces() + + for share in shares: + try: + proj = share.project_id + access_rules = manila_client.shares.access_list(share.id) + existing_ip_to_rule_id = { + rule.access_to: rule.id for rule in access_rules + if rule.access_level == "rw" + } + existing_ips = list(existing_ip_to_rule_id.keys()) + new_ips = ganesha_router_project_cidrs.get(proj, []) + ips_to_add = set(new_ips).difference(existing_ips) + ips_to_delete = set(existing_ips).difference(new_ips) + for ip in ips_to_add: + manila_client.shares.allow( + share.id, "ip", ip, "rw" + ) + for ip in ips_to_delete: + manila_client.shares.deny( + share.id, existing_ip_to_rule_id[ip] + ) + # all users should have ro access to a public share + existing_ro_rule_ids = [ + rule.id for rule in access_rules + if rule.access_level == "ro" + ] + if share.is_public and not existing_ro_rule_ids: + for prefix in self.ganesha_subnetpool["prefixes"]: + manila_client.shares.allow( + share.id, "ip", prefix, "ro" + ) + if not share.is_public and existing_ro_rule_ids: + for rule_id in existing_ro_rule_ids: + manila_client.shares.deny( + share.id, rule_id + ) + except Exception as e: + LOG.exception( + f"Failed to manage access rules for share {share.id}" + ) diff --git a/blazar/plugins/oshosts/host_plugin.py b/blazar/plugins/oshosts/host_plugin.py index 98837a233..93ee8f1ee 100644 --- a/blazar/plugins/oshosts/host_plugin.py +++ b/blazar/plugins/oshosts/host_plugin.py @@ -14,25 +14,31 @@ # License for the specific language governing permissions and limitations # under the License. -import collections import datetime +import random from novaclient import exceptions as nova_exceptions from oslo_config import cfg -from oslo_log import log as logging from oslo_utils import strutils +from blazar import context, policy, exceptions from blazar.db import api as db_api from blazar.db import exceptions as db_ex from blazar.db import utils as db_utils from blazar.manager import exceptions as manager_ex from blazar.plugins import base +from blazar.plugins import monitor from blazar.plugins import oshosts as plugin from blazar import status +from blazar.utils.openstack import heat +from blazar.utils.openstack import ironic from blazar.utils.openstack import nova from blazar.utils.openstack import placement from blazar.utils import plugins as plugins_utils -from blazar.utils import trusts +from oslo_log import log as logging +from random import shuffle +from uuid import UUID + plugin_opts = [ cfg.StrOpt('blazar_az_prefix', @@ -42,19 +48,21 @@ default='', help='Actions which we will be taken before the end of ' 'the lease'), - cfg.BoolOpt('enable_notification_monitor', - default=False, - help='Enable notification-based resource monitoring. ' - 'If it is enabled, the blazar-manager monitors states of ' - 'compute hosts by subscribing to notifications of Nova.'), + cfg.StrOpt('default_resource_properties', + default='', + help='Default resource_properties when creating a lease of ' + 'this type.'), + cfg.BoolOpt('display_default_resource_properties', + default=True, + help='Display default resource_properties if allocation fails ' + 'due to not enough resources'), + cfg.BoolOpt('retry_allocation_without_defaults', + default=True, + help='Whether an allocation should be retried on failure ' + 'without the default properties'), cfg.ListOpt('notification_topics', default=['notifications', 'versioned_notifications'], help='Notification topics to subscribe to.'), - cfg.BoolOpt('enable_polling_monitor', - default=False, - help='Enable polling-based resource monitoring. ' - 'If it is enabled, the blazar-manager monitors states ' - 'of compute hosts by polling the Nova API.'), cfg.IntOpt('polling_interval', default=60, min=1, @@ -65,16 +73,25 @@ help='Interval (minutes) of reservation healing. ' 'If 0 is specified, the interval is infinite and all the ' 'reservations in the future is healed at one time.'), + cfg.BoolOpt('randomize_host_selection', + default=False, + help='Allocate hosts for reservations randomly.'), ] +plugin_opts.extend(monitor.monitor_opts) + CONF = cfg.CONF CONF.register_opts(plugin_opts, group=plugin.RESOURCE_TYPE) LOG = logging.getLogger(__name__) -before_end_options = ['', 'snapshot', 'default'] + +before_end_options = ['', 'snapshot', 'default', 'email'] +on_start_options = ['', 'default', 'orchestration'] QUERY_TYPE_ALLOCATION = 'allocation' +MONITOR_ARGS = {"resource_type": plugin.RESOURCE_TYPE} + class PhysicalHostPlugin(base.BasePlugin, nova.NovaClientWrapper): """Plugin for physical host resource.""" @@ -88,18 +105,14 @@ class PhysicalHostPlugin(base.BasePlugin, nova.NovaClientWrapper): } def __init__(self): - super(PhysicalHostPlugin, self).__init__( - username=CONF.os_admin_username, - password=CONF.os_admin_password, - user_domain_name=CONF.os_admin_user_domain_name, - project_name=CONF.os_admin_project_name, - project_domain_name=CONF.os_admin_project_domain_name) - self.monitor = PhysicalHostMonitorPlugin() - self.monitor.register_healing_handler(self.heal_reservations) + super(PhysicalHostPlugin, self).__init__() + self.monitor = PhysicalHostMonitorPlugin(**MONITOR_ARGS) + self.monitor.register_reallocater(self._reallocate) self.placement_client = placement.BlazarPlacementClient() def reserve_resource(self, reservation_id, values): """Create reservation.""" + ctx = context.current() host_ids = self.allocation_candidates(values) if not host_ids: @@ -109,7 +122,8 @@ def reserve_resource(self, reservation_id, values): pool_name = reservation_id az_name = "%s%s" % (CONF[self.resource_type].blazar_az_prefix, pool_name) - pool_instance = pool.create(name=pool_name, az=az_name) + pool_instance = pool.create( + name=pool_name, project_id=ctx.project_id, az=az_name) host_rsrv_values = { 'reservation_id': reservation_id, 'aggregate_id': pool_instance.id, @@ -117,7 +131,8 @@ def reserve_resource(self, reservation_id, values): 'hypervisor_properties': values['hypervisor_properties'], 'count_range': values['count_range'], 'status': 'pending', - 'before_end': values['before_end'] + 'before_end': values['before_end'], + 'on_start': values['on_start'] } host_reservation = db_api.host_reservation_create(host_rsrv_values) for host_id in host_ids: @@ -146,7 +161,7 @@ def update_reservation(self, reservation_id, values): reservation['resource_id']) self._update_allocations(dates_before, dates_after, reservation_id, reservation['status'], host_reservation, - values) + {'project_id': lease['project_id'], **values}) updates = {} if 'min' in values or 'max' in values: @@ -165,7 +180,7 @@ def update_reservation(self, reservation_id, values): if updates: db_api.host_reservation_update(host_reservation['id'], updates) - def on_start(self, resource_id): + def on_start(self, resource_id, lease=None): """Add the hosts in the pool.""" host_reservation = db_api.host_reservation_get(resource_id) pool = nova.ReservationPool() @@ -173,25 +188,47 @@ def on_start(self, resource_id): for allocation in db_api.host_allocation_get_all_by_values( reservation_id=host_reservation['reservation_id']): host = db_api.host_get(allocation['compute_host_id']) - hosts.append(host['service_name']) + hosts.append(host['hypervisor_hostname']) pool.add_computehost(host_reservation['aggregate_id'], hosts) - def before_end(self, resource_id): + action = host_reservation.get('on_start', 'default') + + if 'orchestration' in action: + stack_id = action.split(':')[-1] + heat_client = heat.BlazarHeatClient() + heat_client.heat.stacks.update( + stack_id=stack_id, + existing=True, + converge=True, + parameters=dict( + reservation_id=host_reservation['reservation_id'])) + + def before_end(self, resource_id, lease=None): """Take an action before the end of a lease.""" host_reservation = db_api.host_reservation_get(resource_id) + action = host_reservation['before_end'] if action == 'default': action = CONF[plugin.RESOURCE_TYPE].before_end + if action == 'snapshot': pool = nova.ReservationPool() client = nova.BlazarNovaClient() for host in pool.get_computehosts( host_reservation['aggregate_id']): for server in client.servers.list( - search_opts={"host": host, "all_tenants": 1}): + search_opts={"node": host, "all_tenants": 1, + "project_id": lease['project_id']}): + # TODO(jason): Unclear if this even works! What happens + # when you try to createImage on a server not owned by the + # authentication context (admin context in this case.) Is + # the snapshot owned by the admin, or the original client.servers.create_image(server=server) + elif action == 'email': + plugins_utils.send_lease_extension_reminder( + lease, CONF.os_region_name) - def on_end(self, resource_id): + def on_end(self, resource_id, lease=None): """Remove the hosts from the pool.""" host_reservation = db_api.host_reservation_get(resource_id) db_api.host_reservation_update(host_reservation['id'], @@ -203,7 +240,7 @@ def on_end(self, resource_id): pool = nova.ReservationPool() for host in pool.get_computehosts(host_reservation['aggregate_id']): for server in self.nova.servers.list( - search_opts={"host": host, "all_tenants": 1}): + search_opts={"node": host, "all_tenants": 1}): try: self.nova.servers.delete(server=server) except nova_exceptions.NotFound: @@ -216,49 +253,11 @@ def on_end(self, resource_id): except manager_ex.AggregateNotFound: pass - def heal_reservations(self, failed_resources, interval_begin, - interval_end): - """Heal reservations which suffer from resource failures. - - :param failed_resources: a list of failed hosts. - :param interval_begin: start date of the period to heal. - :param interval_end: end date of the period to heal. - :return: a dictionary of {reservation id: flags to update} - e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': - {'missing_resources': True}} - """ - reservation_flags = {} - - host_ids = [h['id'] for h in failed_resources] - reservations = db_utils.get_reservations_by_host_ids(host_ids, - interval_begin, - interval_end) - - for reservation in reservations: - if reservation['resource_type'] != plugin.RESOURCE_TYPE: - continue - - for allocation in [alloc for alloc - in reservation['computehost_allocations'] - if alloc['compute_host_id'] in host_ids]: - if self._reallocate(allocation): - if reservation['status'] == status.reservation.ACTIVE: - if reservation['id'] not in reservation_flags: - reservation_flags[reservation['id']] = {} - reservation_flags[reservation['id']].update( - {'resources_changed': True}) - else: - if reservation['id'] not in reservation_flags: - reservation_flags[reservation['id']] = {} - reservation_flags[reservation['id']].update( - {'missing_resources': True}) - - return reservation_flags - - def _reallocate(self, allocation): + def _reallocate(self, allocation, force=False): """Allocate an alternative host. :param allocation: allocation to change. + :param force: Force reallocation off a host even if no other host is available :return: True if an alternative host was successfully allocated. """ reservation = db_api.reservation_get(allocation['reservation_id']) @@ -267,25 +266,27 @@ def _reallocate(self, allocation): lease = db_api.lease_get(reservation['lease_id']) pool = nova.ReservationPool() - # Remove the old host from the aggregate. + # Check for servers. We cannot reallocate active servers + old_host = db_api.host_get(allocation['compute_host_id']) if reservation['status'] == status.reservation.ACTIVE: - host = db_api.host_get(allocation['compute_host_id']) - pool.remove_computehost(h_reservation['aggregate_id'], - host['service_name']) + servers = self.nova.servers.list(search_opts={ + "node": old_host['hypervisor_hostname'], "all_tenants": 1}) + if len(servers) != 0: + raise manager_ex.HostHavingServers( + servers=[s.name for s in servers], + host=old_host['hypervisor_hostname'] + ) # Allocate an alternative host. start_date = max(datetime.datetime.utcnow(), lease['start_date']) new_hostids = self._matching_hosts( reservation['hypervisor_properties'], reservation['resource_properties'], - '1-1', start_date, lease['end_date'] + '1-1', start_date, lease['end_date'], + lease['project_id'], + allow_unreservable=False, # Don't reallocate to an unreservable host in this case ) - if not new_hostids: - db_api.host_allocation_destroy(allocation['id']) - LOG.warn('Could not find alternative host for reservation %s ' - '(lease: %s).', reservation['id'], lease['name']) - return False - else: + if new_hostids: new_hostid = new_hostids.pop() db_api.host_allocation_update(allocation['id'], {'compute_host_id': new_hostid}) @@ -295,17 +296,31 @@ def _reallocate(self, allocation): # Add the alternative host into the aggregate. new_host = db_api.host_get(new_hostid) pool.add_computehost(h_reservation['aggregate_id'], - new_host['service_name']) - + new_host['hypervisor_hostname']) + pool.remove_computehost(h_reservation['aggregate_id'], + old_host['hypervisor_hostname']) return True + elif force: + # If we are forcing reallocation from this host, + # destroy the old allocation and update pool. + LOG.warn('Forcing reservation %s off host %s' + '(lease: %s).', reservation['id'], old_host['hypervisor_hostname'], lease['name']) + db_api.host_allocation_destroy(allocation['id']) + pool.remove_computehost(h_reservation['aggregate_id'], + old_host['hypervisor_hostname']) + return False + else: + LOG.warn('Could not find alternative host for reservation %s ' + '(lease: %s).', reservation['id'], lease['name']) + return False def _get_extra_capabilities(self, host_id): extra_capabilities = {} raw_extra_capabilities = ( db_api.host_extra_capability_get_all_per_host(host_id)) - for capability in raw_extra_capabilities: - key = capability['capability_name'] - extra_capabilities[key] = capability['capability_value'] + for capability, property_name in raw_extra_capabilities: + key = property_name + extra_capabilities[key] = capability.capability_value return extra_capabilities def get(self, host_id): @@ -342,62 +357,98 @@ def create_computehost(self, host_values): if host_ref is None: raise manager_ex.InvalidHost(host=host_values) - with trusts.create_ctx_from_trust(trust_id): - inventory = nova.NovaInventory() - servers = inventory.get_servers_per_host(host_ref) - if servers: - raise manager_ex.HostHavingServers(host=host_ref, - servers=servers) - host_details = inventory.get_host_details(host_ref) - # NOTE(sbauza): Only last duplicate name for same extra capability - # will be stored - to_store = set(host_values.keys()) - set(host_details.keys()) - extra_capabilities_keys = to_store - extra_capabilities = dict( - (key, host_values[key]) for key in extra_capabilities_keys - ) + inventory = nova.NovaInventory() + servers = inventory.get_servers_per_host(host_ref) + if servers: + raise manager_ex.HostHavingServers(host=host_ref, + servers=servers) + host_details = inventory.get_host_details(host_ref) + if 'id' in host_details: + # Do not use nova's primary key for this host. + # Instead, generate a new one. + del host_details['id'] + self.handle_disabled_key(host_id, host_values) + # NOTE(sbauza): Only last duplicate name for same extra capability + # will be stored + to_store = set(host_values.keys()) - set(host_details.keys()) + extra_capabilities_keys = to_store + extra_capabilities = dict( + (key, host_values[key]) for key in extra_capabilities_keys + ) - if any([len(key) > 64 for key in extra_capabilities_keys]): - raise manager_ex.ExtraCapabilityTooLong() + if any([len(key) > 64 for key in extra_capabilities_keys]): + raise manager_ex.ExtraCapabilityTooLong() - self.placement_client.create_reservation_provider( - host_details['hypervisor_hostname']) + self.placement_client.create_reservation_provider( + host_details['hypervisor_hostname']) - pool = nova.ReservationPool() - pool.add_computehost(self.freepool_name, - host_details['service_name']) + pool = nova.ReservationPool() + # NOTE(jason): CHAMELEON-ONLY + # changed from 'service_name' to 'hypervisor_hostname' + pool.add_computehost(self.freepool_name, + host_details['hypervisor_hostname']) - host = None - cantaddextracapability = [] + host = None + cantaddextracapability = [] + try: + if trust_id: + host_details.update({'trust_id': trust_id}) + host = db_api.host_create(host_details) + except db_ex.BlazarDBException as e: + # We need to rollback + # TODO(sbauza): Investigate use of Taskflow for atomic + # transactions + # NOTE(jason): CHAMELEON-ONLY + # changed from 'service_name' to 'hypervisor_hostname' + pool.remove_computehost(self.freepool_name, + host_details['hypervisor_hostname']) + self.placement_client.delete_reservation_provider( + host_details['hypervisor_hostname']) + raise e + for key in extra_capabilities: + values = {'computehost_id': host['id'], + 'property_name': key, + 'capability_value': extra_capabilities[key], + } try: - if trust_id: - host_details.update({'trust_id': trust_id}) - host = db_api.host_create(host_details) - except db_ex.BlazarDBException as e: - # We need to rollback - # TODO(sbauza): Investigate use of Taskflow for atomic - # transactions - pool.remove_computehost(self.freepool_name, - host_details['service_name']) - self.placement_client.delete_reservation_provider( - host_details['hypervisor_hostname']) - raise e - for key in extra_capabilities: - values = {'computehost_id': host['id'], - 'capability_name': key, - 'capability_value': extra_capabilities[key], - } - try: - db_api.host_extra_capability_create(values) - except db_ex.BlazarDBException: - cantaddextracapability.append(key) - if cantaddextracapability: - raise manager_ex.CantAddExtraCapability( - keys=cantaddextracapability, - host=host['id']) - return self.get_computehost(host['id']) - - def is_updatable_extra_capability(self, capability): + db_api.host_extra_capability_create(values) + except db_ex.BlazarDBException: + cantaddextracapability.append(key) + if cantaddextracapability: + raise manager_ex.CantAddExtraCapability( + keys=cantaddextracapability, + host=host['id']) + + # Check for placement details + hostname = host_details['hypervisor_hostname'] + rp = self.placement_client.get_resource_provider(hostname) + if rp is None: + raise manager_ex.ResourceProviderNotFound(host=hostname) + + inventories = self.placement_client.get_inventory(rp['uuid']) + for rc, inventory in inventories['inventories'].items(): + resource_inventory = { + 'computehost_id': host['id'], + 'resource_class': rc, + 'total': inventory['total'], + 'reserved': inventory['reserved'], + 'min_unit': inventory['min_unit'], + 'max_unit': inventory['max_unit'], + 'step_size': inventory['step_size'], + 'allocation_ratio': inventory['allocation_ratio'], + } + db_api.host_resource_inventory_create(resource_inventory) + + traits = self.placement_client.get_traits(rp['uuid']) + for trait in traits: + db_api.host_trait_create({ + 'computehost_id': host['id'], + 'trait': trait, + }) + + return self.get_computehost(host['id']) + + def is_updatable_extra_capability(self, capability, capability_name): reservations = db_utils.get_reservations_by_host_id( capability['computehost_id'], datetime.datetime.utcnow(), datetime.date.max) @@ -414,7 +465,7 @@ def is_updatable_extra_capability(self, capability): # the extra_capability. for requirement in requirements_queries: # A requirement is of the form "key op value" as string - if requirement.split(" ")[0] == capability['capability_name']: + if requirement.split(" ")[0] == capability_name: return False return True @@ -422,49 +473,58 @@ def update_computehost(self, host_id, values): # nothing to update if not values: return self.get_computehost(host_id) - + if 'disabled' in values: + self.handle_disabled_key(host_id, values) cant_update_extra_capability = [] + cant_delete_extra_capability = [] previous_capabilities = self._get_extra_capabilities(host_id) updated_keys = set(values.keys()) & set(previous_capabilities.keys()) new_keys = set(values.keys()) - set(previous_capabilities.keys()) for key in updated_keys: - raw_capability = next(iter( + raw_capability, property_name = next(iter( db_api.host_extra_capability_get_all_per_name(host_id, key))) - capability = { - 'capability_name': key, - 'capability_value': values[key], - } - if self.is_updatable_extra_capability(raw_capability): - try: - db_api.host_extra_capability_update( - raw_capability['id'], capability) - except (db_ex.BlazarDBException, RuntimeError): - cant_update_extra_capability.append( - raw_capability['capability_name']) + + if self.is_updatable_extra_capability( + raw_capability, property_name): + if values[key] is not None: + try: + capability = {'capability_value': values[key]} + db_api.host_extra_capability_update( + raw_capability['id'], capability) + except (db_ex.BlazarDBException, RuntimeError): + cant_update_extra_capability.append(property_name) + else: + try: + db_api.host_extra_capability_destroy( + raw_capability['id']) + except db_ex.BlazarDBException: + cant_delete_extra_capability.append(property_name) else: LOG.info("Capability %s can't be updated because " "existing reservations require it.", - raw_capability['capability_name']) - cant_update_extra_capability.append( - raw_capability['capability_name']) + property_name) + cant_update_extra_capability.append(property_name) for key in new_keys: new_capability = { 'computehost_id': host_id, - 'capability_name': key, + 'property_name': key, 'capability_value': values[key], } try: db_api.host_extra_capability_create(new_capability) except (db_ex.BlazarDBException, RuntimeError): - cant_update_extra_capability.append( - new_capability['capability_name']) + cant_update_extra_capability.append(key) if cant_update_extra_capability: raise manager_ex.CantAddExtraCapability( host=host_id, keys=cant_update_extra_capability) + if cant_delete_extra_capability: + raise manager_ex.ExtraCapabilityNotFound( + resource=host_id, keys=cant_delete_extra_capability) + LOG.info('Extra capabilities on compute host %s updated with %s', host_id, values) return self.get_computehost(host_id) @@ -474,52 +534,139 @@ def delete_computehost(self, host_id): if not host: raise manager_ex.HostNotFound(host=host_id) - with trusts.create_ctx_from_trust(host['trust_id']): - if db_api.host_allocation_get_all_by_values( - compute_host_id=host_id): - raise manager_ex.CantDeleteHost( - host=host_id, - msg='The host is reserved.' - ) + if db_api.host_allocation_get_all_by_values( + compute_host_id=host_id): + raise manager_ex.CantDeleteHost( + host=host_id, + msg='The host is reserved.' + ) + try: inventory = nova.NovaInventory() servers = inventory.get_servers_per_host( host['hypervisor_hostname']) if servers: raise manager_ex.HostHavingServers( host=host['hypervisor_hostname'], servers=servers) + pool = nova.ReservationPool() + # NOTE(jason): CHAMELEON-ONLY + # changed from 'service_name' to 'hypervisor_hostname' + pool.remove_computehost(self.freepool_name, + host['hypervisor_hostname']) + self.placement_client.delete_reservation_provider( + host['hypervisor_hostname']) + except manager_ex.HostNotFound: + LOG.warning( + "Host %s not found in Nova and could not be cleaned up. Some " + "manual cleanup may be required.", host['hypervisor_hostname'] + ) - try: - pool = nova.ReservationPool() - pool.remove_computehost(self.freepool_name, - host['service_name']) - self.placement_client.delete_reservation_provider( - host['hypervisor_hostname']) - # NOTE(sbauza): Extracapabilities will be destroyed thanks to - # the DB FK. - db_api.host_destroy(host_id) - except db_ex.BlazarDBException as e: - # Nothing so bad, but we need to alert admins - # they have to rerun - raise manager_ex.CantDeleteHost(host=host_id, msg=str(e)) - - def list_allocations(self, query): + try: + # NOTE(sbauza): Extracapabilities will be destroyed thanks to + # the DB FK. + db_api.host_destroy(host_id) + except db_ex.BlazarDBException as e: + # Nothing so bad, but we need to alert admins + # they have to rerun + raise manager_ex.CantDeleteHost(host=host_id, msg=str(e)) + + def handle_disabled_key(self, host_id, values): + # only admin can set/unset 'disabled' flag + if "disabled" in values and self._is_admin(): + new_disabled_flag = False if values.get('disabled') is None else True + + disabled_reason = values.get('disabled_reason') + if new_disabled_flag and not disabled_reason: + raise manager_ex.MissingParameter(param='disabled_reason') + # unset reason automatically when re-enabling a host + if not new_disabled_flag: + values["disabled_reason"] = None + + host_details = self.get_computehost(host_id) + # set True if value contains any string other than None + self.set_disabled(host_details, new_disabled_flag, disabled_reason) + del values['disabled'] + # NOTE: disabled_reason is set as normal in create_computehost and update_computehost. + # This method just verifies it is valid. + + def set_disabled(self, resource, is_disabled, disabled_reason): + host_update_values = {"disabled": is_disabled, "disabled_reason": disabled_reason} + # if a host is set as disabled, then it should not be reservable + if is_disabled: + host_update_values['reservable'] = False + db_api.host_update(resource["id"], host_update_values) + LOG.warn( + f"{resource['hypervisor_hostname']}", + f"is set disabled {is_disabled} with reason: {disabled_reason}" + ) + + def list_allocations(self, query, detail=False): hosts_id_list = [h['id'] for h in db_api.host_list()] options = self.get_query_options(query, QUERY_TYPE_ALLOCATION) - - hosts_allocations = self.query_allocations(hosts_id_list, **options) + options['detail'] = detail + hosts_allocations = self.query_host_allocations(hosts_id_list, + **options) + self.add_extra_allocation_info(hosts_allocations) + self.add_allocation_cleaning_time(hosts_allocations, CONF.cleaning_time) return [{"resource_id": host, "reservations": allocs} for host, allocs in hosts_allocations.items()] - def get_allocations(self, host_id, query): + def get_allocations(self, host_id, query, detail=False): options = self.get_query_options(query, QUERY_TYPE_ALLOCATION) - host_allocations = self.query_allocations([host_id], **options) - if host_id not in host_allocations: - host_allocations = {host_id: []} - allocs = host_allocations[host_id] + options['detail'] = detail + host_allocations = self.query_host_allocations([host_id], **options) + allocs = host_allocations.get(host_id, []) + self.add_allocation_cleaning_time({host_id: allocs}, CONF.cleaning_time) return {"resource_id": host_id, "reservations": allocs} - def query_allocations(self, hosts, lease_id=None, reservation_id=None): + def reallocate_computehost(self, host_id, data): + lease_id = data.get('lease_id') + force = data.get("force", False) + if lease_id: + # If we're only reallocating a host for a single lease, + # then we allow non-admin users to perform this action, + # but only on leases they own + lease = db_api.lease_get(lease_id) + ctx = context.current() + prid = lease['project_id'] + policy.check_enforcement('leases', action='reallocate', ctx=ctx, target={ + 'project': prid, + 'user': ctx.user_id, + 'project_id': prid, + 'user_id': ctx.user_id, + }) + else: + policy.check_enforcement('oshosts', action='reallocate') + + allocations = self.get_allocations(host_id, data, detail=True) + + for alloc in allocations['reservations']: + reservation_flags = {} + host_allocation = db_api.host_allocation_get_all_by_values( + compute_host_id=host_id, + reservation_id=alloc['id'])[0] + + if self._reallocate(host_allocation, force=force): + if alloc['status'] == status.reservation.ACTIVE: + reservation_flags.update(dict(resources_changed=True)) + db_api.lease_update(alloc['lease_id'], dict(degraded=True)) + elif force: + # Resources are only missing if reallocated failed, and force was used + reservation_flags.update(dict(missing_resources=True)) + db_api.lease_update(alloc['lease_id'], dict(degraded=True)) + + db_api.reservation_update(alloc['id'], reservation_flags) + + return self.get_allocations(host_id, data) + + def query_allocations(self, hosts, detail=None, lease_id=None, + reservation_id=None): + return self.query_host_allocations(hosts, detail=detail, + lease_id=lease_id, + reservation_id=reservation_id) + + def query_host_allocations(self, hosts, detail=None, lease_id=None, + reservation_id=None): """Return dict of host and its allocations. The list element forms @@ -527,7 +674,9 @@ def query_allocations(self, hosts, lease_id=None, reservation_id=None): 'host-id': [ { 'lease_id': lease_id, - 'id': reservation_id + 'id': reservation_id, + 'start_date': lease_start_date, + 'end_date': lease_end_date }, ] }. @@ -535,28 +684,60 @@ def query_allocations(self, hosts, lease_id=None, reservation_id=None): start = datetime.datetime.utcnow() end = datetime.date.max - # To reduce overhead, this method only executes one query - # to get the allocation information - rsv_lease_host = db_utils.get_reservation_allocations_by_host_ids( + reservations = db_utils.get_reservation_allocations_by_host_ids( hosts, start, end, lease_id, reservation_id) + host_allocations = {h: [] for h in hosts} + + for reservation in reservations: + if not detail: + del reservation['project_id'] + del reservation['lease_name'] + del reservation['status'] + + for host_id in reservation['host_ids']: + if host_id in host_allocations.keys(): + host_allocations[host_id].append({ + k: v for k, v in reservation.items() + if k != 'host_ids'}) + + return host_allocations - hosts_allocs = collections.defaultdict(list) - for rsv, lease, host in rsv_lease_host: - hosts_allocs[host].append({'lease_id': lease, 'id': rsv}) - return hosts_allocs + def update_default_parameters(self, values): + self.add_default_resource_properties(values) def allocation_candidates(self, values): self._check_params(values) - return self._matching_hosts( + host_ids = self._matching_hosts( values['hypervisor_properties'], values['resource_properties'], values['count_range'], values['start_date'], - values['end_date']) + values['end_date'], + values['project_id'], + ) + + min_hosts, _ = [int(n) for n in values['count_range'].split('-')] + + if len(host_ids) < min_hosts: + raise manager_ex.NotEnoughHostsAvailable() + + return host_ids + + def _is_admin(self): + try: + # when monitor calls this will raise an error + ctx = context.current() + except RuntimeError: + return False + if policy.enforce(ctx, 'admin', {}, do_raise=False): + return True + else: + return False def _matching_hosts(self, hypervisor_properties, resource_properties, - count_range, start_date, end_date): + count_range, start_date, end_date, project_id, + allow_unreservable=True): """Return the matching hosts (preferably not allocated) """ @@ -578,7 +759,14 @@ def _matching_hosts(self, hypervisor_properties, resource_properties, if resource_properties: filter_array += plugins_utils.convert_requirements( resource_properties) - for host in db_api.reservable_host_get_all_by_queries(filter_array): + # admin can create a lease for host with 'reservable' False + if self._is_admin() and allow_unreservable: + hosts = db_api.host_get_all_by_queries(filter_array) + else: + hosts = db_api.reservable_host_get_all_by_queries(filter_array) + for host in hosts: + if not self.is_project_allowed(project_id, self.get_computehost(host["id"])): + continue if not db_api.host_allocation_get_all_by_values( compute_host_id=host['id']): not_allocated_host_ids.append(host['id']) @@ -592,9 +780,13 @@ def _matching_hosts(self, hypervisor_properties, resource_properties, ]: allocated_host_ids.append(host['id']) if len(not_allocated_host_ids) >= int(min_host): + if CONF[self.resource_type].randomize_host_selection: + random.shuffle(not_allocated_host_ids) return not_allocated_host_ids[:int(max_host)] all_host_ids = allocated_host_ids + not_allocated_host_ids if len(all_host_ids) >= int(min_host): + if CONF[self.resource_type].randomize_host_selection: + random.shuffle(all_host_ids) return all_host_ids[:int(max_host)] else: return [] @@ -623,9 +815,26 @@ def _check_params(self, values): if values['before_end'] not in before_end_options: raise manager_ex.MalformedParameter(param='before_end') + if 'on_start' not in values: + values['on_start'] = 'default' + if not self._is_valid_on_start_option(values['on_start']): + raise manager_ex.MalformedParameter(param='on_start') + + def _is_valid_on_start_option(self, value): + + if 'orchestration' in value: + stack = value.split(':')[-1] + try: + UUID(stack) + return True + except Exception: + return False + else: + return value in on_start_options + def _validate_min_max_range(self, values, min_hosts, max_hosts): - self._convert_int_param(min_hosts, 'min') - self._convert_int_param(max_hosts, 'max') + min_hosts = self._convert_int_param(min_hosts, 'min') + max_hosts = self._convert_int_param(max_hosts, 'max') if min_hosts <= 0 or max_hosts <= 0: raise manager_ex.MalformedParameter( param='min and max (must be greater than or equal to 1)') @@ -665,7 +874,9 @@ def _update_allocations(self, dates_before, dates_after, reservation_id, host_ids = self._matching_hosts( hypervisor_properties, resource_properties, str(min_hosts) + '-' + str(max_hosts), - dates_after['start_date'], dates_after['end_date']) + dates_after['start_date'], dates_after['end_date'], + values['project_id'] + ) if len(host_ids) >= min_hosts: new_hosts = [] pool = nova.ReservationPool() @@ -674,7 +885,7 @@ def _update_allocations(self, dates_before, dates_after, reservation_id, {'compute_host_id': host_id, 'reservation_id': reservation_id}) new_host = db_api.host_get(host_id) - new_hosts.append(new_host['service_name']) + new_hosts.append(new_host['hypervisor_hostname']) if reservation_status == status.reservation.ACTIVE: # Add new hosts into the aggregate. pool.add_computehost(host_reservation['aggregate_id'], @@ -693,6 +904,11 @@ def _allocations_to_remove(self, dates_before, dates_after, max_hosts, self._filter_hosts_by_properties( hypervisor_properties, resource_properties)] + start_date_with_margin = dates_after['start_date'] - datetime.timedelta( + minutes=CONF.cleaning_time) + end_date_with_margin = dates_after['end_date'] + datetime.timedelta( + minutes=CONF.cleaning_time) + for alloc in allocs: if alloc['compute_host_id'] not in requested_host_ids: allocs_to_remove.append(alloc) @@ -701,21 +917,20 @@ def _allocations_to_remove(self, dates_before, dates_after, max_hosts, dates_before['end_date'] < dates_after['end_date']): reserved_periods = db_utils.get_reserved_periods( alloc['compute_host_id'], - dates_after['start_date'], - dates_after['end_date'], + start_date_with_margin, + end_date_with_margin, datetime.timedelta(seconds=1)) max_start = max(dates_before['start_date'], - dates_after['start_date']) + start_date_with_margin) min_end = min(dates_before['end_date'], - dates_after['end_date']) + end_date_with_margin) if not (len(reserved_periods) == 0 or (len(reserved_periods) == 1 and reserved_periods[0][0] == max_start and reserved_periods[0][1] == min_end)): allocs_to_remove.append(alloc) - continue kept_hosts = len(allocs) - len(allocs_to_remove) if kept_hosts > max_hosts: @@ -740,48 +955,32 @@ def _filter_hosts_by_properties(self, hypervisor_properties, return db_api.host_list() -class PhysicalHostMonitorPlugin(base.BaseMonitorPlugin, +class PhysicalHostMonitorPlugin(monitor.GeneralMonitorPlugin, nova.NovaClientWrapper): """Monitor plugin for physical host resource.""" - # Singleton design pattern - _instance = None - - def __new__(cls): - if not cls._instance: - cls._instance = super(PhysicalHostMonitorPlugin, cls).__new__(cls) - cls._instance.healing_handlers = [] - super(PhysicalHostMonitorPlugin, cls._instance).__init__( - username=CONF.os_admin_username, - password=CONF.os_admin_password, - user_domain_name=CONF.os_admin_user_domain_name, - project_name=CONF.os_admin_project_name, - project_domain_name=CONF.os_admin_project_domain_name) - return cls._instance + def __new__(cls, *args, **kwargs): + return super(PhysicalHostMonitorPlugin, cls).__new__(cls, *args, + **kwargs) - def __init__(self): - """Do nothing. + def filter_allocations(self, reservation, host_ids): + return [alloc for alloc + in reservation['computehost_allocations'] + if alloc['compute_host_id'] in host_ids] - This class uses the Singleton design pattern and an instance of this - class is generated and initialized in __new__(). - """ - pass + def get_reservations_by_resource_ids(self, host_ids, + interval_begin, interval_end): + return db_utils.get_reservations_by_host_ids(host_ids, + interval_begin, + interval_end) - def register_healing_handler(self, handler): - self.healing_handlers.append(handler) - - def is_notification_enabled(self): - """Check if the notification monitor is enabled.""" - return CONF[plugin.RESOURCE_TYPE].enable_notification_monitor + def get_unreservable_resourses(self): + return db_api.unreservable_host_get_all_by_queries([]) def get_notification_event_types(self): """Get event types of notification messages to handle.""" return ['service.update'] - def get_notification_topics(self): - """Get topics of notification to subscribe to.""" - return CONF[plugin.RESOURCE_TYPE].notification_topics - def notification_callback(self, event_type, payload): """Handle a notification message. @@ -804,118 +1003,159 @@ def notification_callback(self, event_type, payload): if failed_hosts: LOG.warn('%s failed.', failed_hosts[0]['hypervisor_hostname']) - reservation_flags = self._handle_failures(failed_hosts) + for host in failed_hosts: + self.set_reservable(host, False) else: recovered_hosts = db_api.host_get_all_by_queries( ['reservable == 0', 'hypervisor_hostname == ' + data['host']]) if recovered_hosts: - db_api.host_update(recovered_hosts[0]['id'], - {'reservable': True}) + self.set_reservable(recovered_hosts[0], True) LOG.warn('%s recovered.', recovered_hosts[0]['hypervisor_hostname']) return reservation_flags - def is_polling_enabled(self): - """Check if the polling monitor is enabled.""" - return CONF[plugin.RESOURCE_TYPE].enable_polling_monitor - - def get_polling_interval(self): - """Get interval of polling.""" - return CONF[plugin.RESOURCE_TYPE].polling_interval - - def poll(self): - """Detect and handle resource failures. - - :return: a dictionary of {reservation id: flags to update} - e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': - {'missing_resources': True}} - """ - LOG.trace('Poll...') - reservation_flags = {} - - failed_hosts, recovered_hosts = self._poll_resource_failures() - if failed_hosts: - for host in failed_hosts: - LOG.warn('%s failed.', host['hypervisor_hostname']) - reservation_flags = self._handle_failures(failed_hosts) - if recovered_hosts: - for host in recovered_hosts: - db_api.host_update(host['id'], {'reservable': True}) - LOG.warn('%s recovered.', host['hypervisor_hostname']) - - return reservation_flags + def set_reservable(self, resource, is_reservable): + if resource.get('disabled', False): + LOG.debug(f"{resource['hypervisor_hostname']} is disabled - cannot set reservable") + return + db_api.host_update(resource["id"], {"reservable": is_reservable}) + LOG.warn('%s %s.', resource["hypervisor_hostname"], + "recovered" if is_reservable else "failed") - def _poll_resource_failures(self): + def poll_resource_failures(self): """Check health of hosts by calling Nova Hypervisors API. :return: a list of failed hosts, a list of recovered hosts. """ hosts = db_api.host_get_all_by_filters({}) - reservable_hosts = [h for h in hosts if h['reservable'] is True] - unreservable_hosts = [h for h in hosts if h['reservable'] is False] + dry_run = CONF[plugin.RESOURCE_TYPE].enable_polling_monitor_dry_run + ironic_hosts = [] + nova_hosts = [] + for h in hosts: + if 'hypervisor_type' in h and h['hypervisor_type'] == 'ironic': + ironic_hosts.append(h) + else: + nova_hosts.append(h) + failed_hosts = [] + recovered_hosts = [] try: - hvs = self.nova.hypervisors.list() - - failed_hv_ids = [str(hv.id) for hv in hvs - if hv.state == 'down' or hv.status == 'disabled'] - failed_hosts = [host for host in reservable_hosts - if host['id'] in failed_hv_ids] - - active_hv_ids = [str(hv.id) for hv in hvs - if hv.state == 'up' and hv.status == 'enabled'] - recovered_hosts = [host for host in unreservable_hosts - if host['id'] in active_hv_ids] + if ironic_hosts: + invalid_power_states = ['error'] + invalid_provision_states = [ + 'error', 'clean failed', 'manageable', 'deploy failed', + 'inspect failed', 'clean failed', 'adopt failed', + 'rescue failed', 'unrescue failed', 'enroll', + ] + reservable_hosts = [h for h in ironic_hosts + if h['reservable'] is True] + unreservable_hosts = [h for h in ironic_hosts + if h['reservable'] is False] + + ironic_client = ironic.BlazarIronicClient() + nodes = ironic_client.ironic.node.list() + nodes_by_uuid = {n.uuid: n for n in nodes} + failed_bm_ids = [n.uuid for n in nodes + if n.maintenance + or n.power_state in invalid_power_states + or n.provision_state + in invalid_provision_states] + + for host in reservable_hosts: + print(host['hypervisor_hostname']) + if host['hypervisor_hostname'] in failed_bm_ids: + node = nodes_by_uuid.get(host['hypervisor_hostname']) + error = f"Node status is maintenance {node.maintenance}, power state {node.power_state} and provision state {node.provision_state}" + db_api.host_update(host["id"], {"last_error": error}) + failed_hosts.append(host) + active_bm_ids = [n.uuid for n in nodes + if not n.maintenance + and n.provision_state in ['available', 'active']] + recovered_hosts.extend([host for host in unreservable_hosts + if host['hypervisor_hostname'] + in active_bm_ids]) + + if nova_hosts: + reservable_hosts = [h for h in nova_hosts + if h['reservable'] is True] + unreservable_hosts = [h for h in nova_hosts + if h['reservable'] is False] + + hvs = self.nova.hypervisors.list() + hvs_by_hostname = {hv.hypervisor_hostname: hv for hv in hvs} + + failed_hv_hostnames = [hv.hypervisor_hostname for hv in hvs + if hv.state == 'down' + or hv.status == 'disabled'] + for host in reservable_hosts: + if host['hypervisor_hostname'] in failed_hv_hostnames: + hv = hvs_by_hostname.get(host['hypervisor_hostname']) + error = f"Hypervisor status is {hv.state} and {hv.status}" + db_api.host_update(host["id"], {"last_error": error}) + failed_hosts.append(host) + + + active_hv_hostnames = [hv.hypervisor_hostname for hv in hvs + if hv.state == 'up' + and hv.status == 'enabled'] + recovered_hosts.extend([host for host in unreservable_hosts + if host['hypervisor_hostname'] in active_hv_hostnames]) + + aggregates = self.nova.aggregates.list() + # create a map to get the current aggregate of a host in nova + host_to_agg_map = {host: agg for agg in aggregates for host in agg.hosts} + # get all physical hosts + all_hosts = db_api.host_list() + pool = nova.ReservationPool() + freepool = pool.get_aggregate_from_name_or_id(pool.freepool_name) + if not freepool: + raise ValueError("Aggregate list does not contain a freepool!") + for host in all_hosts: + # get the most recent reservation for the host_id and check if we need to move to freepool + reservation = db_utils.get_most_recent_reservation_info_by_host_id(host['id']) + # ignore the reservation which is active, as the host must already be in the right pool + if reservation and reservation["reservation_status"] == status.reservation.ACTIVE: + LOG.debug(f"{host['hypervisor_hostname']} is in an active reservation" + f" {reservation['reservation_id']} - skipping aggregate clean up") + continue + # host not in 'active' or 'pending' reservation should be in freepool + host_uuid = host['hypervisor_hostname'] + curr_agg = host_to_agg_map.get(host['hypervisor_hostname']) + if not curr_agg: + msg = f"{host['hypervisor_hostname']} not found in any aggregate - moving to freepool" + if dry_run: + LOG.info(f"DRY RUN: {msg}") + else: + LOG.warning(msg) + freepool.add_host(host_uuid) + continue + if curr_agg.name == freepool.name: + # if the host is already in freepool skip it + LOG.debug(f"{host['hypervisor_hostname']} is already in a freepool - skipping aggregate clean up") + continue + try: + msg = ( + f"Moving {host['hypervisor_hostname']}" + f" from aggregate {curr_agg.name} to freepool" + f" (reservation {reservation['reservation_id']}" + f" status {reservation['reservation_status']})" + ) + if dry_run: + LOG.info(f"DRY RUN: {msg}") + continue + LOG.warning(msg) + curr_agg.remove_host(host_uuid) + freepool.add_host(host_uuid) + hosts_in_agg = pool.get_computehosts(curr_agg) + if not hosts_in_agg: + LOG.warning(f"Removing empty aggregate {curr_agg.name}") + curr_agg.delete() + except Exception as e: + LOG.exception(f"Failed to recover host {host}", exc_info=e) + failed_hosts.append(host) except Exception as e: LOG.exception('Skipping health check. %s', str(e)) return failed_hosts, recovered_hosts - - def _handle_failures(self, failed_hosts): - """Handle resource failures. - - :param failed_hosts: a list of failed hosts. - :return: a dictionary of {reservation id: flags to update} - e.g. {'de27786d-bd96-46bb-8363-19c13b2c6657': - {'missing_resources': True}} - """ - - # Update the computehosts table - for host in failed_hosts: - try: - db_api.host_update(host['id'], {'reservable': False}) - except Exception as e: - LOG.exception('Failed to update %s. %s', - host['hypervisor_hostname'], str(e)) - - # Heal related reservations - return self.heal() - - def get_healing_interval(self): - """Get interval of reservation healing in minutes.""" - return CONF[plugin.RESOURCE_TYPE].healing_interval - - def heal(self): - """Heal suffering reservations in the next healing interval. - - :return: a dictionary of {reservation id: flags to update} - """ - reservation_flags = {} - hosts = db_api.unreservable_host_get_all_by_queries([]) - - interval_begin = datetime.datetime.utcnow() - interval = self.get_healing_interval() - if interval == 0: - interval_end = datetime.date.max - else: - interval_end = interval_begin + datetime.timedelta( - minutes=interval) - - for handler in self.healing_handlers: - reservation_flags.update(handler(hosts, - interval_begin, - interval_end)) - - return reservation_flags diff --git a/blazar/policies/__init__.py b/blazar/policies/__init__.py index a8916c0bc..f779f0f44 100644 --- a/blazar/policies/__init__.py +++ b/blazar/policies/__init__.py @@ -13,8 +13,10 @@ import itertools from blazar.policies import base +from blazar.policies import devices from blazar.policies import floatingips from blazar.policies import leases +from blazar.policies import networks from blazar.policies import oshosts @@ -23,5 +25,7 @@ def list_rules(): base.list_rules(), leases.list_rules(), oshosts.list_rules(), - floatingips.list_rules() + floatingips.list_rules(), + networks.list_rules(), + devices.list_rules() ) diff --git a/blazar/policies/base.py b/blazar/policies/base.py index 2bd7f64a0..94ed65df1 100644 --- a/blazar/policies/base.py +++ b/blazar/policies/base.py @@ -23,7 +23,7 @@ description="Default rule for most Admin APIs."), policy.RuleDefault( name="admin_or_owner", - check_str="rule:admin or project_id:%(project_id)s", + check_str="rule:admin or project:%(project)s", description="Default rule for most non-Admin APIs.") ] diff --git a/blazar/policies/devices.py b/blazar/policies/devices.py new file mode 100644 index 000000000..315e97034 --- /dev/null +++ b/blazar/policies/devices.py @@ -0,0 +1,121 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_policy import policy + +from blazar.policies import base + +POLICY_ROOT = 'blazar:devices:%s' + +devices_policies = [ + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'get', + check_str=base.RULE_ADMIN, + description='Policy rule for List/Show Device(s) API.', + operations=[ + { + 'path': '/{api_version}/devices', + 'method': 'GET' + }, + { + 'path': '/{api_version}/devices/{device_id}', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'post', + check_str=base.RULE_ADMIN, + description='Policy rule for Create Device API.', + operations=[ + { + 'path': '/{api_version}/devices', + 'method': 'POST' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'put', + check_str=base.RULE_ADMIN, + description='Policy rule for Update Host API.', + operations=[ + { + 'path': '/{api_version}/devices/{device_id}', + 'method': 'PUT' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'delete', + check_str=base.RULE_ADMIN, + description='Policy rule for Delete Device API.', + operations=[ + { + 'path': '/{api_version}/devices/{device_id}', + 'method': 'DELETE' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'get_allocations', + check_str=base.RULE_ADMIN, + description='Policy rule for List/Get Device(s) Allocations API.', + operations=[ + { + 'path': '/{api_version}/devices/allocations', + 'method': 'GET' + }, + { + 'path': '/{api_version}/devices/{device_id}/allocation', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'reallocate', + check_str=base.RULE_ADMIN, + description='Policy rule for Reallocate Host API.', + operations=[ + { + 'path': '/{api_version}/devices/{device_id}/allocation', + 'method': 'PUT' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'get_resource_properties', + check_str=base.RULE_ADMIN, + description='Policy rule for Resource Properties API.', + operations=[ + { + 'path': '/{api_version}/devices/resource_properties', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'update_resource_properties', + check_str=base.RULE_ADMIN, + description='Policy rule for Resource Properties API.', + operations=[ + { + 'path': ('/{api_version}/devices/resource_properties/' + '{property_name}'), + 'method': 'PATCH' + } + ] + ), +] + + +def list_rules(): + return devices_policies diff --git a/blazar/policies/leases.py b/blazar/policies/leases.py index e209f69a3..84b388de5 100644 --- a/blazar/policies/leases.py +++ b/blazar/policies/leases.py @@ -29,6 +29,18 @@ { 'path': '/{api_version}/leases/{lease_id}', 'method': 'GET' + }, + { + 'path': '/{api_version}/leases/{lease_id}/hosts', + 'method': 'GET' + }, + { + 'path': '/{api_version}/leases/{lease_id}/networks', + 'method': 'GET' + }, + { + 'path': '/{api_version}/leases/{lease_id}/devices', + 'method': 'GET' } ] ), @@ -54,6 +66,18 @@ } ] ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'reallocate', + check_str=base.RULE_ADMIN_OR_OWNER, + description="Policy rule which allows owners to reallocate " + "the host(s) for their lease", + operations=[ + { + 'path': '/{api_version}/os-hosts/{host_id}/allocation', + 'method': 'PUT' + } + ] + ), policy.DocumentedRuleDefault( name=POLICY_ROOT % 'delete', check_str=base.RULE_ADMIN_OR_OWNER, diff --git a/blazar/policies/networks.py b/blazar/policies/networks.py new file mode 100644 index 000000000..1405b87bd --- /dev/null +++ b/blazar/policies/networks.py @@ -0,0 +1,99 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_policy import policy + +from blazar.policies import base + +POLICY_ROOT = 'blazar:networks:%s' + +networks_policies = [ + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'get', + check_str=base.RULE_ADMIN, + description='Policy rule for List/Show Network(s) API.', + operations=[ + { + 'path': '/{api_version}/networks', + 'method': 'GET' + }, + { + 'path': '/{api_version}/networks/{network_id}', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'post', + check_str=base.RULE_ADMIN, + description='Policy rule for Create Network API.', + operations=[ + { + 'path': '/{api_version}/networks', + 'method': 'POST' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'delete', + check_str=base.RULE_ADMIN, + description='Policy rule for Delete Network API.', + operations=[ + { + 'path': '/{api_version}/networks/{network_id}', + 'method': 'DELETE' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'get_allocations', + check_str=base.RULE_ADMIN, + description='Policy rule for List/Get Network(s) Allocations API.', + operations=[ + { + 'path': '/{api_version}/networks/allocations', + 'method': 'GET' + }, + { + 'path': '/{api_version}/networks/{network_id}/allocation', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'get_resource_properties', + check_str=base.RULE_ADMIN, + description='Policy rule for Resource Properties API.', + operations=[ + { + 'path': '/{api_version}/networks/resource_properties', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'update_resource_properties', + check_str=base.RULE_ADMIN, + description='Policy rule for Resource Properties API.', + operations=[ + { + 'path': ('/{api_version}/networks/resource_properties/' + '{property_name}'), + 'method': 'PATCH' + } + ] + ), +] + + +def list_rules(): + return networks_policies diff --git a/blazar/policies/oshosts.py b/blazar/policies/oshosts.py index 68a9143e7..a0bb5e9d3 100644 --- a/blazar/policies/oshosts.py +++ b/blazar/policies/oshosts.py @@ -79,7 +79,41 @@ 'method': 'GET' } ] - ) + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'reallocate', + check_str=base.RULE_ADMIN, + description='Policy rule for Reallocate Host API.', + operations=[ + { + 'path': '/{api_version}/os-hosts/{host_id}/allocation', + 'method': 'PUT' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'get_resource_properties', + check_str=base.RULE_ADMIN, + description='Policy rule for Resource Properties API.', + operations=[ + { + 'path': '/{api_version}/os-hosts/resource_properties', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=POLICY_ROOT % 'update_resource_properties', + check_str=base.RULE_ADMIN, + description='Policy rule for Resource Properties API.', + operations=[ + { + 'path': ('/{api_version}/os-hosts/resource_properties/' + '{property_name}'), + 'method': 'PATCH' + } + ] + ), ] diff --git a/blazar/policy.py b/blazar/policy.py index 5714cc0dc..f465e807f 100644 --- a/blazar/policy.py +++ b/blazar/policy.py @@ -25,6 +25,7 @@ from blazar import context from blazar import exceptions from blazar import policies +from blazar.db import api as db_api CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -56,6 +57,12 @@ def init(): _ENFORCER.register_defaults(policies.list_rules()) +def register_plugin_opts(rules): + global _ENFORCER + init() + _ENFORCER.register_defaults(rules) + + def set_rules(data, default_rule=None): default_rule = default_rule or CONF.policy_default_rule if not _ENFORCER: @@ -92,6 +99,10 @@ def enforce(context, action, target, do_raise=True): init() credentials = context.to_dict() + # NOTE(jasonandersonatuchicago): Keep for backwards compabitility with + # deployments using the old %(project_id)s policy syntax. + credentials['project_id'] = credentials.get('project') + credentials['user_id'] = credentials.get('user') # Add the exceptions arguments if asked to do a raise extra = {} @@ -102,20 +113,41 @@ def enforce(context, action, target, do_raise=True): **extra) +def check_enforcement( + extension, action=None, api='blazar', ctx=None, target=None): + cur_ctx = ctx or context.current() + tgt = target or {'project': cur_ctx.project_id, + 'user': cur_ctx.user_id, + # NOTE(jasonandersonatuchicago): Keep for + # backwards compabitility with deployments using + # the old %(project_id)s policy syntax. + 'project_id': cur_ctx.project_id, + 'user_id': cur_ctx.user_id} + if action is None: + act = '%s:%s' % (api, extension) + else: + act = '%s:%s:%s' % (api, extension, action) + enforce(cur_ctx, act, tgt) + + def authorize(extension, action=None, api='blazar', ctx=None, target=None): def decorator(func): - @functools.wraps(func) def wrapped(self, *args, **kwargs): - cur_ctx = ctx or context.current() - tgt = target or {'project_id': cur_ctx.project_id, - 'user_id': cur_ctx.user_id} - if action is None: - act = '%s:%s' % (api, extension) - else: - act = '%s:%s:%s' % (api, extension, action) - enforce(cur_ctx, act, tgt) + new_target = target + if new_target is None: + obj = None + if kwargs.get("lease_id"): + obj = db_api.lease_get(kwargs.get("lease_id")) + if obj: + new_target = { + 'project': obj.get("project_id"), + 'user': obj.get("user_id"), + 'project_id': obj.get("project_id"), + 'user_id': obj.get("user_id"), + } + check_enforcement(extension, action, api, ctx, new_target) return func(self, *args, **kwargs) return wrapped diff --git a/blazar/status.py b/blazar/status.py index 805f46be2..17cc48ab8 100644 --- a/blazar/status.py +++ b/blazar/status.py @@ -11,6 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from functools import wraps + from oslo_log import log as logging from blazar.db import api as db_api @@ -179,7 +181,7 @@ def is_stable(cls, lease_id): return (lease['status'] in cls.STABLE) @classmethod - def lease_status(cls, transition, result_in): + def lease_status(cls, transition, result_in, non_fatal_exceptions=[]): """Decorator for managing a lease status. This checks and updates a lease status before and after executing a @@ -189,52 +191,71 @@ def lease_status(cls, transition, result_in): decorated function. :param result_in: A tuple of statuses to which a lease transits after executing the decorated function. + :param non_fatal_exceptions: A list of exceptions that are non fatal. + If one is raised during execution, the lease status + will be restored. """ def decorator(func): + @wraps(func) def wrapper(*args, **kwargs): # Update a lease status - lease_id = kwargs['lease_id'] + if "lease_id" in kwargs: + lease_id = kwargs["lease_id"] + else: + lease_id = args[1] lease = db_api.lease_get(lease_id) - if cls.is_valid_transition(lease['status'], + original_status = lease['status'] + if cls.is_valid_transition(original_status, transition, lease_id=lease_id): db_api.lease_update(lease_id, {'status': transition}) LOG.debug('Status of lease %s changed from %s to %s.', - lease_id, lease['status'], transition) + lease_id, original_status, transition) else: - LOG.warn('Aborting %s. ' - 'Invalid lease status transition from %s to %s.', - func.__name__, lease['status'], - transition) + LOG.warning('Aborting %s. ' + 'Invalid lease status transition ' + 'from %s to %s.', + func.__name__, original_status, + transition) raise exceptions.InvalidStatus # Executing the wrapped function try: result = func(*args, **kwargs) except Exception as e: - LOG.exception('Lease %s went into ERROR status. %s', - lease_id, str(e)) - db_api.lease_update(lease_id, - {'status': cls.ERROR}) + is_non_fatal = any( + [isinstance(e, non_fatal_type) + for non_fatal_type in non_fatal_exceptions]) + if is_non_fatal: + LOG.exception('Non-fatal exception during transition ' + 'of lease %s', lease_id) + db_api.lease_update(lease_id, + {'status': original_status}) + else: + LOG.exception('Lease %s went into ERROR status. %s', + lease_id, str(e)) + db_api.lease_update(lease_id, + {'status': cls.ERROR}) raise e - - # Update a lease status if it exists - if db_api.lease_get(lease_id): - next_status = cls.derive_stable_status(lease_id) - if (next_status in result_in - and cls.is_valid_transition(transition, + else: + # Update a lease status if it exists + if db_api.lease_get(lease_id): + next_status = cls.derive_stable_status(lease_id) + if (next_status in result_in and + cls.is_valid_transition(transition, next_status, lease_id=lease_id)): - db_api.lease_update(lease_id, - {'status': next_status}) - LOG.debug('Status of lease %s changed from %s to %s.', - lease_id, transition, next_status) - else: - LOG.error('Lease %s went into ERROR status.', - lease_id) - db_api.lease_update(lease_id, {'status': cls.ERROR}) - raise exceptions.InvalidStatus + db_api.lease_update(lease_id, + {'status': next_status}) + LOG.debug('Status of lease %s changed from %s to ' + '%s.', lease_id, transition, next_status) + else: + LOG.error('Lease %s went into ERROR status.', + lease_id) + db_api.lease_update(lease_id, + {'status': cls.ERROR}) + raise exceptions.InvalidStatus return result return wrapper diff --git a/blazar/tests/__init__.py b/blazar/tests/__init__.py index ad2e15e15..1579ab9ad 100644 --- a/blazar/tests/__init__.py +++ b/blazar/tests/__init__.py @@ -17,10 +17,10 @@ import tempfile import testscenarios -from oslo_config import cfg from oslo_log import log as logging from oslotest import base +from blazar import config as cfg from blazar import context from blazar.db.sqlalchemy import api as db_api from blazar.db.sqlalchemy import facade_wrapper @@ -86,3 +86,11 @@ def setUp(self): _DB_CACHE = Database() self.useFixture(_DB_CACHE) + + +class FakeServiceCatalog(object): + def __init__(self, catalog): + self._catalog = catalog + + def normalize_catalog(self): + return self._catalog diff --git a/blazar/tests/api/__init__.py b/blazar/tests/api/__init__.py index 18d5325a9..640233f09 100644 --- a/blazar/tests/api/__init__.py +++ b/blazar/tests/api/__init__.py @@ -15,6 +15,7 @@ """Base classes for API tests.""" from keystonemiddleware import fixture +from oslo_log import log as logging from oslo_utils import uuidutils import pecan import pecan.testing @@ -25,8 +26,13 @@ from blazar.manager.oshosts import rpcapi as hosts_rpcapi from blazar import tests +LOG = logging.getLogger(__name__) + PATH_PREFIX = '/v2' +FAKE_PROJECT = '981b767265174c108bc5a61185b748ac' +FAKE_USER = 'b4fdb2fff13545ceb751295096cc18ee' + class APITest(tests.TestCase): """Used for unittests tests of Pecan controllers.""" @@ -40,11 +46,12 @@ def setUp(self): def fake_ctx_from_headers(headers): if not headers: return context.BlazarContext( - user_id='fake', project_id='fake', roles=['member']) + user_id=FAKE_USER, project_id=FAKE_PROJECT, + roles=['member']) roles = headers.get('X-Roles', str('member')).split(',') return context.BlazarContext( - user_id=headers.get('X-User-Id', 'fake'), - project_id=headers.get('X-Project-Id', 'fake'), + user_id=headers.get('X-User-Id', FAKE_USER), + project_id=headers.get('X-Project-Id', FAKE_PROJECT), auth_token=headers.get('X-Auth-Token', None), service_catalog=None, user_name=headers.get('X-User-Name', 'fake'), @@ -123,7 +130,7 @@ def _request_json(self, path, params, expect_errors=False, headers=None, :param path_prefix: prefix of the url path """ full_path = path_prefix + path - print('%s: %s %s' % (method.upper(), full_path, params)) + LOG.debug('%s: %s %s' % (method.upper(), full_path, params)) response = getattr(self.app, "%s_json" % method)( str(full_path), params=params, @@ -132,7 +139,7 @@ def _request_json(self, path, params, expect_errors=False, headers=None, extra_environ=extra_environ, expect_errors=expect_errors ) - print('GOT:%s' % response) + LOG.debug('GOT:%s' % response) return response def put_json(self, path, params, expect_errors=False, headers=None, @@ -185,13 +192,13 @@ def delete(self, path, expect_errors=False, headers=None, :param path_prefix: prefix of the url path """ full_path = path_prefix + path - print('DELETE: %s' % (full_path)) + LOG.debug('DELETE: %s' % (full_path)) response = self.app.delete(str(full_path), headers=headers, status=status, extra_environ=extra_environ, expect_errors=expect_errors) - print('GOT:%s' % response) + LOG.debug('GOT:%s' % response) return response def get_json(self, path, expect_errors=False, headers=None, @@ -221,7 +228,7 @@ def get_json(self, path, expect_errors=False, headers=None, all_params.update(params) if q: all_params.update(query_params) - print('GET: %s %r' % (full_path, all_params)) + LOG.debug('GET: %s %r' % (full_path, all_params)) response = self.app.get(full_path, params=all_params, headers=headers, @@ -229,5 +236,5 @@ def get_json(self, path, expect_errors=False, headers=None, expect_errors=expect_errors) if not expect_errors: response = response.json - print('GOT:%s' % response) + LOG.debug('GOT:%s' % response) return response diff --git a/blazar/tests/api/test_context.py b/blazar/tests/api/test_context.py index c2c02046d..4206c6708 100644 --- a/blazar/tests/api/test_context.py +++ b/blazar/tests/api/test_context.py @@ -19,7 +19,6 @@ from werkzeug import wrappers from blazar.api import context as api_context -from blazar import context from blazar import exceptions from blazar import tests @@ -33,9 +32,10 @@ def setUp(self): 'X-Project-Id': uuidsentinel.project_id, 'X-Auth-Token': '111-111-111', 'X-User-Name': 'user_name', + 'X-User-Domain-Name': 'user_domain_name', 'X-Project-Name': 'project_name', + 'X-Project-Domain-Name': 'project_domain_name', 'X-Roles': 'user_name0, user_name1'} - self.context = self.patch(context, 'BlazarContext') self.catalog = jsonutils.dump_as_bytes({'nova': 'catalog'}) def test_ctx_from_headers_no_catalog(self): @@ -64,19 +64,24 @@ def test_ctx_from_headers(self): '/v1/leases', headers=self.fake_headers, environ_base=environ_base) - api_context.ctx_from_headers(req.headers) - - self.context.assert_called_once_with( + context = api_context.ctx_from_headers(req.headers) + expected = dict( user_id=uuidsentinel.user_id, roles=['user_name0', 'user_name1'], project_name='project_name', + project_domain_name='project_domain_name', auth_token='111-111-111', service_catalog={'nova': 'catalog'}, project_id=uuidsentinel.project_id, user_name='user_name', + user_domain_name='user_domain_name', request_id='req-' + uuidsentinel.reqid, - global_request_id='req-' + uuidsentinel.globalreqid) + global_request_id='req-' + uuidsentinel.globalreqid + ) + + for k, v in expected.items(): + self.assertEqual(getattr(context, k, None), v) class ContextTestCaseV2(ContextTestCase): @@ -85,14 +90,19 @@ def test_ctx_from_headers(self): self.fake_headers['X-Service-Catalog'] = self.catalog req = webob.Request.blank('/v2/leases') req.headers = self.fake_headers - api_context.ctx_from_headers(req.headers) - - self.context.assert_called_once_with( + context = api_context.ctx_from_headers(req.headers) + expected = dict( user_id=uuidsentinel.user_id, roles=['user_name0', 'user_name1'], project_name='project_name', + project_domain_name='project_domain_name', auth_token='111-111-111', service_catalog={'nova': 'catalog'}, project_id=uuidsentinel.project_id, - user_name='user_name') + user_name='user_name', + user_domain_name='user_domain_name' + ) + + for k, v in expected.items(): + self.assertEqual(getattr(context, k, None), v) diff --git a/blazar/tests/api/v1/leases/test_v1_0.py b/blazar/tests/api/v1/leases/test_v1_0.py index bd74a4208..09d964865 100644 --- a/blazar/tests/api/v1/leases/test_v1_0.py +++ b/blazar/tests/api/v1/leases/test_v1_0.py @@ -190,6 +190,7 @@ def test_delete(self): res = c.delete('/v1/leases/{0}'.format(self.lease_uuid), headers=self.headers) res_id = res.headers.get(id.HTTP_RESP_HEADER_REQUEST_ID) - self.assertEqual(204, res.status_code) + # self.assertEqual(204, res.status_code) + self.assertEqual(200, res.status_code) self.assertIn(id.HTTP_RESP_HEADER_REQUEST_ID, res.headers) self.assertThat(res_id, matchers.StartsWith('req-')) diff --git a/blazar/tests/api/v1/oshosts/test_v1_0.py b/blazar/tests/api/v1/oshosts/test_v1_0.py index c54a31fb8..8abf5d9c2 100644 --- a/blazar/tests/api/v1/oshosts/test_v1_0.py +++ b/blazar/tests/api/v1/oshosts/test_v1_0.py @@ -15,7 +15,9 @@ import ddt import flask +from oslo_config import cfg from oslo_utils import uuidutils +from stevedore import enabled from testtools import matchers from oslo_middleware import request_id as id @@ -27,6 +29,8 @@ from blazar.api.v1 import request_id from blazar.api.v1 import request_log from blazar import context +from blazar.manager import service +from blazar.plugins.oshosts import host_plugin from blazar import tests @@ -81,6 +85,18 @@ class OsHostAPITestCase(tests.TestCase): def setUp(self): super(OsHostAPITestCase, self).setUp() + + class FakeExtension(): + def __init__(self, name, plugin): + self.name = name + self.plugin = plugin + ext_manager = self.patch(enabled, 'EnabledExtensionManager') + ext_manager.return_value.extensions = [ + FakeExtension('physical:host', host_plugin.PhysicalHostPlugin), + ] + cfg.CONF.set_override('plugins', ['physical:host'], group='manager') + service.get_plugins.cache_clear() + self.app = make_app() self.headers = {'Accept': 'application/json', 'OpenStack-API-Version': 'reservation 1.0'} @@ -100,6 +116,14 @@ def setUp(self): self.list_allocations = self.patch(service_api.API, 'list_allocations') self.get_allocations = self.patch(service_api.API, 'get_allocations') + self.list_resource_properties = self.patch(service_api.API, + 'list_resource_properties') + self.update_resource_property = self.patch(service_api.API, + 'update_resource_property') + + def tearDown(self): + super(OsHostAPITestCase, self).tearDown() + cfg.CONF.clear_override('plugins', 'manager') def _assert_response(self, actual_resp, expected_status_code, expected_resp_body, key='host', @@ -205,7 +229,8 @@ def test_delete(self): res = c.delete('/v1/{0}'.format(self.host_id), headers=self.headers) res_id = res.headers.get(id.HTTP_RESP_HEADER_REQUEST_ID) - self.assertEqual(204, res.status_code) + # self.assertEqual(204, res.status_code) + self.assertEqual(200, res.status_code) self.assertIn(id.HTTP_RESP_HEADER_REQUEST_ID, res.headers) self.assertThat(res_id, matchers.StartsWith('req-')) @@ -237,3 +262,20 @@ def test_allocation_get_with_query_params(self, query_params): res = c.get('/v1/{0}/allocation?{1}'.format( self.host_id, query_params), headers=self.headers) self._assert_response(res, 200, {}, key='allocation') + + def test_resource_properties_list(self): + with self.app.test_client() as c: + self.list_resource_properties.return_value = [] + res = c.get('/v1/properties', headers=self.headers) + self._assert_response(res, 200, [], key='resource_properties') + + def test_resource_property_update(self): + resource_property = 'fake_property' + resource_property_body = {'private': True} + + with self.app.test_client() as c: + + res = c.patch('/v1/properties/{0}'.format(resource_property), + json=resource_property_body, + headers=self.headers) + self._assert_response(res, 200, {}, 'resource_property') diff --git a/blazar/tests/api/v1/test_app.py b/blazar/tests/api/v1/test_app.py index 0338d16bf..58255fbe3 100644 --- a/blazar/tests/api/v1/test_app.py +++ b/blazar/tests/api/v1/test_app.py @@ -20,6 +20,7 @@ from blazar.api.v1 import app from blazar.api.v1.leases import v1_0 as lease_api_v1_0 +from blazar.api.v1.networks import v1_0 as network_api_v1_0 from blazar.api.v1.oshosts import v1_0 as host_api_v1_0 from blazar.api.v1 import utils as api_utils from blazar import tests @@ -88,9 +89,32 @@ def setUp(self): self.fake_blueprint = self.patch(self.flask.Flask, 'register_blueprint') + def tearDown(self): + super(AppTestCaseForHostsPlugin, self).tearDown() + cfg.CONF.clear_override('plugins', 'manager') + def test_make_app_with_host_plugin(self): self.app.make_app() self.fake_blueprint.assert_any_call(self.lease_api_v1_0.rest, url_prefix='/v1') self.fake_blueprint.assert_any_call(self.host_api_v1_0.rest, url_prefix='/v1/os-hosts') + + +class AppTestCaseForNetworksPlugin(tests.TestCase): + + def setUp(self): + super(AppTestCaseForNetworksPlugin, self).setUp() + + cfg.CONF.set_override( + 'plugins', ['network.plugin'], 'manager') + self.app = app + self.network_api_v1_0 = network_api_v1_0 + self.flask = flask + self.fake_blueprint = self.patch(self.flask.Flask, + 'register_blueprint') + + def test_make_app_with_network_plugin(self): + self.app.make_app() + self.fake_blueprint.assert_any_call(self.network_api_v1_0.rest, + url_prefix='/v1/networks') diff --git a/blazar/tests/api/v1/test_utils.py b/blazar/tests/api/v1/test_utils.py index dc50b2339..34596098a 100644 --- a/blazar/tests/api/v1/test_utils.py +++ b/blazar/tests/api/v1/test_utils.py @@ -35,7 +35,8 @@ def setUp(self): self.utils = utils self.flask = flask - self.rest = self.patch(self.utils, "Rest") + self.rest = utils.Rest('v1_0', __name__) + self.patch(self.rest, "_mroute") self.response = self.patch(self.flask, "Response") self.abort = self.patch(self.flask, "abort") self.request = self.patch(self.flask, "request") @@ -44,19 +45,20 @@ def setUp(self): def test_get(self): self.rest.get('rule', status_code=200) - self.rest._mroute.called_once_with('GET', 'rule', 200) + self.rest._mroute.assert_called_once_with('GET', 'rule', 200, + query=False) def test_post(self): self.rest.post('rule', status_code=201) - self.rest._mroute.called_once_with('POST', 'rule', 201) + self.rest._mroute.assert_called_once_with('POST', 'rule', 201) def test_put(self): self.rest.put('rule', status_code=200) - self.rest._mroute.called_once_with('PUT', 'rule', 200) + self.rest._mroute.assert_called_once_with('PUT', 'rule', 200) def test_delete(self): self.rest.delete('rule', status_code=204) - self.rest._mroute.called_once_with('DELETE', 'rule', 204) + self.rest._mroute.assert_called_once_with('DELETE', 'rule', 204) def test_route_ok(self): pass @@ -104,7 +106,7 @@ def test_get_request_args(self): def test_abort_and_log(self): self.utils.abort_and_log(400, "Funny error") - self.abort.called_once_with(400, description="Funny error") + self.abort.assert_called_once_with(400, description="Funny error") def test_render_error_message(self): render = self.patch(self.utils, 'render') diff --git a/blazar/tests/api/v1/test_validation.py b/blazar/tests/api/v1/test_validation.py index fa205d66e..0eb2d735d 100644 --- a/blazar/tests/api/v1/test_validation.py +++ b/blazar/tests/api/v1/test_validation.py @@ -44,7 +44,7 @@ def trap(fake_id): def test_check_false(self): fake_get = self.patch( - self.s_api.API, 'get_lease').side_effect = self.exc.NotFound() + self.s_api.API, 'get_lease').side_effect = self.exc.NotFound(message="test message") @self.v_api.check_exists(fake_get, self.fake_id) def trap(fake_id): diff --git a/blazar/tests/api/v2/test_leases.py b/blazar/tests/api/v2/test_leases.py index 1bfac7c51..c93991eb9 100644 --- a/blazar/tests/api/v2/test_leases.py +++ b/blazar/tests/api/v2/test_leases.py @@ -28,9 +28,8 @@ def fake_lease(**kw): 'end_date': kw.get('end_date', '2014-02-01 13:37'), 'trust_id': kw.get('trust_id', '35b17138b3644e6aa1318f3099c5be68'), - 'user_id': kw.get('user_id', 'efd8780712d24b389c705f5c2ac427ff'), - 'project_id': kw.get('project_id', - 'bd9431c18d694ad3803a8d4a6b89fd36'), + 'user_id': kw.get('user_id', api.FAKE_USER), + 'project_id': kw.get('project_id', api.FAKE_PROJECT), 'reservations': kw.get('reservations', [ { 'resource_id': '1234', diff --git a/blazar/tests/db/migration/test_migration_graph.py b/blazar/tests/db/migration/test_migration_graph.py new file mode 100644 index 000000000..8b170b5d3 --- /dev/null +++ b/blazar/tests/db/migration/test_migration_graph.py @@ -0,0 +1,68 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Static checks on the alembic revision graph. + +These validate the migration scripts themselves, without a database. +They exist because `alembic upgrade head` refuses to run when the graph has +more than one head, which breaks `blazar-db-manage upgrade head` at deploy +time, but is not otherwise caught by unit-tests. +""" + +import os + +from alembic import config as alembic_config +from alembic import script as alembic_script + +from blazar.db import migration +from blazar.tests import TestCase + + +def _find_alembic_conf(): + """Build an alembic Config for blazar's migration repository.""" + path = os.path.join( + os.path.abspath(os.path.dirname(migration.__file__)), 'alembic.ini') + config = alembic_config.Config(path) + config.attributes['configure_logger'] = False + return config + + +class TestMigrationGraph(TestCase): + + def setUp(self): + super(TestMigrationGraph, self).setUp() + self.script = alembic_script.ScriptDirectory.from_config( + _find_alembic_conf()) + + def test_single_base_revision(self): + """Ensure we only have a single base revision. + + There's no good reason for us to have diverging history, so + validate that only one base revision exists. If this fails for + your change, look for migrations that do not have a 'Revises' + line in them. + """ + self.assertEqual(1, len(self.script.get_bases())) + + def test_single_head_revision(self): + """Ensure we only have a single head revision. + + There's no good reason for us to have diverging history, so validate + that only one head revision exists. This will prevent merge conflicts + adding additional head revision points. If this fails for your change, + look for migrations with the same 'revises' line in them. + """ + heads = self.script.get_heads() + self.assertEqual( + 1, len(heads), 'expected a single head, found: %s' + % ', '.join(heads)) diff --git a/blazar/tests/db/sqlalchemy/test_sqlalchemy_api.py b/blazar/tests/db/sqlalchemy/test_sqlalchemy_api.py index 2e4886c95..2667f5864 100644 --- a/blazar/tests/db/sqlalchemy/test_sqlalchemy_api.py +++ b/blazar/tests/db/sqlalchemy/test_sqlalchemy_api.py @@ -193,7 +193,7 @@ def _get_fake_host_extra_capabilities(id=None, computehost_id = _get_fake_random_uuid() return {'id': id, 'computehost_id': computehost_id, - 'capability_name': name, + 'property_name': name, 'capability_value': value} @@ -332,7 +332,7 @@ def test_lease_list(self): self.assertEqual(['1', '2'], db_api.lease_list()) def test_lease_update(self): - """Update both start_time and name and check lease has been updated.""" + """Update both start_date and name and check lease has been updated.""" result = _create_physical_lease() result = db_api.lease_update(result['id'], values={'name': 'lease_renamed'}) @@ -507,6 +507,12 @@ def test_search_for_hosts_by_extra_capability(self): """Create one host and test extra capability queries.""" # We create a first host, with extra capabilities db_api.host_create(_get_fake_host_values(id=1)) + db_api.resource_property_create(dict( + id='a', resource_type='physical:host', private=False, + property_name='vgpu')) + db_api.resource_property_create(dict( + id='b', resource_type='physical:host', private=False, + property_name='nic_model')) db_api.host_extra_capability_create( _get_fake_host_extra_capabilities(computehost_id=1)) db_api.host_extra_capability_create(_get_fake_host_extra_capabilities( @@ -533,6 +539,84 @@ def test_search_for_hosts_by_extra_capability(self): db_api.host_get_all_by_queries(['nic_model == ACME Model A']) )) + def test_search_for_hosts_by_capability_subset(self): + """A capability filter returns the right subset of several hosts.""" + _ = db_api.host_create(_get_fake_host_values()) + with_16 = db_api.host_create(_get_fake_host_values()) + with_32 = db_api.host_create(_get_fake_host_values()) + db_api.host_extra_capability_create( + _get_fake_host_extra_capabilities( + computehost_id=with_16['id'], + name='vgpu', + value='16', + ) + ) + db_api.host_extra_capability_create( + _get_fake_host_extra_capabilities( + computehost_id=with_32['id'], + name='vgpu', + value='32', + ) + ) + + match_eq = db_api.host_get_all_by_queries(['vgpu == 16']) + self.assertEqual( + [with_16['id']], + [h['id'] for h in match_eq] + ) + + # != matches the host with vgpu=32, but NOT the host with no vgpu extra + # capability. `!=` will only match against hosts that have the property + # defined. + match_ne = db_api.host_get_all_by_queries(['vgpu != 16']) + self.assertEqual( + [with_32['id']], + [h['id'] for h in match_ne] + ) + + def test_search_for_hosts_by_two_capabilities(self): + """Two capability filters are ANDed together.""" + both = db_api.host_create(_get_fake_host_values()) + vgpu_only = db_api.host_create(_get_fake_host_values()) + db_api.host_extra_capability_create(_get_fake_host_extra_capabilities( + computehost_id=both['id'], name='vgpu', value='16')) + db_api.host_extra_capability_create(_get_fake_host_extra_capabilities( + computehost_id=both['id'], name='nic_model', value='ACME Model A')) + db_api.host_extra_capability_create(_get_fake_host_extra_capabilities( + computehost_id=vgpu_only['id'], name='vgpu', value='16')) + + result = db_api.host_get_all_by_queries( + ['vgpu == 16', 'nic_model == ACME Model A']) + self.assertEqual([both['id']], [h['id'] for h in result]) + + def test_search_for_hosts_by_capability_then_column(self): + """A capability predicate first must AND with a following column.""" + host = db_api.host_create(_get_fake_host_values(mem=8192)) + db_api.host_extra_capability_create(_get_fake_host_extra_capabilities( + computehost_id=host['id'], name='vgpu', value='16')) + + match = db_api.host_get_all_by_queries( + ['vgpu == 16', 'memory_mb >= 2048']) + self.assertEqual([host['id']], [h['id'] for h in match]) + + no_match = db_api.host_get_all_by_queries( + ['vgpu == 16', 'memory_mb < 2048']) + self.assertEqual([], [h['id'] for h in no_match]) + + def test_resource_properties_list(self): + """Create one host and test extra capability queries.""" + # We create a first host, with extra capabilities + db_api.host_create(_get_fake_host_values(id=1)) + db_api.resource_property_create(dict( + id='a', resource_type='physical:host', private=False, + property_name='vgpu')) + db_api.host_extra_capability_create( + _get_fake_host_extra_capabilities(computehost_id=1)) + + result = db_api.resource_properties_list('physical:host') + + self.assertListEqual(result, [('vgpu', False, '2', False)]) + def test_search_for_hosts_by_composed_queries(self): """Create one host and test composed queries.""" @@ -580,9 +664,13 @@ def test_delete_host(self): db_api.host_destroy, 2) def test_create_host_extra_capability(self): - result = db_api.host_extra_capability_create( - _get_fake_host_extra_capabilities(id=1)) - self.assertEqual(result['id'], _get_fake_host_values(id='1')['id']) + db_api.resource_property_create(dict( + id='id', resource_type='physical:host', private=False, + property_name='vgpu')) + result, _ = db_api.host_extra_capability_create( + _get_fake_host_extra_capabilities(id=1, name='vgpu')) + + self.assertEqual(result.id, _get_fake_host_values(id='1')['id']) def test_create_duplicated_host_extra_capability(self): db_api.host_extra_capability_create( @@ -594,8 +682,8 @@ def test_create_duplicated_host_extra_capability(self): def test_get_host_extra_capability_per_id(self): db_api.host_extra_capability_create( _get_fake_host_extra_capabilities(id='1')) - result = db_api.host_extra_capability_get('1') - self.assertEqual('1', result['id']) + result, _ = db_api.host_extra_capability_get('1') + self.assertEqual('1', result.id) def test_host_extra_capability_get_all_per_host(self): db_api.host_extra_capability_create( @@ -609,8 +697,8 @@ def test_update_host_extra_capability(self): db_api.host_extra_capability_create( _get_fake_host_extra_capabilities(id='1')) db_api.host_extra_capability_update('1', {'capability_value': '2'}) - res = db_api.host_extra_capability_get('1') - self.assertEqual('2', res['capability_value']) + res, _ = db_api.host_extra_capability_get('1') + self.assertEqual('2', res.capability_value) def test_delete_host_extra_capability(self): db_api.host_extra_capability_create( @@ -629,6 +717,77 @@ def test_host_extra_capability_get_all_per_name(self): db_api.host_extra_capability_get_all_per_name('1', 'bad')) + # Resource Inventory + + def test_host_resource_inventory_create(self): + db_api.host_create(_get_fake_host_values(id=1)) + fake_inventory_values = { + 'id': _get_fake_random_uuid(), + 'computehost_id': '1', + 'resource_class': 'VCPU', + 'total': 10, + 'reserved': 2, + 'min_unit': 1, + 'max_unit': 1, + 'step_size': 1, + 'allocation_ratio': 1.0 + } + + db_api.host_resource_inventory_create(fake_inventory_values) + + host = db_api.host_get(1) + actual = host.computehost_resource_inventories[0].to_dict() + self.assertIsNotNone(actual["created_at"]) + del actual["created_at"] + expected = fake_inventory_values.copy() + expected["updated_at"] = None + self.assertDictEqual(expected, actual) + + def test_host_resource_inventory_get_all_per_host(self): + db_api.host_create(_get_fake_host_values(id=1)) + fake_inventory_values = { + 'id': _get_fake_random_uuid(), + 'computehost_id': '1', + 'resource_class': 'VCPU', + 'total': 10, + 'reserved': 2, + 'min_unit': 1, + 'max_unit': 1, + 'step_size': 1, + 'allocation_ratio': 1.0 + } + db_api.host_resource_inventory_create(fake_inventory_values) + + result = db_api.host_resource_inventory_get_all_per_host("1") + + result = list(result) + self.assertEqual(1, len(result)) + first_result = result[0].to_dict() + expected = fake_inventory_values.copy() + expected["updated_at"] = None + expected["created_at"] = first_result["created_at"] + self.assertDictEqual(expected, first_result) + + # Trait + + def test_host_traits_create(self): + db_api.host_create(_get_fake_host_values(id=1)) + fake_trait_values = { + 'id': _get_fake_random_uuid(), + 'computehost_id': '1', + 'trait': 'HW_CPU_X86_AVX' + } + + db_api.host_trait_create(fake_trait_values) + + host = db_api.host_get(1) + actual = host.computehost_traits[0].to_dict() + self.assertIsNotNone(actual["created_at"]) + del actual["created_at"] + expected = fake_trait_values.copy() + expected["updated_at"] = None + self.assertDictEqual(expected, actual) + # Instance reservation def check_instance_reservation_values(self, expected, reservation_id): diff --git a/blazar/tests/db/sqlalchemy/test_utils.py b/blazar/tests/db/sqlalchemy/test_utils.py index 8bb5df04c..8f02d56e7 100644 --- a/blazar/tests/db/sqlalchemy/test_utils.py +++ b/blazar/tests/db/sqlalchemy/test_utils.py @@ -38,7 +38,8 @@ def _get_fake_phys_reservation_values(lease_id=_get_fake_lease_uuid(), resource_id='1234'): return {'lease_id': lease_id, 'resource_id': resource_id, - 'resource_type': 'physical:host'} + 'resource_type': 'physical:host', + 'status': 'active'} def _get_fake_inst_reservation_values(lease_id=_get_fake_lease_uuid(), @@ -65,10 +66,30 @@ def _get_fake_phys_lease_values(id=_get_fake_lease_uuid(), 'reservations': [_get_fake_phys_reservation_values( lease_id=id, resource_id=resource_id)], - 'events': [] + 'events': [], + 'project_id': 'fake_project', + 'status': 'ACTIVE' } +def _create_allocation_dicts(lease_ids): + allocs = [] + + for lease_id in lease_ids: + reservations = db_api.reservation_get_all_by_lease_id( + lease_id) + + for reservation in reservations: + allocs.append( + { + 'id': reservation['id'], + 'host_ids': [ + x['compute_host_id'] for x + in db_api.host_allocation_get_all_by_values( + reservation_id=reservation['id'])]}) + return allocs + + def _create_physical_lease(values=_get_fake_phys_lease_values(), random=False): """Creating fake lease having a single physical resource.""" @@ -86,6 +107,15 @@ def _create_physical_lease(values=_get_fake_phys_lease_values(), return lease +def _delete_physical_lease(id): + """Delete fake lease.""" + db_api.lease_destroy(id) + + +def _filter_dicts_for_keys(keys, items): + return [{k: v for k, v in x.items() if k in keys} for x in items] + + class SQLAlchemyDBUtilsTestCase(tests.DBTestCase): """Test case for SQLAlchemy DB utils.""" @@ -113,9 +143,17 @@ def _setup_leases(self): start_date=_get_datetime('2030-01-01 13:00'), end_date=_get_datetime('2030-01-01 14:00'), resource_id='r1') + r_del = _get_fake_phys_lease_values( + id='lease_del', + name='fake_phys_lease_r_del', + start_date=_get_datetime('2030-01-01 14:30'), + end_date=_get_datetime('2030-01-01 15:00'), + resource_id='r1') _create_physical_lease(values=r1) _create_physical_lease(values=r2) _create_physical_lease(values=r3) + _create_physical_lease(values=r_del) + _delete_physical_lease(r_del['id']) def check_reservation(self, expect, host_ids, start, end): expect.sort(key=lambda x: x['lease_id']) @@ -259,72 +297,48 @@ def test_get_reservations_by_host_ids(self): self.check_reservation([], ['r4'], '2030-01-01 07:00', '2030-01-01 15:00') - def _create_allocation_tuple(self, lease_id): - reservation = db_api.reservation_get_all_by_lease_id(lease_id)[0] - allocation = db_api.host_allocation_get_all_by_values( - reservation_id=reservation['id'])[0] - return (reservation['id'], - reservation['lease_id'], - allocation['compute_host_id']) - def test_get_reservation_allocations_by_host_ids(self): self._setup_leases() # query all allocations of lease1, lease2 and lease3 - expected = [ - self._create_allocation_tuple('lease1'), - self._create_allocation_tuple('lease2'), - self._create_allocation_tuple('lease3'), - ] + expected = _create_allocation_dicts(['lease1', 'lease2', 'lease3']) ret = db_utils.get_reservation_allocations_by_host_ids( ['r1', 'r2'], '2030-01-01 08:00', '2030-01-01 15:00') - self.assertListEqual(expected, ret) + self.assertListEqual( + expected, _filter_dicts_for_keys(['id', 'host_ids'], ret)) # query allocations of lease2 and lease3 - expected = [ - self._create_allocation_tuple('lease2'), - self._create_allocation_tuple('lease3'), - ] + expected = _create_allocation_dicts(['lease2', 'lease3']) ret = db_utils.get_reservation_allocations_by_host_ids( ['r1', 'r2'], '2030-01-01 11:30', '2030-01-01 15:00') - self.assertListEqual(expected, ret) + self.assertListEqual( + expected, _filter_dicts_for_keys(['id', 'host_ids'], ret)) def test_get_reservation_allocations_by_host_ids_with_lease_id(self): self._setup_leases() # query all allocations of lease1, lease2 and lease3 - expected = [ - self._create_allocation_tuple('lease1'), - ] + expected = _create_allocation_dicts(['lease1']) ret = db_utils.get_reservation_allocations_by_host_ids( ['r1', 'r2'], '2030-01-01 08:00', '2030-01-01 15:00', 'lease1') - self.assertListEqual(expected, ret) + self.assertListEqual( + expected, _filter_dicts_for_keys(['id', 'host_ids'], ret)) def test_get_reservation_allocations_by_host_ids_with_reservation_id(self): self._setup_leases() reservation1 = db_api.reservation_get_all_by_lease_id('lease1')[0] # query allocations of lease1 - expected = [ - self._create_allocation_tuple('lease1'), - ] + expected = _create_allocation_dicts(['lease1']) ret = db_utils.get_reservation_allocations_by_host_ids( ['r1', 'r2'], '2030-01-01 08:00', '2030-01-01 15:00', reservation_id=reservation1['id']) - self.assertListEqual(expected, ret) - - def test_get_plugin_reservation_with_host(self): - patch_host_reservation_get = self.patch(db_api, 'host_reservation_get') - patch_host_reservation_get.return_value = { - 'id': 'id', - 'reservation_id': 'reservation-id', - } - db_utils.get_plugin_reservation('physical:host', 'id-1') - patch_host_reservation_get.assert_called_once_with('id-1') + self.assertListEqual( + expected, _filter_dicts_for_keys(['id', 'host_ids'], ret)) def test_get_plugin_reservation_with_instance(self): patch_inst_reservation_get = self.patch(db_api, @@ -336,6 +350,26 @@ def test_get_plugin_reservation_with_instance(self): db_utils.get_plugin_reservation('virtual:instance', 'id-1') patch_inst_reservation_get.assert_called_once_with('id-1') + def test_get_plugin_reservation_with_device(self): + patch_dev_reservation_get = self.patch(db_api, + 'device_reservation_get') + patch_dev_reservation_get.return_value = { + 'id': 'id', + 'reservation_id': 'reservation-id', + } + db_utils.get_plugin_reservation('device', 'id-1') + patch_dev_reservation_get.assert_called_once_with('id-1') + + def test_get_plugin_reservation_with_network(self): + patch_net_reservation_get = self.patch(db_api, + 'network_reservation_get') + patch_net_reservation_get.return_value = { + 'id': 'id', + 'reservation_id': 'reservation-id', + } + db_utils.get_plugin_reservation('network', 'id-1') + patch_net_reservation_get.assert_called_once_with('id-1') + def test_get_plugin_reservation_with_invalid(self): self.assertRaises(mgr_exceptions.UnsupportedResourceType, db_utils.get_plugin_reservation, 'invalid', 'id1') diff --git a/blazar/tests/enforcement/filters/test_external_service_filter.py b/blazar/tests/enforcement/filters/test_external_service_filter.py new file mode 100644 index 000000000..32ce6991a --- /dev/null +++ b/blazar/tests/enforcement/filters/test_external_service_filter.py @@ -0,0 +1,256 @@ +# Copyright (c) 2022 Radosław Piliszek +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import json +from unittest import mock + +from blazar.enforcement.exceptions import ExternalServiceFilterException +from blazar.enforcement.filters import external_service_filter +from blazar.tests import TestCase + +from oslo_config.cfg import CONF + + +class FakeResponse204(): + status_code = 204 + + +class FakeResponse403Empty(): + status_code = 403 + content = "irrelevant_but_logged" + + @staticmethod + def json(): + return {} + + +class FakeJSONDecodeError(Exception): + pass + + +class FakeResponse403InvalidJSON(): + status_code = 403 + content = "NOT_JSON" + + @staticmethod + def json(): + raise FakeJSONDecodeError() + + +class FakeResponse403WithMessage(): + status_code = 403 + content = "irrelevant" + + @staticmethod + def json(): + return {"message": "Hello!"} + + +class FakeResponse500(): + status_code = 500 + content = "ALL_YOUR_BUGS_BELONG_TO_US" + + +class ISODateTimeEncoderTestCase(TestCase): + + def test_json_date(self): + dt = datetime.datetime(2022, 9, 8, 13, 31, 44, 12345) + obj = {"datetime": dt} + x = json.dumps(obj, cls=external_service_filter.ISODateTimeEncoder) + self.assertEqual('{"datetime": "2022-09-08T13:31:44.012345"}', x) + + def test_json_with_tz(self): + tz = datetime.timezone(datetime.timedelta(hours=2)) + dt = datetime.datetime(2022, 9, 8, 13, 31, 44, 12345, tz) + obj = {"datetime": dt} + x = json.dumps(obj, cls=external_service_filter.ISODateTimeEncoder) + self.assertEqual('{"datetime": "2022-09-08T13:31:44.012345+02:00"}', x) + + +class ConfiguringExternalServiceFilterTestCase(TestCase): + def setUp(self): + super().setUp() + + external_service_filter.ExternalServiceFilter.register_opts(CONF) + + def test_basic_misconfiguration(self): + self.assertRaises(external_service_filter.ExternalServiceMisconfigured, + external_service_filter.ExternalServiceFilter, CONF) + + def test_bad_url(self): + CONF.set_override( + 'external_service_base_endpoint', 'this_url_cOuLDnOtBeWoRsE', + group='enforcement') + self.addCleanup(CONF.clear_override, 'external_service_base_endpoint', + group='enforcement') + + self.assertRaises(external_service_filter.ExternalServiceMisconfigured, + external_service_filter.ExternalServiceFilter, CONF) + + def test_check_create_endpoint_is_enough(self): + CONF.set_override( + 'external_service_check_create_endpoint', 'http://localhost', + group='enforcement') + self.addCleanup(CONF.clear_override, + 'external_service_check_create_endpoint', + group='enforcement') + + external_service_filter.ExternalServiceFilter(CONF) + + def test_check_updaye_endpoint_is_enough(self): + CONF.set_override( + 'external_service_check_update_endpoint', 'http://localhost', + group='enforcement') + self.addCleanup(CONF.clear_override, + 'external_service_check_update_endpoint', + group='enforcement') + + external_service_filter.ExternalServiceFilter(CONF) + + def test_on_end_endpoint_is_enough(self): + CONF.set_override( + 'external_service_on_end_endpoint', 'http://localhost', + group='enforcement') + self.addCleanup(CONF.clear_override, + 'external_service_on_end_endpoint', + group='enforcement') + + external_service_filter.ExternalServiceFilter(CONF) + + +class ExternalServiceFilterTestCase(TestCase): + def setUp(self): + super().setUp() + + external_service_filter.ExternalServiceFilter.register_opts(CONF) + + CONF.set_override( + 'external_service_base_endpoint', 'http://localhost', + group='enforcement') + CONF.set_override( + 'external_service_token', 'test_token', group='enforcement') + self.addCleanup(CONF.clear_override, 'external_service_base_endpoint', + group='enforcement') + + self.filter = external_service_filter.ExternalServiceFilter(CONF) + + self.ctx = { + "is_context": True + } + + self.lease = { + "is_lease": True + } + + self.old_lease = { + "is_old_lease": True + } + + @mock.patch("requests.post") + def test_check_create_allowed(self, post_mock): + post_mock.return_value = FakeResponse204() + self.filter.check_create(self.ctx, self.lease) + post_mock.assert_called_with( + "http://localhost/check-create", + headers={'Content-Type': 'application/json', 'X-Auth-Token': 'test_token'}, + data='{"context": {"is_context": true}, ' + '"lease": {"is_lease": true}}') + + @mock.patch("requests.post") + def test_check_create_denied(self, post_mock): + post_mock.return_value = FakeResponse403WithMessage() + self.assertRaises(ExternalServiceFilterException, + self.filter.check_create, + self.ctx, self.lease) + post_mock.assert_called_with( + "http://localhost/check-create", + headers={'Content-Type': 'application/json', 'X-Auth-Token': 'test_token'}, + data='{"context": {"is_context": true}, ' + '"lease": {"is_lease": true}}') + + @mock.patch("requests.post") + def test_check_create_failed(self, post_mock): + post_mock.return_value = FakeResponse403Empty() + self.assertRaises(ExternalServiceFilterException, + self.filter.check_create, + self.ctx, self.lease) + post_mock.assert_called_with( + "http://localhost/check-create", + headers={'Content-Type': 'application/json', 'X-Auth-Token': 'test_token'}, + data='{"context": {"is_context": true}, ' + '"lease": {"is_lease": true}}') + + @mock.patch("requests.post") + def test_check_update_allowed(self, post_mock): + post_mock.return_value = FakeResponse204() + self.filter.check_update(self.ctx, self.old_lease, self.lease) + post_mock.assert_called_with( + "http://localhost/check-update", + headers={'Content-Type': 'application/json', 'X-Auth-Token': 'test_token'}, + data='{"context": {"is_context": true}, ' + '"current_lease": {"is_old_lease": true}, ' + '"lease": {"is_lease": true}}') + + @mock.patch("requests.post") + def test_check_update_denied(self, post_mock): + post_mock.return_value = FakeResponse403WithMessage() + self.assertRaises(ExternalServiceFilterException, + self.filter.check_update, + self.ctx, self.old_lease, self.lease) + post_mock.assert_called_with( + "http://localhost/check-update", + headers={'Content-Type': 'application/json', 'X-Auth-Token': 'test_token'}, + data='{"context": {"is_context": true}, ' + '"current_lease": {"is_old_lease": true}, ' + '"lease": {"is_lease": true}}') + + @mock.patch("requests.post") + @mock.patch("requests.JSONDecodeError", FakeJSONDecodeError) + def test_check_update_failed(self, post_mock): + post_mock.return_value = FakeResponse403InvalidJSON() + self.assertRaises(ExternalServiceFilterException, + self.filter.check_update, + self.ctx, self.old_lease, self.lease) + post_mock.assert_called_with( + "http://localhost/check-update", + headers={'Content-Type': 'application/json', 'X-Auth-Token': 'test_token'}, + data='{"context": {"is_context": true}, ' + '"current_lease": {"is_old_lease": true}, ' + '"lease": {"is_lease": true}}') + + @mock.patch("requests.post") + def test_on_end_success(self, post_mock): + post_mock.return_value = FakeResponse204() + self.filter.on_end(self.ctx, self.lease) + post_mock.assert_called_with( + "http://localhost/on-end", + headers={'Content-Type': 'application/json', 'X-Auth-Token': 'test_token'}, + data='{"context": {"is_context": true}, ' + '"lease": {"is_lease": true}}') + + @mock.patch("requests.post") + def test_on_end_failure(self, post_mock): + CONF.set_override( + 'external_service_token', 'test_token', group='enforcement') + post_mock.return_value = FakeResponse500() + self.assertRaises(ExternalServiceFilterException, + self.filter.on_end, + self.ctx, self.lease) + post_mock.assert_called_with( + "http://localhost/on-end", + headers={'Content-Type': 'application/json', 'X-Auth-Token': 'test_token'}, + data='{"context": {"is_context": true}, ' + '"lease": {"is_lease": true}}') diff --git a/blazar/tests/enforcement/filters/test_max_lease_duration_filter.py b/blazar/tests/enforcement/filters/test_max_lease_duration_filter.py index c78c00c6b..7fc154be4 100755 --- a/blazar/tests/enforcement/filters/test_max_lease_duration_filter.py +++ b/blazar/tests/enforcement/filters/test_max_lease_duration_filter.py @@ -81,7 +81,7 @@ def setUp(self): self.enforcement.load_filters() cfg.CONF.set_override('max_lease_duration', 3600, group='enforcement') - self.fake_service_catalog = [ + self.fake_service_catalog = tests.FakeServiceCatalog([ dict( type='identity', endpoints=[ dict( @@ -89,7 +89,7 @@ def setUp(self): url='https://fakeauth.com') ] ) - ] + ]) self.ctx = context.BlazarContext( user_id='111', project_id='222', @@ -178,6 +178,8 @@ def test_check_update_denied(self): del new_lease_values['reservations'] ctx = context.current() + cfg.CONF.set_override('max_lease_duration', 1, group='enforcement') + with mock.patch.object(datetime, 'datetime', mock.Mock(wraps=datetime.datetime)) as patched: patched.utcnow.return_value = datetime.datetime(2014, 1, 1, 1, 1) @@ -187,6 +189,8 @@ def test_check_update_denied(self): allocation_candidates, reservations, new_reservations) + self.cfg.CONF.clear_override('max_lease_duration', group='enforcement') + def test_check_update_active_lease_allowed(self): current_allocations = {'virtual:instance': [get_fake_host('1')]} lease = get_fake_lease(end_date=datetime.datetime(2014, 1, 1, 1, 53)) diff --git a/blazar/tests/enforcement/test_enforcement.py b/blazar/tests/enforcement/test_enforcement.py index 0bb8dc2ea..0350faacc 100755 --- a/blazar/tests/enforcement/test_enforcement.py +++ b/blazar/tests/enforcement/test_enforcement.py @@ -112,7 +112,7 @@ def setUp(self): cfg.CONF.set_override('os_region_name', self.region) self.enforcement.load_filters() - self.fake_service_catalog = [ + self.fake_service_catalog = tests.FakeServiceCatalog([ dict( type='identity', endpoints=[ dict( @@ -120,7 +120,7 @@ def setUp(self): url='https://fakeauth.com') ] ) - ] + ]) self.ctx = context.BlazarContext( user_id='111', project_id='222', diff --git a/blazar/tests/manager/test_service.py b/blazar/tests/manager/test_service.py index 9d1b18cdf..483a2c082 100644 --- a/blazar/tests/manager/test_service.py +++ b/blazar/tests/manager/test_service.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import datetime from unittest import mock @@ -72,10 +73,10 @@ def list_allocations(self, query, defail=False): def update_reservation(self, reservation_id, values): return None - def on_start(self, resource_id): + def on_start(self, resource_id, lease=None): return 'Resource %s should be started this moment.' % resource_id - def on_end(self, resource_id): + def on_end(self, resource_id, lease=None): return 'Resource %s should be deleted this moment.' % resource_id @@ -87,22 +88,16 @@ class FakePluginRaisesException(base.BasePlugin): def __init__(self): raise Exception - def reserve_resource(self, reservation_id, values): - return None - - def update_reservation(self, reservation_id, values): - return None - - def on_start(self, resource_id): + def on_start(self, resource_id, lease=None): return 'Resource %s should be started this moment.' % resource_id - def on_end(self, resource_id): + def on_end(self, resource_id, lease=None): return 'Resource %s should be deleted this moment.' % resource_id class FakeLeaseStatus(object): @classmethod - def lease_status(cls, transition, result_in): + def lease_status(cls, transition, result_in, non_fatal_exceptions=[]): def decorator(func): def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -111,7 +106,7 @@ def wrapper(*args, **kwargs): @ddt.ddt -class ServiceTestCase(tests.TestCase): +class ServiceTestCase(tests.DBTestCase): def setUp(self): super(ServiceTestCase, self).setUp() @@ -151,6 +146,8 @@ def setUp(self): importlib.reload(service) self.service = service self.manager = self.service.ManagerService() + self.get_plugins = self.service.get_plugins + self.get_plugins.cache_clear() self.enforcement = self.patch(self.manager, 'enforcement') self.lease_id = '11-22-33' @@ -160,7 +157,9 @@ def setUp(self): 'user_id': self.user_id, 'project_id': self.project_id, 'events': [ - {'event_type': 'start_lease', + { + 'id': 'asdf', + 'event_type': 'start_lease', 'time': datetime.datetime(2013, 12, 20, 13, 00), 'status': 'UNDONE'}, {'event_type': 'end_lease', @@ -237,7 +236,7 @@ def test_multiple_plugins_same_resource_type(self): FakeExtension("fake.plugin.2", FakePlugin)] self.assertRaises(manager_ex.PluginConfigurationError, - self.manager._get_plugins) + self.get_plugins) def test_plugins_that_fail_to_init(self): config = self.patch(cfg.CONF, "manager") @@ -246,7 +245,7 @@ def test_plugins_that_fail_to_init(self): FakeExtension("fake.plugin.1", FakePlugin), FakeExtension("fake.plugin.2", FakePluginRaisesException)] - plugins = self.manager._get_plugins() + plugins = self.get_plugins() self.assertIn("fake:plugin", plugins) self.assertNotIn("fake:plugin:raise", plugins) @@ -255,7 +254,7 @@ def test_get_bad_config_plugins(self): config.plugins = ['foo.plugin'] self.assertRaises(exceptions.BlazarException, - self.manager._get_plugins) + self.get_plugins) def test_setup_actions(self): actions = {'virtual:instance': @@ -269,36 +268,104 @@ def test_no_events(self): event_update = self.patch(self.db_api, 'event_update') events.return_value = None - self.manager._event() + self.manager._process_events() self.assertFalse(event_update.called) def test_event_success(self): events = self.patch(self.db_api, 'event_get_all_sorted_by_filters') event_update = self.patch(self.db_api, 'event_update') - events.return_value = [{'id': '111-222-333', - 'lease_id': 'lease_id1', - 'time': self.good_date}, - {'id': '444-555-666', - 'lease_id': 'lease_id2', - 'time': self.good_date}] - self.patch(eventlet, 'spawn_n') + events.return_value = [{'id': '111-222-333', 'time': self.good_date, + 'lease_id': 'aaa-bbb-ccc', + 'event_type': 'start_lease'}, + {'id': '444-555-666', 'time': self.good_date, + 'lease_id': 'bbb-ccc-ddd', + 'event_type': 'start_lease'}] + self.patch(eventlet, 'spawn') - self.manager._event() + self.manager._process_events() event_update.assert_has_calls([ mock.call('111-222-333', {'status': status.event.IN_PROGRESS}), mock.call('444-555-666', {'status': status.event.IN_PROGRESS})]) + def test_concurrent_events(self): + events = self.patch(self.db_api, 'event_get_all_sorted_by_filters') + self.patch(self.db_api, 'event_update') + events.return_value = [{'id': '111-222-333', 'time': self.good_date, + 'lease_id': 'aaa-bbb-ccc', + 'event_type': 'start_lease'}, + {'id': '222-333-444', 'time': self.good_date, + 'lease_id': 'bbb-ccc-ddd', + 'event_type': 'end_lease'}, + {'id': '333-444-555', 'time': self.good_date, + 'lease_id': 'bbb-ccc-ddd', + 'event_type': 'before_end_lease'}, + {'id': '444-555-666', 'time': self.good_date, + # Same lease as start_lease event above + 'lease_id': 'aaa-bbb-ccc', + 'event_type': 'before_end_lease'}, + {'id': '444-555-666', 'time': self.good_date, + # Same lease as start_lease event above + 'lease_id': 'aaa-bbb-ccc', + 'event_type': 'end_lease'}, + {'id': '555-666-777', 'time': self.good_date, + 'lease_id': 'ccc-ddd-eee', + 'event_type': 'end_lease'}, + {'id': '666-777-888', + 'time': self.good_date + datetime.timedelta( + minutes=1), + 'lease_id': 'ddd-eee-fff', + 'event_type': 'end_lease'}] + events_values = copy.copy(events.return_value) + _process_events_concurrently = self.patch( + self.manager, '_process_events_concurrently') + + self.manager._process_events() + _process_events_concurrently.assert_has_calls([ + # First execute the before_end_lease event which doesn't have a + # corresponding start_lease + mock.call([events_values[2]]), + # Then end_lease events + mock.call([events_values[1], events_values[5]]), + # Then the start_lease event + mock.call([events_values[0]]), + # Then the before_end_lease which is for the same lease as the + # previous start_lease event + mock.call([events_values[3]]), + # Then the end_lease which is for the same lease as the previous + # start_lease event + mock.call([events_values[4]]), + # Finally the event scheduled at the next minute + mock.call([events_values[6]])]) + + def test_process_events_concurrently(self): + events = [{'id': '111-222-333', 'time': self.good_date, + 'lease_id': 'aaa-bbb-ccc', + 'event_type': 'start_lease'}, + {'id': '222-333-444', 'time': self.good_date, + 'lease_id': 'bbb-ccc-ddd', + 'event_type': 'start_lease'}, + {'id': '333-444-555', 'time': self.good_date, + 'lease_id': 'ccc-ddd-eee', + 'event_type': 'start_lease'}] + spawn = self.patch(eventlet, 'spawn') + + self.manager._process_events_concurrently(events) + spawn.assert_has_calls([ + mock.call(mock.ANY, events[0]), + mock.call(mock.ANY, events[1]), + mock.call(mock.ANY, events[2])]) + def test_event_spawn_fail(self): events = self.patch(self.db_api, 'event_get_all_sorted_by_filters') event_update = self.patch(self.db_api, 'event_update') - self.patch(eventlet, 'spawn_n').side_effect = Exception - events.return_value = [{'id': '111-222-333', - 'lease_id': self.lease_id, - 'time': self.good_date}] + self.patch(eventlet, 'spawn').side_effect = Exception + events.return_value = [{'id': '111-222-333', 'time': self.good_date, + 'lease_id': 'aaa-bbb-ccc', + 'event_type': 'start_lease'}] - self.manager._event() + self.manager._process_events() event_update.assert_has_calls([ mock.call('111-222-333', {'status': status.event.IN_PROGRESS}), @@ -308,7 +375,8 @@ def test_event_pass(self): events = self.patch(self.db_api, 'event_get_all_sorted_by_filters') events.return_value = [{'id': '111-222-333', 'lease_id': self.lease_id, - 'time': self.good_date}] + 'time': self.good_date, + 'event_type': 'start_lease'}] self.lease_get = self.patch(self.db_api, 'lease_get') lease = self.lease.copy() @@ -317,7 +385,7 @@ def test_event_pass(self): event_update = self.patch(self.db_api, 'event_update') - self.manager._event() + self.manager._process_events() event_update.assert_not_called() @@ -332,10 +400,8 @@ def test_exec_event_success(self): start_lease.assert_called_once_with(lease_id=event['lease_id'], event_id=event['id']) self.lease_get.assert_called_once_with(event['lease_id']) - expected_context = self.trust_ctx.return_value self.fake_notifier.assert_called_once_with( - expected_context.__enter__.return_value, - notifier_api.format_lease_payload(self.lease), + {}, notifier_api.format_lease_payload(self.lease), 'lease.event.start_lease') def test_exec_event_invalid_event_type(self): @@ -429,11 +495,8 @@ def test_create_lease_now(self): self.trust_ctx.assert_called_once_with(lease_values['trust_id']) self.lease_create.assert_called_once_with(lease_values) self.assertEqual(lease, self.lease) - expected_context = self.trust_ctx.return_value - self.fake_notifier.assert_called_once_with( - expected_context.__enter__.return_value, - notifier_api.format_lease_payload(lease), + {}, notifier_api.format_lease_payload(lease), 'lease.create') def test_create_lease_some_time(self): @@ -701,6 +764,7 @@ def test_create_lease_without_required_params(self): def test_create_lease_with_filter_exception(self): lease_values = self.lease_values.copy() + self.lease_create.return_value = self.lease self.enforcement.check_create.side_effect = ( enforcement_ex.MaxLeaseDurationException(lease_duration=200, @@ -709,7 +773,20 @@ def test_create_lease_with_filter_exception(self): self.assertRaises(exceptions.NotAuthorized, self.manager.create_lease, lease_values=lease_values) - self.lease_create.assert_not_called() + self.assertEqual(1, self.lease_create.call_count) + + def test_create_lease_with_unexpected_filter_exception(self): + lease_values = self.lease_values.copy() + self.lease_create.return_value = self.lease + + self.enforcement.check_create.side_effect = RuntimeError( + 'Generic error from enforcement filter') + + self.assertRaises(RuntimeError, + self.manager.create_lease, + lease_values=lease_values) + self.lease_destroy.assert_called_once_with(self.lease_id) + self.assertEqual(1, self.lease_create.call_count) def test_update_lease_completed_lease_rename(self): lease_values = {'name': 'renamed'} @@ -794,7 +871,8 @@ def fake_event_get(sort_key, sort_dir, filters): 'max': 3, 'resource_type': 'virtual:instance' } - ] + ], + 'project_id': u'e33c952e-6321-4ce6-93e3-c58d4925c5f8' } reservation_get_all = ( self.patch(self.db_api, 'reservation_get_all_by_lease_id')) @@ -1008,12 +1086,9 @@ def fake_event_get(sort_key, sort_dir, filters): 'end_date': datetime.datetime(2013, 12, 20, 16, 00) } ) - expected_context = self.trust_ctx.return_value - calls = [mock.call(expected_context.__enter__.return_value, - notifier_api.format_lease_payload(self.lease), + calls = [mock.call({}, notifier_api.format_lease_payload(self.lease), 'lease.update'), - mock.call(expected_context.__enter__.return_value, - notifier_api.format_lease_payload(self.lease), + mock.call({}, notifier_api.format_lease_payload(self.lease), 'lease.event.before_end_lease.stop'), ] self.fake_notifier.assert_has_calls(calls) @@ -1073,12 +1148,9 @@ def fake_event_get(sort_key, sort_dir, filters): 'end_date': datetime.datetime(2013, 12, 20, 16, 00) } ) - expected_context = self.trust_ctx.return_value - calls = [mock.call(expected_context.__enter__.return_value, - notifier_api.format_lease_payload(self.lease), + calls = [mock.call({}, notifier_api.format_lease_payload(self.lease), 'lease.update'), - mock.call(expected_context.__enter__.return_value, - notifier_api.format_lease_payload(self.lease), + mock.call({}, notifier_api.format_lease_payload(self.lease), 'lease.event.before_end_lease.stop'), ] self.fake_notifier.assert_has_calls(calls) @@ -1253,6 +1325,8 @@ def test_update_lease_not_started_start_date_before_current_time(self): lease_id=self.lease_id, values=lease_values) def test_update_lease_end_date_before_current_time(self): + events = self.patch(self.db_api, 'event_get_first_sorted_by_filters') + events.return_value = self.lease['events'][0] lease_values = { 'name': 'renamed', 'end_date': '2013-12-14 13:00' @@ -1345,7 +1419,8 @@ def fake_event_get(sort_key, sort_dir, filters): 'max': 3, 'resource_type': 'virtual:instance' } - ] + ], + 'project_id': u'e33c952e-6321-4ce6-93e3-c58d4925c5f8' } reservation_get_all = ( self.patch(self.db_api, 'reservation_get_all_by_lease_id')) @@ -1386,9 +1461,8 @@ def fake_event_get(sort_key, sort_dir, filters): self.manager.delete_lease(self.lease_id) - self.trust_ctx.assert_called_once_with(self.lease['trust_id']) self.lease_destroy.assert_called_once_with(self.lease_id) - self.fake_plugin.on_end.assert_called_with('111') + self.fake_plugin.on_end.assert_called_with('111', lease=self.lease) enforcement_on_end.assert_called_once() def test_delete_lease_after_ending(self): @@ -1409,11 +1483,9 @@ def fake_event_get(sort_key, sort_dir, filters): self.manager.delete_lease(self.lease_id) - expected_context = self.trust_ctx.return_value self.lease_destroy.assert_called_once_with(self.lease_id) self.fake_notifier.assert_called_once_with( - expected_context.__enter__.return_value, - self.notifier_api.format_lease_payload(self.lease), + {}, self.notifier_api.format_lease_payload(self.lease), 'lease.delete') self.fake_plugin.on_end.assert_not_called() enforcement_on_end.assert_not_called() @@ -1435,9 +1507,11 @@ def fake_event_get(sort_key, sort_dir, filters): self.manager.delete_lease(self.lease_id) - self.event_update.assert_called_once_with('fake', - {'status': 'IN_PROGRESS'}) - self.fake_plugin.on_end.assert_called_with('111') + self.event_update.assert_has_calls([ + mock.call('fake', {'status': 'IN_PROGRESS'}), + mock.call('fake', {'status': 'DONE'}), + ]) + self.fake_plugin.on_end.assert_called_with('111', lease=self.lease) self.lease_destroy.assert_called_once_with(self.lease_id) enforcement_on_end.assert_called_once() @@ -1458,10 +1532,12 @@ def fake_event_get(sort_key, sort_dir, filters): self.manager.delete_lease(self.lease_id) - self.event_update.assert_called_once_with('fake', - {'status': 'IN_PROGRESS'}) + self.event_update.assert_has_calls([ + mock.call('fake', {'status': 'IN_PROGRESS'}), + mock.call('fake', {'status': 'DONE'}), + ]) - self.fake_plugin.on_end.assert_called_with('111') + self.fake_plugin.on_end.assert_called_with('111', lease=self.lease) self.lease_destroy.assert_called_once_with(self.lease_id) enforcement_on_end.assert_called_once() @@ -1485,9 +1561,11 @@ def fake_event_get(sort_key, sort_dir, filters): self.manager.delete_lease(self.lease_id) - self.event_update.assert_called_once_with('fake', - {'status': 'IN_PROGRESS'}) - self.fake_plugin.on_end.assert_called_with('111') + self.event_update.assert_has_calls([ + mock.call('fake', {'status': 'IN_PROGRESS'}), + mock.call('fake', {'status': 'DONE'}), + ]) + self.fake_plugin.on_end.assert_called_with('111', lease=self.lease) self.lease_destroy.assert_called_once_with(self.lease_id) enforcement_on_end.assert_called_once() @@ -1496,7 +1574,6 @@ def test_start_lease(self): self.manager.start_lease(self.lease_id, '1') - self.trust_ctx.assert_called_once_with(self.lease['trust_id']) basic_action.assert_called_once_with(self.lease_id, '1', 'on_start', 'active') @@ -1506,7 +1583,6 @@ def test_end_lease(self): self.manager.end_lease(self.lease_id, '1') - self.trust_ctx.assert_called_once_with(self.lease['trust_id']) basic_action.assert_called_once_with(self.lease_id, '1', 'on_end', 'deleted') enforcement_on_end.assert_called_once() @@ -1536,7 +1612,7 @@ def test_basic_action_with_res_status(self): self.event_update.assert_called_once_with('1', {'status': 'DONE'}) def test_basic_action_raise_exception(self): - def raiseBlazarException(resource_id): + def raiseBlazarException(resource_id, lease): raise exceptions.BlazarException(resource_id) self.manager.resource_actions = ( @@ -1556,7 +1632,7 @@ def raiseBlazarException(resource_id): self.event_update.assert_called_once_with('1', {'status': 'ERROR'}) def test_basic_action_raise_exception_no_reservation_status(self): - def raiseBlazarException(resource_id): + def raiseBlazarException(resource_id, lease): raise exceptions.BlazarException(resource_id) self.manager.resource_actions = ( @@ -1572,42 +1648,6 @@ def raiseBlazarException(resource_id): '111', {'status': 'error'}) self.event_update.assert_called_once_with('1', {'status': 'ERROR'}) - def test_getattr_with_correct_plugin_and_method(self): - self.fake_list_computehosts = ( - self.patch(self.fake_phys_plugin, 'list_computehosts')) - self.fake_list_computehosts.return_value = 'foo' - - self.manager.plugins = {'physical:host': self.fake_phys_plugin} - self.assertEqual('foo', getattr(self.manager, - 'physical:host:list_computehosts')()) - - def test_getattr_with_incorrect_method_name(self): - self.fake_list_computehosts = ( - self.patch(self.fake_phys_plugin, 'list_computehosts')) - self.fake_list_computehosts.return_value = 'foo' - - self.manager.plugins = {'physical:host': self.fake_phys_plugin} - self.assertRaises(AttributeError, getattr, self.manager, - 'simplefakecallwithValueError') - - def test_getattr_with_missing_plugin(self): - self.fake_list_computehosts = ( - self.patch(self.fake_phys_plugin, 'list_computehosts')) - self.fake_list_computehosts.return_value = 'foo' - - self.manager.plugins = {'physical:host': self.fake_phys_plugin} - self.assertRaises(manager_ex.UnsupportedResourceType, getattr, - self.manager, 'plugin:not_present:list_computehosts') - - def test_getattr_with_missing_method_in_plugin(self): - self.fake_list_computehosts = ( - self.patch(self.fake_phys_plugin, 'list_computehosts')) - self.fake_list_computehosts.return_value = 'foo' - - self.manager.plugins = {'physical:host': None} - self.assertRaises(AttributeError, getattr, self.manager, - 'physical:host:method_not_present') - @mock.patch.object(messaging, 'get_rpc_server') def test_rpc_server(self, mock_get_rpc_server): server = service.ManagerService() @@ -1619,3 +1659,168 @@ def test_rpc_server(self, mock_get_rpc_server): server.wait() server._server.wait.assert_called_once() self.assertEqual(1, mock_get_rpc_server.call_count) + + def test_update_non_fatal_max_lease_duration_exception(self): + importlib.reload(service) + # import service again without mocking the lease_status + # decorator + manager = service.ManagerService() + enforcement_mngr = self.patch(manager, 'enforcement') + enforcement_mngr.check_update.side_effect = ( + enforcement_ex.MaxLeaseDurationException(lease_duration=200, + max_duration=100)) + manager.plugins = {'virtual:instance': self.fake_plugin} + manager.resource_actions = ( + {'virtual:instance': + {'on_start': self.fake_plugin.on_start, + 'on_end': self.fake_plugin.on_end}}) + events = self.patch(self.db_api, 'event_get_first_sorted_by_filters') + events.return_value = self.lease['events'][0] + lease_values = { + 'name': 'renamed', + 'prolong_for': '8d' + } + target = datetime.datetime(2013, 12, 14) + with mock.patch.object(datetime, + 'datetime', + mock.Mock(wraps=datetime.datetime)) as patched: + patched.utcnow.return_value = target + self.assertRaises( + enforcement_ex.MaxLeaseDurationException, + manager.update_lease, + lease_id=self.lease_id, + values=lease_values + ) + self.lease_update.assert_called_with( + '11-22-33', {'status': 'PENDING'} + ) + + def test_update_non_fatal_max_lease_update_window_exception(self): + importlib.reload(service) + manager = service.ManagerService() + enforcement_mngr = self.patch(manager, 'enforcement') + enforcement_mngr.check_update.side_effect = ( + enforcement_ex.MaxLeaseUpdateWindowException(extension_window=604900)) + manager.plugins = {'virtual:instance': self.fake_plugin} + manager.resource_actions = ( + {'virtual:instance': + {'on_start': self.fake_plugin.on_start, + 'on_end': self.fake_plugin.on_end}}) + events = self.patch(self.db_api, 'event_get_first_sorted_by_filters') + events.return_value = self.lease['events'][0] + lease_values = { + 'name': 'renamed', + 'prolong_for': '8d' + } + target = datetime.datetime(2013, 12, 14) + with mock.patch.object(datetime, + 'datetime', + mock.Mock(wraps=datetime.datetime)) as patched: + patched.utcnow.return_value = target + self.assertRaises( + enforcement_ex.MaxLeaseUpdateWindowException, + manager.update_lease, + lease_id=self.lease_id, + values=lease_values + ) + self.lease_update.assert_called_with( + '11-22-33', {'status': 'PENDING'} + ) + + def test_update_non_fatal_external_service_unsupported_http_response(self): + importlib.reload(service) + manager = service.ManagerService() + enforcement_mngr = self.patch(manager, 'enforcement') + enforcement_mngr.check_update.side_effect = ( + enforcement.exceptions.ExternalServiceUnsupportedHTTPResponse(status=503)) + manager.plugins = {'virtual:instance': self.fake_plugin} + manager.resource_actions = ( + {'virtual:instance': + {'on_start': self.fake_plugin.on_start, + 'on_end': self.fake_plugin.on_end}}) + events = self.patch(self.db_api, 'event_get_first_sorted_by_filters') + events.return_value = self.lease['events'][0] + lease_values = { + 'name': 'renamed', + 'prolong_for': '8d' + } + target = datetime.datetime(2013, 12, 14) + with mock.patch.object(datetime, + 'datetime', + mock.Mock(wraps=datetime.datetime)) as patched: + patched.utcnow.return_value = target + self.assertRaises( + enforcement.exceptions.ExternalServiceUnsupportedHTTPResponse, + manager.update_lease, + lease_id=self.lease_id, + values=lease_values + ) + self.lease_update.assert_called_with( + '11-22-33', {'status': 'PENDING'} + ) + + def test_update_non_fatal_external_service_filter_exception(self): + importlib.reload(service) + manager = service.ManagerService() + enforcement_mngr = self.patch(manager, 'enforcement') + enforcement_mngr.check_update.side_effect = ( + enforcement.exceptions.ExternalServiceFilterException(message="filter exception")) + manager.plugins = {'virtual:instance': self.fake_plugin} + manager.resource_actions = ( + {'virtual:instance': + {'on_start': self.fake_plugin.on_start, + 'on_end': self.fake_plugin.on_end}}) + events = self.patch(self.db_api, 'event_get_first_sorted_by_filters') + events.return_value = self.lease['events'][0] + lease_values = { + 'name': 'renamed', + 'prolong_for': '8d' + } + target = datetime.datetime(2013, 12, 14) + with mock.patch.object(datetime, + 'datetime', + mock.Mock(wraps=datetime.datetime)) as patched: + patched.utcnow.return_value = target + self.assertRaises( + enforcement.exceptions.ExternalServiceFilterException, + manager.update_lease, + lease_id=self.lease_id, + values=lease_values + ) + self.lease_update.assert_called_with( + '11-22-33', {'status': 'PENDING'} + ) + + def test_update_fatal_extra_capability_too_long_exception(self): + # lease status ERROR when a fatal exception occurs + importlib.reload(service) + manager = service.ManagerService() + enforcement_mngr = self.patch(manager, 'enforcement') + enforcement_mngr.check_update.side_effect = ( + manager_ex.ExtraCapabilityTooLong() + ) + manager.plugins = {'virtual:instance': self.fake_plugin} + manager.resource_actions = ( + {'virtual:instance': + {'on_start': self.fake_plugin.on_start, + 'on_end': self.fake_plugin.on_end}}) + events = self.patch(self.db_api, 'event_get_first_sorted_by_filters') + events.return_value = self.lease['events'][0] + lease_values = { + 'name': 'renamed', + 'prolong_for': '8d' + } + target = datetime.datetime(2013, 12, 14) + with mock.patch.object(datetime, + 'datetime', + mock.Mock(wraps=datetime.datetime)) as patched: + patched.utcnow.return_value = target + self.assertRaises( + manager_ex.ExtraCapabilityTooLong, + manager.update_lease, + lease_id=self.lease_id, + values=lease_values + ) + self.lease_update.assert_called_with( + '11-22-33', {'status': 'ERROR'} + ) diff --git a/blazar/tests/notification/test_notifier.py b/blazar/tests/notification/test_notifier.py index ec78f489d..e7ad1c647 100644 --- a/blazar/tests/notification/test_notifier.py +++ b/blazar/tests/notification/test_notifier.py @@ -39,7 +39,7 @@ def setUp(self): self.fake_notifier.return_value = FakeNotifier() self.fake_transport = self.patch( messaging, - 'get_notification_transport').return_value + 'get_notification_transport') self.info_method = self.patch(FakeNotifier, 'info') @@ -64,14 +64,14 @@ def test_send_lease_event(self): def test_cleanup(self): notification.cleanup() - self.fake_transport.cleanup.assert_called_once_with() + self.fake_transport.return_value.cleanup.assert_called_once_with() self.assertIsNone(notification.NOTIFIER) self.assertIsNone(notification.TRANSPORT) def test_init(self): - self.fake_transport.called_once - self.fake_notifier.called_once_with(self.fake_transport, - publisher_id='lease-service') + self.fake_transport.assert_called_once_with(notification.CONF) + self.fake_notifier.assert_called_once_with( + self.fake_transport.return_value, publisher_id='lease-service') def test_init_called_twice_returns_same_instance(self): prev_notifier = notification.NOTIFIER diff --git a/blazar/tests/plugins/devices/__init__.py b/blazar/tests/plugins/devices/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/blazar/tests/plugins/devices/test_device_plugin.py b/blazar/tests/plugins/devices/test_device_plugin.py new file mode 100644 index 000000000..605aa5dda --- /dev/null +++ b/blazar/tests/plugins/devices/test_device_plugin.py @@ -0,0 +1,2554 @@ +# Copyright (c) 2013 Bull. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +from unittest import mock + +from blazar.plugins.devices import k8s_plugin +import ddt +from unittest import skip +from oslo_config import cfg +from oslo_config import fixture as conf_fixture +import random +import testtools + +from blazar import context, policy +from blazar import status +from blazar.db import api as db_api +from blazar.db import exceptions as db_exceptions +from blazar.db import utils as db_utils +from blazar.manager import exceptions as manager_exceptions +from blazar.manager import service +from blazar.plugins import devices as plugin +from blazar.plugins.devices import device_plugin +from blazar import tests +from blazar.utils.openstack import base +from blazar.utils import trusts + +from kubernetes import client +from kubernetes import config + +CONF = cfg.CONF + + +@ddt.ddt +class DevicePluginTestCase(tests.TestCase): + + def setUp(self): + super(DevicePluginTestCase, self).setUp() + self.cfg = cfg + self.context = context + self.patch(self.context, 'BlazarContext') + + self.service = service + self.manager = self.service.ManagerService() + + self.fake_device_id = '1' + self.fake_device = { + 'id': self.fake_device_id, + "name": "fake-rpi-1", + "device_type": "container", + "device_driver": "k8s", + } + + self.patch(base, 'url_for').return_value = 'http://foo.bar' + self.device_plugin = device_plugin + self._get_plugins = self.patch(device_plugin, "_get_plugins") + + self.patch(config, 'load_kube_config').return_value = mock.MagicMock() + self.core_v1 = self.patch(client, 'CoreV1Api') + self.core_v1.return_value = mock.MagicMock() + self.apps_v1 = self.patch(client, 'AppsV1Api') + self.apps_v1.return_value = mock.MagicMock() + + self.fake_k8s_plugin = k8s_plugin.K8sPlugin() + + self._get_plugins.return_value = { + "k8s": self.fake_k8s_plugin, + } + + self.fake_dev_plugin = self.device_plugin.DevicePlugin() + self.db_api = db_api + self.db_utils = db_utils + + self.db_device_get = self.patch(self.db_api, 'device_get') + self.db_device_get.return_value = self.fake_device + self.db_device_list = self.patch(self.db_api, 'device_list') + self.db_device_create = self.patch(self.db_api, 'device_create') + self.db_device_update = self.patch(self.db_api, 'device_update') + self.db_device_destroy = self.patch(self.db_api, 'device_destroy') + + self.db_device_extra_capability_get_all_per_device = self.patch( + self.db_api, 'device_extra_capability_get_all_per_device') + + self.db_device_extra_capability_get_all_per_name = self.patch( + self.db_api, 'device_extra_capability_get_all_per_name') + + self.db_device_extra_capability_create = self.patch( + self.db_api, 'device_extra_capability_create') + + self.db_device_extra_capability_update = self.patch( + self.db_api, 'device_extra_capability_update') + + self.db_device_extra_capability_destroy = self.patch( + self.db_api, 'device_extra_capability_destroy') + + self.get_extra_capabilities = self.patch( + self.fake_dev_plugin, '_get_extra_capabilities') + self.get_extra_capabilities.return_value = { + 'foo': 'bar', + 'buzz': 'word', + } + + self.fake_dev_plugin.setup(None) + + self.trusts = trusts + self.trust_ctx = self.patch(self.trusts, 'create_ctx_from_trust') + self.trust_create = self.patch(self.trusts, 'create_trust') + + self.cfg = cfg + self.cfg.CONF.set_override('plugins', ["k8s.plugin"], group=plugin.RESOURCE_TYPE) + + + def reservation_allocation_dict(self, r_id, l_id, p_id, h_ids): + return { + 'id': r_id, 'status': 'active', 'lease_id': l_id, + 'start_date': datetime.datetime(2015, 1, 1, 0, 0), + 'end_date': datetime.datetime(2015, 1, 2, 0, 0), + 'lease_name': l_id, 'project_id': p_id, + 'device_ids': h_ids} + + def test_get_device(self): + device = self.fake_dev_plugin.get_device(self.fake_device_id) + self.db_device_get.assert_called_once_with('1') + expected = self.fake_device.copy() + expected.update({'foo': 'bar', 'buzz': 'word'}) + self.assertEqual(expected, device) + + def test_get_device_without_extracapabilities(self): + self.get_extra_capabilities.return_value = {} + device = self.fake_dev_plugin.get_device(self.fake_device_id) + self.db_device_get.assert_called_once_with('1') + self.assertEqual(self.fake_device, device) + + def test_list_devices(self): + self.fake_dev_plugin.list_devices() + self.db_device_list.assert_called_once_with() + + def test_create_device_without_extra_capabilities(self): + self.get_extra_capabilities.return_value = {} + device = self.fake_dev_plugin.create_device(self.fake_device) + expected_device = self.fake_device.copy() + del expected_device["id"] + self.db_device_create.assert_called_once_with(expected_device) + self.assertEqual(self.fake_device, device) + + def test_create_device_with_extra_capabilities(self): + fake_device = self.fake_device.copy() + fake_device.update({'foo': 'bar'}) + fake_request = fake_device.copy() + del fake_request["id"] + fake_capa = {'device_id': '1', + 'capability_name': 'foo', + 'capability_value': 'bar', + } + self.get_extra_capabilities.return_value = {'foo': 'bar'} + self.db_device_create.return_value = fake_device + device = self.fake_dev_plugin.create_device(fake_request) + self.db_device_create.assert_called_once_with( + { + 'name': self.fake_device["name"], + 'device_type': 'container', + 'device_driver': "k8s", + } + ) + self.db_device_extra_capability_create.assert_called_once_with(fake_capa) + self.assertEqual(fake_device, device) + + + def test_create_device_issuing_rollback(self): + def fake_db_device_create(*args, **kwargs): + raise db_exceptions.BlazarDBException + self.patch(self.fake_k8s_plugin, "create_device").side_effect = fake_db_device_create + self.assertRaises(db_exceptions.BlazarDBException, + self.fake_dev_plugin.create_device, + self.fake_device) + + def test_create_duplicate_device(self): + def fake_db_device_create(*args, **kwargs): + raise db_exceptions.BlazarDBDuplicateEntry + self.db_device_create.side_effect = fake_db_device_create + self.assertRaises(db_exceptions.BlazarDBDuplicateEntry, + self.fake_dev_plugin.create_device, + self.fake_device) + + def test_create_device_having_issue_when_storing_extra_capability(self): + def fake_db_device_extra_capability_create(*args, **kwargs): + raise db_exceptions.BlazarDBException + fake_device = self.fake_device.copy() + fake_device.update({'foo': 'bar'}) + fake_request = fake_device.copy() + self.get_extra_capabilities.return_value = {'foo': 'bar'} + self.db_device_create.return_value = fake_device + fake = self.db_device_extra_capability_create + fake.side_effect = fake_db_device_extra_capability_create + self.assertRaises(manager_exceptions.CantAddExtraCapability, + self.fake_dev_plugin.create_device, + fake_request) + + def test_update_device(self): + device_values = {'foo': 'baz'} + + self.db_device_extra_capability_get_all_per_name.return_value = [ + ({'id': 'extra_id1', + 'device_id': self.fake_device_id, + 'capability_value': 'bar'}, + 'foo'), + ] + + self.get_reservations_by_device = self.patch( + self.db_utils, 'get_reservations_by_device_id') + self.get_reservations_by_device.return_value = [] + + self.fake_dev_plugin.update_device(self.fake_device_id, + device_values) + self.db_device_extra_capability_update.assert_called_once_with( + 'extra_id1', {'capability_value': 'baz'}) + + def test_update_device_having_issue_when_storing_extra_capability(self): + def fake_db_device_extra_capability_update(*args, **kwargs): + raise RuntimeError + device_values = {'foo': 'baz'} + self.get_reservations_by_device = self.patch( + self.db_utils, 'get_reservations_by_device_id') + self.get_reservations_by_device.return_value = [] + self.db_device_extra_capability_get_all_per_name.return_value = [ + ({'id': 'extra_id1', + 'device_id': self.fake_device_id, + 'capability_value': 'bar'}, + 'foo'), + ] + fake = self.db_device_extra_capability_update + fake.side_effect = fake_db_device_extra_capability_update + self.assertRaises(manager_exceptions.CantAddExtraCapability, + self.fake_dev_plugin.update_device, + self.fake_device_id, device_values) + + def test_update_device_with_new_extra_capability(self): + device_values = {'qux': 'word'} + + self.db_device_extra_capability_get_all_per_device.return_value = [] + self.fake_dev_plugin.update_device(self.fake_device_id, + device_values) + self.db_device_extra_capability_create.assert_called_once_with({ + 'device_id': '1', + 'capability_name': 'qux', + 'capability_value': 'word' + }) + + def test_update_device_with_removed_capability(self): + device_values = {'foo': None} + + self.db_device_extra_capability_get_all_per_name.return_value = [ + ({'id': 'extra_id1', + 'device_id': self.fake_device_id, + 'capability_value': 'bar'}, + 'foo'), + ] + + self.get_reservations_by_device = self.patch( + self.db_utils, 'get_reservations_by_device_id') + self.get_reservations_by_device.return_value = [] + + self.fake_dev_plugin.update_device(self.fake_device_id, + device_values) + self.db_device_extra_capability_destroy.assert_called_once_with( + 'extra_id1') + + def test_update_device_with_used_capability(self): + device_values = {'foo': 'buzz'} + + self.db_device_extra_capability_get_all_per_name.return_value = [ + ({'id': 'extra_id1', + 'device_id': self.fake_device_id, + 'capability_value': 'bar'}, + 'foo'), + ] + fake_phys_reservation = { + 'resource_type': plugin.RESOURCE_TYPE, + 'resource_id': 'resource-1', + } + + fake_get_reservations = self.patch(self.db_utils, + 'get_reservations_by_device_id') + fake_get_reservations.return_value = [fake_phys_reservation] + + fake_get_plugin_reservation = self.patch(self.db_utils, + 'get_plugin_reservation') + fake_get_plugin_reservation.return_value = { + 'resource_properties': '["==", "$foo", "bar"]' + } + self.assertRaises(manager_exceptions.CantAddExtraCapability, + self.fake_dev_plugin.update_device, + self.fake_device_id, device_values) + fake_get_plugin_reservation.assert_called_once_with( + plugin.RESOURCE_TYPE, 'resource-1') + + def test_delete_device(self): + device_allocation_get_all = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [] + self.fake_dev_plugin.delete_device(self.fake_device_id) + + self.db_device_destroy.assert_called_once_with(self.fake_device_id) + + def test_delete_device_reserved(self): + device_allocation_get_all = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': self.fake_device_id + } + ] + + self.assertRaises(manager_exceptions.CantDeleteDevice, + self.fake_dev_plugin.delete_device, + self.fake_device_id) + + def test_delete_device_having_vms(self): + device_allocation_get_all = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [] + + def test_delete_device_not_existing_in_db(self): + self.db_device_get.return_value = None + self.assertRaises(manager_exceptions.DeviceNotFound, + self.fake_dev_plugin.delete_device, + self.fake_device_id) + + def test_delete_device_issuing_rollback(self): + def fake_db_device_destroy(*args, **kwargs): + raise db_exceptions.BlazarDBException + device_allocation_get_all = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [] + self.db_device_destroy.side_effect = fake_db_device_destroy + self.assertRaises(manager_exceptions.CantDeleteDevice, + self.fake_dev_plugin.delete_device, + self.fake_device_id) + + def test_list_allocations(self): + self.db_get_reserv_allocs = self.patch( + self.db_utils, 'get_reservation_allocations_by_device_ids') + + # Expecting a list of (Reservation, Allocation) + self.db_get_reserv_allocs.return_value = [ + self.reservation_allocation_dict(*r) for r + in [ + ('reservation-1', 'lease-1', + 'project-1', ['device-1', 'device-2']), + ('reservation-2', 'lease-1', + 'project-1', ['device-2', 'device-3']), + ('reservation-3', 'lease-2', 'project-2', ['device-1'])]] + + self.db_device_list = self.patch(self.db_api, 'device_list') + self.db_device_list.return_value = [ + {'id': 'device-1'}, {'id': 'device-2'}, {'id': 'device-3'}] + + expected = [ + { + 'resource_id': 'device-1', + 'reservations': [ + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + {'id': 'reservation-3', + 'lease_id': 'lease-2', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + ] + }, + { + 'resource_id': 'device-2', + 'reservations': [ + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + {'id': 'reservation-2', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + ] + }, + { + 'resource_id': 'device-3', + 'reservations': [ + {'id': 'reservation-2', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + ] + } + ] + ret = self.fake_dev_plugin.list_allocations({}) + + # Sort returned value to use assertListEqual + for r in ret: + r['reservations'].sort(key=lambda x: x['id']) + ret.sort(key=lambda x: x['resource_id']) + + self.assertListEqual(expected, ret) + + def test_list_allocations_with_lease_id(self): + self.db_get_reserv_allocs = self.patch( + self.db_utils, 'get_reservation_allocations_by_device_ids') + + # Expecting a list of (Reservation, Allocation) + self.db_get_reserv_allocs.return_value = [ + self.reservation_allocation_dict(*r) for r + in [ + ('reservation-1', 'lease-1', + 'project-1', ['device-1', 'device-2']), + ('reservation-2', 'lease-1', + 'project-1', ['device-2', 'device-3'])]] + + self.db_device_list = self.patch(self.db_api, 'device_list') + self.db_device_list.return_value = [ + {'id': 'device-1'}, {'id': 'device-2'}, {'id': 'device-3'}] + + expected = [ + { + 'resource_id': 'device-1', + 'reservations': [ + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + ] + }, + { + 'resource_id': 'device-2', + 'reservations': [ + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + {'id': 'reservation-2', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + ] + }, + { + 'resource_id': 'device-3', + 'reservations': [ + {'id': 'reservation-2', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + ] + } + ] + ret = self.fake_dev_plugin.list_allocations({'lease_id': 'lease-1'}) + + # Sort returned value to use assertListEqual + for r in ret: + r['reservations'].sort(key=lambda x: x['id']) + ret.sort(key=lambda x: x['resource_id']) + + self.assertListEqual(expected, ret) + + def test_list_allocations_with_reservation_id(self): + self.db_get_reserv_allocs = self.patch( + self.db_utils, 'get_reservation_allocations_by_device_ids') + + # Expecting a list of (Reservation, Allocation) + self.db_get_reserv_allocs.return_value = [ + self.reservation_allocation_dict(*r) for r + in [ + ('reservation-1', 'lease-1', + 'project-1', ['device-1', 'device-2'])]] + + self.db_device_list = self.patch(self.db_api, 'device_list') + self.db_device_list.return_value = [{'id': 'device-1'}, {'id': 'device-2'}] + + expected = [ + { + 'resource_id': 'device-1', + 'reservations': [ + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + ] + }, + { + 'resource_id': 'device-2', + 'reservations': [ + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + ] + }, + ] + ret = self.fake_dev_plugin.list_allocations( + {'reservation_id': 'reservation-1'}) + + # Sort returned value to use assertListEqual + for r in ret: + r['reservations'].sort(key=lambda x: x['id']) + ret.sort(key=lambda x: x['resource_id']) + + self.assertListEqual(expected, ret) + + def test_get_allocations(self): + self.db_get_reserv_allocs = self.patch( + self.db_utils, 'get_reservation_allocations_by_device_ids') + + # Expecting a list of (Reservation, Allocation) + self.db_get_reserv_allocs.return_value = [ + self.reservation_allocation_dict(*r) for r + in [ + ('reservation-1', 'lease-1', + 'project-1', ['device-1', 'device-2']), + ('reservation-2', 'lease-1', + 'project-1', ['device-2', 'device-3']), + ('reservation-3', 'lease-2', 'project-2', ['device-1'])]] + + self.db_device_list = self.patch(self.db_api, 'device_list') + self.db_device_list.return_value = [ + {'id': 'device-1'}, {'id': 'device-2'}, {'id': 'device-3'}] + + expected = { + 'resource_id': 'device-1', + 'reservations': [ + {'id': 'reservation-1', 'lease_id': 'lease-1', + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + {'id': 'reservation-3', 'lease_id': 'lease-2', + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + ] + } + ret = self.fake_dev_plugin.get_allocations('device-1', {}) + + # sort returned value to use assertListEqual + ret['reservations'].sort(key=lambda x: x['id']) + + self.assertDictEqual(expected, ret) + + def test_get_allocations_with_lease_id(self): + self.db_get_reserv_allocs = self.patch( + self.db_utils, 'get_reservation_allocations_by_device_ids') + + # Expecting a list of (Reservation, Allocation) + self.db_get_reserv_allocs.return_value = [ + self.reservation_allocation_dict( + 'reservation-1', 'lease-1', 'project-1', ['device-1']), + ] + + self.db_device_list = self.patch(self.db_api, 'device_list') + self.db_device_list.return_value = [{'id': 'device-1'}] + + expected = { + 'resource_id': 'device-1', + 'reservations': [ + {'id': 'reservation-1', 'lease_id': 'lease-1', + 'start_date': datetime.datetime(2015, 1, 1, 0, 0), 'end_date': datetime.datetime(2015, 1, 2, 0, 0)}]} + + ret = self.fake_dev_plugin.get_allocations('device-1', + {'lease_id': 'lease-1'}) + + # sort returned value to use assertListEqual + ret['reservations'].sort(key=lambda x: x['id']) + + self.assertDictEqual(expected, ret) + + def test_get_allocations_with_reservation_id(self): + self.db_get_reserv_allocs = self.patch( + self.db_utils, 'get_reservation_allocations_by_device_ids') + + # Expecting a list of (Reservation, Allocation) + self.db_get_reserv_allocs.return_value = [ + self.reservation_allocation_dict( + 'reservation-1', 'lease-1', 'project-1', ['device-1'])] + + self.db_device_list = self.patch(self.db_api, 'device_list') + self.db_device_list.return_value = [{'id': 'device-1'}] + + expected = { + 'resource_id': 'device-1', + 'reservations': [ + {'id': 'reservation-1', 'lease_id': 'lease-1', + 'start_date': datetime.datetime(2015, 1, 1, 0, 0), 'end_date': datetime.datetime(2015, 1, 2, 0, 0)}]} + + ret = self.fake_dev_plugin.get_allocations( + 'device-1', {'reservation_id': 'reservation-1'}) + + # sort returned value to use assertListEqual + ret['reservations'].sort(key=lambda x: x['id']) + + self.assertDictEqual(expected, ret) + + def test_get_allocations_with_invalid_device(self): + self.db_get_reserv_allocs = self.patch( + self.db_utils, 'get_reservation_allocations_by_device_ids') + + # Expecting a list of (Reservation, Allocation) + self.db_get_reserv_allocs.return_value = [ + self.reservation_allocation_dict(*r) for r + in [ + ('reservation-1', 'lease-1', + 'project-1', ['device-1', 'device-2']), + ('reservation-2', 'lease-1', + 'project-1', ['device-2', 'device-3']), + ('reservation-3', 'lease-2', 'project-2', ['device-1'])]] + + self.db_device_list = self.patch(self.db_api, 'device_list') + self.db_device_list.return_value = [ + {'id': 'device-1'}, {'id': 'device-2'}, {'id': 'device-3'}] + + expected = {'resource_id': 'no-reserved-device', 'reservations': []} + ret = self.fake_dev_plugin.get_allocations('no-reserved-device', {}) + + self.assertDictEqual(expected, ret) + + def test_create_reservation_no_devices_available(self): + now = datetime.datetime.utcnow() + values = { + 'lease_id': '018c1b43-e69e-4aef-a543-09681539cf4c', + 'min': 1, + 'max': 1, + 'resource_properties': '["=", "$memory_mb", "256"]', + 'start_date': now, + 'end_date': now + datetime.timedelta(hours=1), + 'resource_type': plugin.RESOURCE_TYPE, + 'project_id': 'fake-project' + } + device_reservation_create = self.patch(self.db_api, + 'device_reservation_create') + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = [] + self.assertRaises(manager_exceptions.NotEnoughDevicesAvailable, + self.fake_dev_plugin.reserve_resource, + 'f9894fcf-e2ed-41e9-8a4c-92fac332608e', + values) + device_reservation_create.assert_not_called() + + def test_create_reservation_devices_available(self): + values = { + 'lease_id': '018c1b43-e69e-4aef-a543-09681539cf4c', + 'min': 1, + 'max': 1, + 'resource_properties': '["=", "$memory_mb", "256"]', + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'resource_type': plugin.RESOURCE_TYPE, + 'project_id': 'fake-project' + } + device_reservation_create = self.patch(self.db_api, + 'device_reservation_create') + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = ['device1', 'device2'] + device_allocation_create = self.patch( + self.db_api, + 'device_allocation_create') + self.fake_dev_plugin.reserve_resource( + '441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + device_values = { + 'reservation_id': '441c1476-9f8f-4700-9f30-cd9b6fef3509', + 'resource_properties': '["=", "$memory_mb", "256"]', + 'count_range': '1-1', + 'status': 'pending', + 'before_end': 'default', + } + device_reservation_create.assert_called_once_with(device_values) + calls = [ + mock.call( + {'device_id': 'device1', + 'reservation_id': '441c1476-9f8f-4700-9f30-cd9b6fef3509', + }), + mock.call( + {'device_id': 'device2', + 'reservation_id': '441c1476-9f8f-4700-9f30-cd9b6fef3509', + }), + ] + device_allocation_create.assert_has_calls(calls) + + def test_create_reservation_devices_non_reservable(self): + values = { + 'lease_id': '018c1b43-e69e-4aef-a543-09681539cf4c', + 'min': 1, + 'max': 1, + 'resource_properties': '["=", "$memory_mb", "256"]', + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'resource_type': plugin.RESOURCE_TYPE, + 'project_id': 'fake-project' + } + device_reservation_create = self.patch(self.db_api, + 'device_reservation_create') + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = ['device1', 'device2'] + device_allocation_create = self.patch( + self.db_api, + 'device_allocation_create') + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + self.fake_dev_plugin.reserve_resource( + '441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + device_values = { + 'reservation_id': '441c1476-9f8f-4700-9f30-cd9b6fef3509', + 'resource_properties': '["=", "$memory_mb", "256"]', + 'count_range': '1-1', + 'status': 'pending', + 'before_end': 'default', + } + device_reservation_create.assert_called_once_with(device_values) + calls = [ + mock.call( + {'device_id': 'device1', + 'reservation_id': '441c1476-9f8f-4700-9f30-cd9b6fef3509', + }), + mock.call( + {'device_id': 'device2', + 'reservation_id': '441c1476-9f8f-4700-9f30-cd9b6fef3509', + }), + ] + device_allocation_create.assert_has_calls(calls) + + @ddt.data({"params": {'max': 0}}, + {"params": {'max': -1}}, + {"params": {'max': 'one'}}, + {"params": {'min': 0}}, + {"params": {'min': -1}}, + {"params": {'min': 'one'}}, + {"params": {'before_end': 'invalid'}}) + @ddt.unpack + def test_create_reservation_with_invalid_param(self, params): + values = { + 'lease_id': '018c1b43-e69e-4aef-a543-09681539cf4c', + 'min': 1, + 'max': 2, + 'before_end': 'default', + 'resource_properties': '["=", "$memory_mb", "256"]', + 'start_date': datetime.datetime(2017, 3, 1, 20, 00), + 'end_date': datetime.datetime(2017, 3, 2, 20, 00), + 'resource_type': plugin.RESOURCE_TYPE} + for key, value in params.items(): + values[key] = value + self.patch(db_api, 'device_allocation_get_all_by_values') + self.assertRaises( + manager_exceptions.MalformedParameter, + self.fake_dev_plugin.reserve_resource, + '441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + + @ddt.data({"params": {'max': 0}}, + {"params": {'max': -1}}, + {"params": {'max': 'one'}}, + {"params": {'min': 0}}, + {"params": {'min': -1}}, + {"params": {'min': 'one'}}) + @ddt.unpack + def test_update_reservation_with_invalid_param(self, params): + values = { + 'lease_id': '018c1b43-e69e-4aef-a543-09681539cf4c', + 'min': 1, + 'max': 2, + 'before_end': 'default', + 'resource_properties': '["=", "$memory_mb", "256"]', + 'start_date': datetime.datetime(2017, 3, 1, 20, 00), + 'end_date': datetime.datetime(2017, 3, 2, 20, 00), + 'resource_type': plugin.RESOURCE_TYPE} + self.patch(self.db_api, 'reservation_get') + self.patch(self.db_api, 'lease_get') + device_reservation_get = self.patch(self.db_api, + 'device_reservation_get') + device_reservation_get.return_value = { + 'count_range': '1-1', + 'resource_properties': '["=", "$memory_mb", "256"]', + } + for key, value in params.items(): + values[key] = value + self.patch(db_api, 'device_allocation_get_all_by_values') + self.assertRaises( + manager_exceptions.MalformedParameter, + self.fake_dev_plugin.update_reservation, + '441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + + def test_create_update_reservation_with_invalid_range(self): + values = { + 'lease_id': '018c1b43-e69e-4aef-a543-09681539cf4c', + 'min': 2, + 'max': 1, + 'resource_properties': '["=", "$memory_mb", "256"]', + 'start_date': datetime.datetime(2017, 3, 1, 20, 00), + 'end_date': datetime.datetime(2017, 3, 2, 20, 00), + 'resource_type': plugin.RESOURCE_TYPE, + } + self.patch(self.db_api, 'reservation_get') + self.patch(self.db_api, 'lease_get') + self.patch(db_api, 'device_allocation_get_all_by_values') + device_reservation_get = self.patch(self.db_api, + 'device_reservation_get') + device_reservation_get.return_value = { + 'count_range': '1-1', + 'resource_properties': '["=", "$memory_mb", "256"]', + } + self.assertRaises( + manager_exceptions.InvalidRange, + self.fake_dev_plugin.reserve_resource, + '441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + self.assertRaises( + manager_exceptions.InvalidRange, + self.fake_dev_plugin.update_reservation, + '441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + + def test_update_reservation_shorten(self): + values = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 30), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + device_reservation_get.assert_not_called() + + def test_update_reservation_extend(self): + values = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 30) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'count_range': '1-1', + 'resource_properties': '["=", "$memory_mb", "256"]', + } + device_allocation_get_all = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [{'id': 'device1'}] + get_reserved_periods = self.patch(self.db_utils, + 'get_reserved_periods') + get_reserved_periods.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)) + ] + device_allocation_create = self.patch( + self.db_api, + 'device_allocation_create') + device_allocation_destroy = self.patch( + self.db_api, + 'device_allocation_destroy') + + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + device_allocation_create.assert_not_called() + device_allocation_destroy.assert_not_called() + + def test_update_reservation_move_failure(self): + values = { + 'start_date': datetime.datetime(2013, 12, 20, 20, 00), + 'end_date': datetime.datetime(2013, 12, 20, 21, 30), + 'project_id': 'fake-project' + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'active' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch( + self.db_api, + 'device_reservation_get') + device_reservation_get.return_value = { + 'count_range': '1-1', + 'resource_properties': '["=", "$memory_mb", "256"]', + } + device_allocation_get_all = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [{'id': 'device1'}] + get_reserved_periods = self.patch(self.db_utils, + 'get_reserved_periods') + get_reserved_periods.return_value = [ + (datetime.datetime(2013, 12, 20, 20, 30), + datetime.datetime(2013, 12, 20, 21, 00)) + ] + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = [] + self.assertRaises( + manager_exceptions.NotEnoughDevicesAvailable, + self.fake_dev_plugin.update_reservation, + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + reservation_get.assert_called() + + def test_update_reservation_move_overlap(self): + values = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 30), + 'end_date': datetime.datetime(2013, 12, 19, 21, 30) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch( + self.db_api, + 'device_reservation_get') + device_reservation_get.return_value = { + 'count_range': '1-1', + 'resource_properties': '["=", "$memory_mb", "256"]', + } + device_allocation_get_all = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [{'id': 'device1'}] + get_reserved_periods = self.patch(self.db_utils, + 'get_reserved_periods') + get_reserved_periods.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 30), + datetime.datetime(2013, 12, 19, 21, 00)) + ] + device_allocation_create = self.patch( + self.db_api, + 'device_allocation_create') + device_allocation_destroy = self.patch( + self.db_api, + 'device_allocation_destroy') + + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + device_allocation_create.assert_not_called() + device_allocation_destroy.assert_not_called() + + def test_update_reservation_move_realloc(self): + values = { + 'start_date': datetime.datetime(2013, 12, 20, 20, 00), + 'end_date': datetime.datetime(2013, 12, 20, 21, 30) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch( + self.db_api, + 'device_reservation_get') + device_reservation_get.return_value = { + 'count_range': '1-1', + 'resource_properties': '["=", "$memory_mb", "256"]', + } + device_allocation_get_all = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [{'id': 'device1'}, + {'id': 'device2'}] + device_allocation_create = self.patch( + self.db_api, + 'device_allocation_create') + device_allocation_destroy = self.patch( + self.db_api, + 'device_allocation_destroy') + get_reserved_periods = self.patch(self.db_utils, + 'get_reserved_periods') + get_reserved_periods.return_value = [ + (datetime.datetime(2013, 12, 20, 20, 30), + datetime.datetime(2013, 12, 20, 21, 00)) + ] + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = ['device2'] + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + device_reservation_get.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b') + device_allocation_destroy.assert_called_with( + 'dd305477-4df8-4547-87f6-69069ee546a6') + device_allocation_create.assert_called_with( + { + 'device_id': 'device2', + 'reservation_id': '706eb3bc-07ed-4383-be93-b32845ece672' + } + ) + + def test_update_reservation_min_increase_success(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'min': 3 + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'count_range': '2-3', + 'resource_properties': '["=", "$memory_mb", "16384"]', + } + device_allocation_get_all = self.patch( + self.db_api, 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + }, + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a7', + 'device_id': 'device2' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'} + ] + device_allocation_destroy = self.patch(self.db_api, + 'device_allocation_destroy') + device_allocation_create = self.patch(self.db_api, + 'device_allocation_create') + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = ['device3'] + device_reservation_update = self.patch(self.db_api, + 'device_reservation_update') + + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + device_reservation_get.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b') + matching_devices.assert_called_with( + '["=", "$memory_mb", "16384"]', + '1-1', + datetime.datetime(2017, 7, 12, 20, 00), + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' + ) + device_allocation_destroy.assert_not_called() + device_allocation_create.assert_called_with( + { + 'device_id': 'device3', + 'reservation_id': '706eb3bc-07ed-4383-be93-b32845ece672' + } + ) + device_reservation_update.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + {'count_range': '3-3'} + ) + + def test_update_reservation_min_increase_fail(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'min': 3 + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'count_range': '2-3', + 'resource_properties': '["=", "$memory_mb", "16384"]', + } + device_allocation_get_all = self.patch( + self.db_api, 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + }, + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a7', + 'device_id': 'device2' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [ + {'id': 'device1'}, + {'id': 'device2'} + ] + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = [] + + self.assertRaises( + manager_exceptions.NotEnoughDevicesAvailable, + self.fake_dev_plugin.update_reservation, + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + matching_devices.assert_called_with( + '["=", "$memory_mb", "16384"]', + '1-1', + datetime.datetime(2017, 7, 12, 20, 00), + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' + ) + + def test_update_reservation_min_decrease(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'min': 1 + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'count_range': '2-2', + 'resource_properties': '["=", "$memory_mb", "16384"]', + } + device_allocation_get_all = self.patch( + self.db_api, 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + }, + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a7', + 'device_id': 'device2' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [ + {'id': 'device1'}, + {'id': 'device2'} + ] + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + device_allocation_destroy = self.patch(self.db_api, + 'device_allocation_destroy') + device_allocation_create = self.patch(self.db_api, + 'device_allocation_create') + device_reservation_update = self.patch(self.db_api, + 'device_reservation_update') + + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + matching_devices.assert_not_called() + device_allocation_destroy.assert_not_called() + device_allocation_create.assert_not_called() + device_reservation_update.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + {'count_range': '1-2'} + ) + + def test_update_reservation_max_increase_alloc(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'max': 3 + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'count_range': '1-2', + 'resource_properties': '["=", "$memory_mb", "16384"]' + } + device_allocation_get_all = self.patch( + self.db_api, 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + }, + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a7', + 'device_id': 'device2' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'} + ] + device_allocation_destroy = self.patch(self.db_api, + 'device_allocation_destroy') + device_allocation_create = self.patch(self.db_api, + 'device_allocation_create') + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = ['device3'] + device_reservation_update = self.patch(self.db_api, + 'device_reservation_update') + + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + device_reservation_get.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b') + matching_devices.assert_called_with( + '["=", "$memory_mb", "16384"]', + '0-1', + datetime.datetime(2017, 7, 12, 20, 00), + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' + ) + device_allocation_destroy.assert_not_called() + device_allocation_create.assert_called_with( + { + 'device_id': 'device3', + 'reservation_id': '706eb3bc-07ed-4383-be93-b32845ece672' + } + ) + device_reservation_update.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + {'count_range': '1-3'} + ) + + def test_update_active_reservation_max_increase_alloc(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'max': 3 + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'active' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'count_range': '1-2', + 'resource_properties': '["=", "$memory_mb", "16384"]', + 'reservation_id': '706eb3bc-07ed-4383-be93-b32845ece672', + } + device_allocation_get_all = self.patch( + self.db_api, 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1', + "name": "rpi1", + }, + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a7', + 'device_id': 'device2', + "name": "rpi1", + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'} + ] + device_allocation_destroy = self.patch(self.db_api, + 'device_allocation_destroy') + device_allocation_create = self.patch(self.db_api, + 'device_allocation_create') + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = ['device3'] + device_get = self.patch(self.db_api, 'device_get') + device_get.return_value = { + 'service_name': 'service1', + "device_driver": "k8s", + "name": "rpi1", + } + add_device = self.patch( + self.fake_k8s_plugin, 'add_active_device') + device_reservation_update = self.patch(self.db_api, + 'device_reservation_update') + + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + device_reservation_get.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b') + matching_devices.assert_called_with( + '["=", "$memory_mb", "16384"]', + '0-1', + datetime.datetime(2017, 7, 12, 20, 00), + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' + ) + device_allocation_destroy.assert_not_called() + device_allocation_create.assert_called_with( + { + 'device_id': 'device3', + 'reservation_id': '706eb3bc-07ed-4383-be93-b32845ece672' + } + ) + add_device.assert_called_with( + device_get.return_value, + device_reservation_get.return_value, + lease_get.return_value, + ) + device_reservation_update.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + {'count_range': '1-3'} + ) + + def test_update_reservation_max_increase_noalloc(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'max': 3 + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'count_range': '1-2', + 'resource_properties': '["=", "$memory_mb", "16384"]', + } + device_allocation_get_all = self.patch( + self.db_api, 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + }, + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a7', + 'device_id': 'device2' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [ + {'id': 'device1'}, + {'id': 'device2'} + ] + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = [] + device_reservation_update = self.patch(self.db_api, + 'device_reservation_update') + + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + device_reservation_get.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b') + matching_devices.assert_called_with( + '["=", "$memory_mb", "16384"]', + '0-1', + datetime.datetime(2017, 7, 12, 20, 00), + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' + ) + device_reservation_update.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + {'count_range': '1-3'} + ) + + def test_update_reservation_max_decrease(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'max': 1 + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'count_range': '1-2', + 'resource_properties': '["=", "$memory_mb", "16384"]', + 'project_id': 'fake-project' + } + device_allocation_get_all = self.patch( + self.db_api, 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + }, + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a7', + 'device_id': 'device2' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [ + {'id': 'device1'}, + {'id': 'device2'} + ] + device_allocation_destroy = self.patch(self.db_api, + 'device_allocation_destroy') + device_reservation_update = self.patch(self.db_api, + 'device_reservation_update') + + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + device_reservation_get.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b') + device_allocation_destroy.assert_called_with( + 'dd305477-4df8-4547-87f6-69069ee546a6') + device_reservation_update.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + {'count_range': '1-1'} + ) + + def test_update_reservation_realloc_with_properties_change(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'resource_properties': '["=", "$memory_mb", "32768"]', + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'count_range': '1-1', + 'resource_properties': '["=", "$memory_mb", "16384"]', + } + device_allocation_get_all = self.patch( + self.db_api, 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [{'id': 'device2'}] + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = ['device2'] + device_allocation_create = self.patch(self.db_api, + 'device_allocation_create') + device_allocation_destroy = self.patch(self.db_api, + 'device_allocation_destroy') + device_reservation_update = self.patch(self.db_api, + 'device_reservation_update') + + self.fake_dev_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + device_reservation_get.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b') + matching_devices.assert_called_with( + '["=", "$memory_mb", "32768"]', + '1-1', + datetime.datetime(2017, 7, 12, 20, 00), + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' + ) + device_allocation_create.assert_called_with( + { + 'device_id': 'device2', + 'reservation_id': '706eb3bc-07ed-4383-be93-b32845ece672' + } + ) + device_allocation_destroy.assert_called_with( + 'dd305477-4df8-4547-87f6-69069ee546a6' + ) + device_reservation_update.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + {'resource_properties': '["=", "$memory_mb", "32768"]'} + ) + + def test_update_reservation_no_requested_devices_available(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'resource_properties': '[">=", "$vcpus", "32768"]' + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'project_id': 'fake-project' + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'count_range': '1-1', + 'resource_properties': '["=", "$memory_mb", "16384"]', + } + device_allocation_get_all = self.patch( + self.db_api, 'device_allocation_get_all_by_values') + device_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'device_id': 'device1' + } + ] + device_get_all_by_queries = self.patch(self.db_api, + 'device_get_all_by_queries') + device_get_all_by_queries.return_value = [] + matching_devices = self.patch(self.fake_dev_plugin, '_matching_devices') + matching_devices.return_value = [] + + self.assertRaises( + manager_exceptions.NotEnoughDevicesAvailable, + self.fake_dev_plugin.update_reservation, + '441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + + def test_on_start(self): + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'reservation_id': '593e7028-c0d1-4d76-8642-2ffd890b324c', + "device_driver": "k8s", + } + device_allocation_get_all_by_values = self.patch( + self.db_api, 'device_allocation_get_all_by_values') + device_allocation_get_all_by_values.return_value = [ + { + 'device_id': 'device1', + "device_driver": "k8s", + }, + ] + device_get = self.patch(self.db_api, 'device_get') + device_get.return_value = { + "device_driver": "k8s", + "name": "rpi1", + } + + self.fake_dev_plugin.on_start('04de74e8-193a-49d2-9ab8-cba7b49e45e8', lease={ + "project_id": "fake-project", + }) + + def test_before_end_with_no_action(self): + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = {'before_end': ''} + fake_lease = {'project_id': 'fake-project'} + self.fake_dev_plugin.before_end( + '04de74e8-193a-49d2-9ab8-cba7b49e45e8', lease=fake_lease) + + def test_before_end_with_email(self): + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'before_end': '' + } + fake_lease = {'project_id': 'fake-project', "user_id": "fake-user"} + self.fake_dev_plugin.before_end( + '04de74e8-193a-49d2-9ab8-cba7b49e45e8', lease=fake_lease) + + def test_on_end_with_instances(self): + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '04de74e8-193a-49d2-9ab8-cba7b49e45e8', + 'reservation_id': '593e7028-c0d1-4d76-8642-2ffd890b324c', + } + device_reservation_update = self.patch( + self.db_api, + 'device_reservation_update') + device_allocation_get_all_by_values = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_allocation_get_all_by_values.return_value = [ + {'id': 'bfa9aa0b-8042-43eb-a4e6-4555838bf64f', + 'device_id': 'cdae2a65-236f-475a-977d-f6ad82f828b7', + }, + ] + device_allocation_destroy = self.patch( + self.db_api, + 'device_allocation_destroy') + + self.fake_dev_plugin.on_end( + '04de74e8-193a-49d2-9ab8-cba7b49e45e8', + lease={ + "project_id": "fake-project", + }, + ) + device_reservation_update.assert_called_with( + '04de74e8-193a-49d2-9ab8-cba7b49e45e8', {'status': 'completed'}) + device_allocation_destroy.assert_called_with( + 'bfa9aa0b-8042-43eb-a4e6-4555838bf64f') + + def test_on_end_without_instances(self): + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = { + 'id': '04de74e8-193a-49d2-9ab8-cba7b49e45e8', + 'reservation_id': '593e7028-c0d1-4d76-8642-2ffd890b324c', + } + device_reservation_update = self.patch( + self.db_api, + 'device_reservation_update') + device_allocation_get_all_by_values = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_allocation_get_all_by_values.return_value = [ + {'id': 'bfa9aa0b-8042-43eb-a4e6-4555838bf64f', + 'device_id': 'cdae2a65-236f-475a-977d-f6ad82f828b7', + }, + ] + device_allocation_destroy = self.patch( + self.db_api, + 'device_allocation_destroy') + self.fake_dev_plugin.on_end( + '04de74e8-193a-49d2-9ab8-cba7b49e45e8', + lease={ + "project_id": "fake-project", + }, + + ) + device_reservation_update.assert_called_with( + '04de74e8-193a-49d2-9ab8-cba7b49e45e8', {'status': 'completed'}) + device_allocation_destroy.assert_called_with( + 'bfa9aa0b-8042-43eb-a4e6-4555838bf64f') + + @skip # these tests pass when ran individually + def test_heal_reservations_before_start_and_resources_changed(self): + failed_device = {'id': '1'} + dummy_reservation = { + 'id': 'rsrv-1', + 'resource_type': plugin.RESOURCE_TYPE, + 'lease_id': 'lease-1', + 'status': 'pending', + 'resource_properties': [], + 'resource_id': 'resource-1', + 'device_allocations': [{ + 'id': 'alloc-1', 'device_id': failed_device['id'], + 'reservation_id': 'rsrv-1' + }] + } + get_reservations = self.patch(self.db_utils, + 'get_reservations_by_device_ids') + get_reservations.return_value = [dummy_reservation] + reallocate = self.patch(self.fake_dev_plugin.monitor, '_reallocate') + reallocate.return_value = True + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + + dummy_device_reservation = { + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = dummy_device_reservation + + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'}, + ] + + device_get = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + + def device_allocation_get_all_by_values(**kwargs): + if kwargs['device_id'] == 'device1': + return True + device_get.side_effect = device_allocation_get_all_by_values + device_get = self.patch( + self.db_utils, + 'get_free_periods') + device_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + self.patch(self.db_api, 'device_allocation_update') + + result = self.fake_dev_plugin.monitor.heal_reservations( + [failed_device], + datetime.datetime(2020, 1, 1, 12, 00), + datetime.datetime(2020, 1, 1, 13, 00)) + reallocate.assert_called_once_with( + dummy_reservation['device_allocations'][0]) + self.assertEqual({}, result) + + @skip + def test_heal_reservations_before_start_and_missing_resources(self): + failed_device = {'id': '1'} + dummy_reservation = { + 'id': 'rsrv-1', + 'resource_type': plugin.RESOURCE_TYPE, + 'lease_id': 'lease-1', + 'status': 'pending', + 'resource_properties': [], + 'resource_id': 'resource-1', + 'device_allocations': [{ + 'id': 'alloc-1', 'device_id': failed_device['id'], + 'reservation_id': 'rsrv-1' + }] + } + get_reservations = self.patch(self.db_utils, + 'get_reservations_by_device_ids') + get_reservations.return_value = [dummy_reservation] + reallocate = self.patch(self.fake_dev_plugin.monitor, '_reallocate') + reallocate.return_value = False + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + + dummy_device_reservation = { + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = dummy_device_reservation + + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'}, + ] + + device_get = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + + def device_allocation_get_all_by_values(**kwargs): + if kwargs['device_id'] == 'device1': + return True + device_get.side_effect = device_allocation_get_all_by_values + device_get = self.patch( + self.db_utils, + 'get_free_periods') + device_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + self.patch(self.db_api, 'device_allocation_update') + + result = self.fake_dev_plugin.monitor.heal_reservations( + [failed_device], + datetime.datetime(2020, 1, 1, 12, 00), + datetime.datetime(2020, 1, 1, 13, 00)) + reallocate.assert_called_once_with( + dummy_reservation['device_allocations'][0]) + self.assertEqual( + {dummy_reservation['id']: {'missing_resources': True}}, + result) + + def test_heal_active_reservations_and_resources_changed(self): + failed_device = {'id': '1'} + dummy_reservation = { + 'id': 'rsrv-1', + 'resource_type': plugin.RESOURCE_TYPE, + 'lease_id': 'lease-1', + 'status': 'active', + 'resource_properties': [], + 'resource_id': 'resource-1', + 'device_allocations': [{ + 'id': 'alloc-1', 'device_id': failed_device['id'], + 'reservation_id': 'rsrv-1' + }] + } + get_reservations = self.patch(self.db_utils, + 'get_reservations_by_device_ids') + get_reservations.return_value = [dummy_reservation] + + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + + dummy_device_reservation = { + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = dummy_device_reservation + + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'}, + ] + + device_get = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + + def device_allocation_get_all_by_values(**kwargs): + if kwargs['device_id'] == 'device1': + return True + device_get.side_effect = device_allocation_get_all_by_values + device_get = self.patch( + self.db_utils, + 'get_free_periods') + device_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + + reallocate = self.patch(self.fake_dev_plugin.monitor, '_reallocate') + reallocate.return_value = True + + self.patch(self.db_api, 'device_allocation_update') + + result = self.fake_dev_plugin.monitor.heal_reservations( + [failed_device], + datetime.datetime(2020, 1, 1, 12, 00), + datetime.datetime(2020, 1, 1, 13, 00)) + # skipped for leases which are active d285bc9 + reallocate.assert_not_called() + self.assertEqual({}, result) + + def test_heal_active_reservations_and_missing_resources(self): + failed_device = {'id': '1'} + dummy_reservation = { + 'id': 'rsrv-1', + 'resource_type': plugin.RESOURCE_TYPE, + 'lease_id': 'lease-1', + 'status': 'active', + 'resource_properties': [], + 'resource_id': 'resource-1', + 'device_allocations': [{ + 'id': 'alloc-1', 'device_id': failed_device['id'], + 'reservation_id': 'rsrv-1' + }] + } + get_reservations = self.patch(self.db_utils, + 'get_reservations_by_device_ids') + get_reservations.return_value = [dummy_reservation] + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + reallocate = self.patch(self.fake_dev_plugin.monitor, '_reallocate') + reallocate.return_value = False + dummy_device_reservation = { + } + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = dummy_device_reservation + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'}, + ] + device_get = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + + def device_allocation_get_all_by_values(**kwargs): + if kwargs['device_id'] == 'device1': + return True + device_get.side_effect = device_allocation_get_all_by_values + device_get = self.patch( + self.db_utils, + 'get_free_periods') + device_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + self.patch(self.db_api, 'device_allocation_update') + + result = self.fake_dev_plugin.monitor.heal_reservations( + [failed_device], + datetime.datetime(2020, 1, 1, 12, 00), + datetime.datetime(2020, 1, 1, 13, 00)) + # skipped for leases which are active d285bc9 + reallocate.assert_not_called() + self.assertEqual({}, result) + + def test_reallocate_before_start(self): + failed_device = {'id': '1'} + new_device = {'id': '2'} + dummy_allocation = { + 'id': 'alloc-1', + 'device_id': failed_device['id'], + 'reservation_id': 'rsrv-1', + } + dummy_reservation = { + 'id': 'rsrv-1', + 'resource_type': plugin.RESOURCE_TYPE, + 'lease_id': 'lease-1', + 'status': 'pending', + 'resource_properties': [], + 'resource_id': 'resource-1' + } + dummy_device_reservation = { + "resource_properties": [], + } + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1', + 'project_id': 'fake-project' + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = dummy_device_reservation + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + matching_devices = self.patch(device_plugin.DevicePlugin, + '_matching_devices') + matching_devices.return_value = [new_device['id']] + alloc_update = self.patch(self.db_api, 'device_allocation_update') + + with mock.patch.object(datetime, 'datetime', + mock.Mock(wraps=datetime.datetime)) as patched: + patched.utcnow.return_value = datetime.datetime( + 2020, 1, 1, 11, 00) + result = self.fake_dev_plugin._reallocate(dummy_allocation) + + matching_devices.assert_called_once_with( + dummy_reservation['resource_properties'], + '1-1', dummy_lease['start_date'], dummy_lease['end_date'], + dummy_lease['project_id'], + ) + alloc_update.assert_called_once_with( + dummy_allocation['id'], + {'device_id': new_device['id']}) + self.assertEqual(True, result) + + def test_reallocate_active(self): + failed_device = {'id': '1', + "device_driver": "k8s", + "name": "rpi1", + } + new_device = {'id': '2', + "device_driver": "k8s", + "name": "rpi1" + } + dummy_allocation = { + 'id': 'alloc-1', + 'device_id': failed_device['id'], + 'reservation_id': 'rsrv-1' + } + dummy_reservation = { + 'id': 'rsrv-1', + 'resource_type': plugin.RESOURCE_TYPE, + 'lease_id': 'lease-1', + 'status': 'active', + 'resource_properties': [], + 'resource_id': 'resource-1', + } + dummy_device_reservation = { + "resource_properties": [], + "reservation_id": 'rsrv-1', + } + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1', + 'project_id': 'fake-project' + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = dummy_device_reservation + device_get = self.patch(self.db_api, 'device_get') + device_get.side_effect = [failed_device, new_device] + matching_devices = self.patch(device_plugin.DevicePlugin, + '_matching_devices') + matching_devices.return_value = [new_device['id']] + alloc_update = self.patch(self.db_api, 'device_allocation_update') + + with mock.patch.object(datetime, 'datetime', + mock.Mock(wraps=datetime.datetime)) as patched: + patched.utcnow.return_value = datetime.datetime( + 2020, 1, 1, 13, 00) + result = self.fake_dev_plugin._reallocate(dummy_allocation) + + matching_devices.assert_called_once_with( + dummy_reservation['resource_properties'], + '1-1', datetime.datetime(2020, 1, 1, 13, 00), + dummy_lease['end_date'], + 'fake-project', + ) + alloc_update.assert_called_once_with( + dummy_allocation['id'], + {'device_id': new_device['id']}) + self.assertEqual(True, result) + + def test_reallocate_missing_resources(self): + failed_device = {'id': '1'} + dummy_allocation = { + 'id': 'alloc-1', + 'device_id': failed_device['id'], + 'reservation_id': 'rsrv-1' + } + dummy_reservation = { + 'id': 'rsrv-1', + 'resource_type': plugin.RESOURCE_TYPE, + 'lease_id': 'lease-1', + 'status': 'pending', + 'resource_properties': [], + 'resource_id': 'resource-1' + } + dummy_device_reservation = { + 'resource_properties': [], + } + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1', + 'project_id': 'fake-project' + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + device_reservation_get = self.patch(self.db_api, 'device_reservation_get') + device_reservation_get.return_value = dummy_device_reservation + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + matching_devices = self.patch(device_plugin.DevicePlugin, + '_matching_devices') + matching_devices.return_value = [] + alloc_destroy = self.patch(self.db_api, 'device_allocation_destroy') + + with mock.patch.object(datetime, 'datetime', + mock.Mock(wraps=datetime.datetime)) as patched: + patched.utcnow.return_value = datetime.datetime( + 2020, 1, 1, 11, 00) + result = self.fake_dev_plugin._reallocate(dummy_allocation) + + matching_devices.assert_called_once_with( + dummy_reservation['resource_properties'], + '1-1', dummy_lease['start_date'], dummy_lease['end_date'], + dummy_lease['project_id'], + ) + alloc_destroy.assert_called_once_with(dummy_allocation['id']) + self.assertEqual(False, result) + + def test_matching_devices_not_allocated_devices(self): + def device_allocation_get_all_by_values(**kwargs): + if kwargs['device_id'] == 'device1': + return True + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'}, + ] + device_get = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_get.side_effect = device_allocation_get_all_by_values + device_get = self.patch( + self.db_utils, + 'get_free_periods') + device_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + result = self.fake_dev_plugin._matching_devices( + '[]', '1-3', + datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00), + 'fake-project' + ) + self.assertEqual(set(['device2', 'device3']), set(result)) + + def test_matching_devices_allocated_devices(self): + def device_allocation_get_all_by_values(**kwargs): + if kwargs['device_id'] == 'device1': + return True + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'}, + ] + device_get = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_get.side_effect = device_allocation_get_all_by_values + device_get = self.patch( + self.db_utils, + 'get_free_periods') + device_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + result = self.fake_dev_plugin._matching_devices( + '[]', '3-3', + datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00), + None + ) + self.assertEqual(set(['device1', 'device2', 'device3']), set(result)) + + @mock.patch.object(random, "shuffle") + def test_matching_devices_allocated_devices_with_cleaning_time(self, mock_shuffle): + def device_allocation_get_all_by_values(**kwargs): + if kwargs['device_id'] == 'device1': + return True + self.cfg.CONF.set_override('cleaning_time', '5') + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'}, + ] + device_get = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_get.side_effect = device_allocation_get_all_by_values + get_free_periods = self.patch( + self.db_utils, + 'get_free_periods') + get_free_periods.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00) + - datetime.timedelta(minutes=5), + datetime.datetime(2013, 12, 19, 21, 00) + + datetime.timedelta(minutes=5)) + ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + result = self.fake_dev_plugin._matching_devices( + '[]', '3-3', + datetime.datetime(2013, 12, 19, 20, 00) - datetime.timedelta(minutes=5), + datetime.datetime(2013, 12, 19, 21, 00) + datetime.timedelta(minutes=5), + None) + mock_shuffle.assert_called_once_with(['device1', 'device2', 'device3']) + self.addCleanup(CONF.clear_override, 'cleaning_time') + + + @mock.patch.object(random, "shuffle") + def test_random_matching_devices_not_allocated_devices(self, mock_shuffle): + def device_allocation_get_all_by_values(**kwargs): + if kwargs['device_id'] == 'device1': + return True + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'}, + ] + device_get = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_get.side_effect = device_allocation_get_all_by_values + device_get = self.patch( + self.db_utils, + 'get_free_periods') + device_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + self.fake_dev_plugin._matching_devices( + '[]', '1-3', + datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00), + None) + mock_shuffle.assert_called_once_with(['device2', 'device3']) + + @mock.patch.object(random, "shuffle") + def test_random_matching_devices_allocated_devices(self, mock_shuffle): + def device_allocation_get_all_by_values(**kwargs): + if kwargs['device_id'] == 'device1': + return True + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'}, + ] + device_get = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_get.side_effect = device_allocation_get_all_by_values + device_get = self.patch( + self.db_utils, + 'get_free_periods') + device_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + self.fake_dev_plugin._matching_devices( + '[]', '3-3', + datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00), + None) + mock_shuffle.assert_called_once_with(['device1', 'device2', 'device3']) + + @mock.patch.object(random, "shuffle") + def test_random_matching_devices_allocated_cleaning_time(self, mock_shuffle): + def device_allocation_get_all_by_values(**kwargs): + if kwargs['device_id'] == 'device1': + return True + self.cfg.CONF.set_override('cleaning_time', '5') + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [ + {'id': 'device1'}, + {'id': 'device2'}, + {'id': 'device3'}, + ] + device_get = self.patch( + self.db_api, + 'device_allocation_get_all_by_values') + device_get.side_effect = device_allocation_get_all_by_values + device_get = self.patch( + self.db_utils, + 'get_free_periods') + device_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00) + - datetime.timedelta(minutes=5), + datetime.datetime(2013, 12, 19, 21, 00) + + datetime.timedelta(minutes=5)) + ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + self.fake_dev_plugin._matching_devices( + '[]', '2-3', + datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00), + None) + self.addCleanup(CONF.clear_override, 'cleaning_time') + mock_shuffle.assert_called_once_with(['device2', 'device3']) + + def test_matching_devices_not_matching(self): + device_get = self.patch( + self.db_api, + 'reservable_device_get_all_by_queries') + device_get.return_value = [] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + result = self.fake_dev_plugin._matching_devices( + '["=", "$memory_mb", "2048"]', '1-1', + datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00), + None + ) + self.assertEqual([], result) + + def test_check_params_with_valid_before_end(self): + values = { + 'min': 1, + 'max': 2, + 'resource_properties': '', + 'before_end': 'email' + } + self.fake_dev_plugin._check_params(values) + self.assertEqual(values['before_end'], 'email') + + def test_check_params_with_invalid_before_end(self): + values = { + 'min': 1, + 'max': 2, + 'resource_properties': '', + 'before_end': 'invalid' + } + self.assertRaises(manager_exceptions.MalformedParameter, + self.fake_dev_plugin._check_params, + values) + + def test_check_params_without_before_end(self): + self.cfg.CONF.set_override('before_end', '', + group='device') + values = { + 'min': 1, + 'max': 2, + 'resource_properties': '', + } + self.fake_dev_plugin._check_params(values) + self.assertEqual(values['before_end'], 'default') + + def test_list_resource_properties(self): + self.db_list_resource_properties = self.patch( + self.db_api, 'resource_properties_list') + + # Expecting a list of (Reservation, Allocation) + self.db_list_resource_properties.return_value = [ + ('prop1', False, 'aaa', False), + ('prop1', False, 'bbb', False), + ('prop2', False, 'aaa', False), + ('prop2', False, 'aaa', False), + ('prop3', True, 'aaa', False) + ] + + expected = [ + {'property': 'prop1',}, + {'property': 'prop2',} + ] + + ret = self.fake_dev_plugin.list_resource_properties( + query={'detail': False}) + + # Sort returned value to use assertListEqual + ret.sort(key=lambda x: x['property']) + + self.assertListEqual(expected, ret) + self.db_list_resource_properties.assert_called_once_with( + 'device') + + def test_list_resource_properties_with_detail(self): + self.db_list_resource_properties = self.patch( + self.db_api, 'resource_properties_list') + + # Expecting a list of (Reservation, Allocation) + self.db_list_resource_properties.return_value = [ + ('prop1', False, 'aaa', False), + ('prop1', False, 'bbb', False), + ('prop2', False, 'ccc', False), + ('prop3', True, 'aaa', False) + ] + + expected = [ + {'property': 'prop1', 'private': False, 'values': ['aaa', 'bbb'], 'is_unique': False}, + {'property': 'prop2', 'private': False, 'values': ['ccc'], 'is_unique': False} + ] + + ret = self.fake_dev_plugin.list_resource_properties( + query={'detail': True}) + + # Sort returned value to use assertListEqual + ret.sort(key=lambda x: x['property']) + + self.assertListEqual(expected, ret) + self.db_list_resource_properties.assert_called_once_with( + 'device') + + def test_update_resource_property(self): + resource_property_values = { + 'resource_type': 'device', + 'private': False} + + db_resource_property_update = self.patch( + self.db_api, 'resource_property_update') + + self.fake_dev_plugin.update_resource_property( + 'foo', resource_property_values) + db_resource_property_update.assert_called_once_with( + 'device', 'foo', resource_property_values) diff --git a/blazar/tests/plugins/flavor/__init__.py b/blazar/tests/plugins/flavor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/blazar/tests/plugins/flavor/test_flavor_plugin.py b/blazar/tests/plugins/flavor/test_flavor_plugin.py new file mode 100644 index 000000000..9db229b35 --- /dev/null +++ b/blazar/tests/plugins/flavor/test_flavor_plugin.py @@ -0,0 +1,697 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import datetime +import json +from unittest import mock + +from novaclient.v2 import flavors +from oslo_config import cfg + +from blazar import context +from blazar.db.sqlalchemy import api as db_api +from blazar.db import utils as db_utils +from blazar.manager import exceptions as mgr_exceptions +from blazar.plugins.flavor import flavor_plugin +from blazar.plugins.oshosts import host_plugin +from blazar import tests +from blazar.tests.db.sqlalchemy import test_sqlalchemy_api as fake +from blazar.utils.openstack import nova +from blazar.utils.openstack import placement + + +class TestFlavorPlugin(tests.DBTestCase): + def _create_fake_host(self, id=123, hypervisor_hostname=None, **kwargs): + host_values = fake._get_fake_host_values(id=id) + if hypervisor_hostname: + host_values['hypervisor_hostname'] = hypervisor_hostname + host_values.update(kwargs) + host_values["reservable"] = 1 + db_api.host_create(host_values) + + def _create_lease_and_reservation(self, lease_id, start_date, end_date, + host_id, reservation_id, + flavor_id='flavor1'): + db_api.lease_create({ + 'id': lease_id, + 'name': lease_id, + 'project_id': 'proj1', + 'start_date': start_date, + 'end_date': end_date, + 'user_id': 'user1', + 'trust_id': 'trust1', + }) + for event_type, event_time in [('start_lease', start_date), + ('end_lease', end_date)]: + db_api.event_create({ + 'lease_id': lease_id, + 'event_type': event_type, + 'time': event_time, + 'status': 'pending' if event_type == 'end_lease' else 'done', + }) + + inst_res = { + 'id': reservation_id, + 'lease_id': lease_id, + 'resource_id': 'inst-' + reservation_id, + 'resource_type': 'virtual:instance', + 'status': 'active', + 'start_date': start_date, + 'end_date': end_date, + 'project_id': 'proj1', + } + db_api.reservation_create(inst_res) + inst_res_details = { + 'id': 'inst-' + reservation_id, + 'reservation_id': reservation_id, + 'flavor_id': flavor_id, + 'amount': 1, + 'vcpus': 1, + 'memory_mb': 1024, + 'disk_gb': 10, + 'affinity': False, + 'resource_properties': ( + '{"OS-FLV-EXT-DATA:ephemeral": 0, "disk": 10, "ram": 1024, ' + '"vcpus": 1, "extra_specs": {}}'), + } + db_api.instance_reservation_create(inst_res_details) + db_api.host_allocation_create({ + 'compute_host_id': host_id, + 'reservation_id': reservation_id, + }) + + def test_get(self): + plugin = flavor_plugin.FlavorPlugin() + resource_id = '123' + self._create_fake_host() + + result = plugin.get(resource_id) + + self.assertEqual(resource_id, result['id']) + + def test_list_allocations(self): + plugin = flavor_plugin.FlavorPlugin() + + result = plugin.list_allocations({'lease_id': '2001'}) + + self.assertEqual(0, len(result)) + + @mock.patch.object(host_plugin.PhysicalHostPlugin, 'query_allocations') + def test_query_allocations(self, mock_query): + plugin = flavor_plugin.FlavorPlugin() + mock_query.return_value = "fake" + + result = plugin.query_allocations(['123'], lease_id='2001') + + self.assertEqual("fake", result) + mock_query.assert_called_once_with(['123'], '2001', None) + + @mock.patch.object(flavor_plugin.FlavorPlugin, '_get_flavor_details') + def test_allocation_candidates(self, mock_get_flavor): + self._create_fake_host() + fake_inventory_values = { + 'computehost_id': 123, + 'resource_class': 'PCPU', + 'total': 10, + 'reserved': 2, + 'min_unit': 1, + 'max_unit': 10, + 'step_size': 1, + 'allocation_ratio': 1.0 + } + db_api.host_resource_inventory_create(fake_inventory_values) + plugin = flavor_plugin.FlavorPlugin() + reservation = { + 'flavor_id': "34eb7166-0e9b-432c-96fd-dff37f22e36e", + 'amount': 4, + 'project_id': 'fake', + 'affinity': None, + 'start_date': datetime.datetime(2030, 1, 1, 8, 00), + 'end_date': datetime.datetime(2030, 1, 1, 12, 00) + } + mock_get_flavor.return_value = ({"PCPU": 2}, {}, + {"flavor_id": "fake"}) + + result = plugin.allocation_candidates(reservation) + + self.assertEqual(4, len(result)) + mock_get_flavor.assert_called_once_with( + "34eb7166-0e9b-432c-96fd-dff37f22e36e") + + @mock.patch.object(flavor_plugin.FlavorPlugin, '_get_flavor_details') + def test_allocation_candidates_fails_no_space(self, mock_get_flavor): + self._create_fake_host() + fake_inventory_values = { + 'computehost_id': 123, + 'resource_class': 'PCPU', + 'total': 10, + 'reserved': 2, + 'min_unit': 1, + 'max_unit': 10, + 'step_size': 1, + 'allocation_ratio': 1.0 + } + db_api.host_resource_inventory_create(fake_inventory_values) + plugin = flavor_plugin.FlavorPlugin() + reservation = { + 'flavor_id': "34eb7166-0e9b-432c-96fd-dff37f22e36e", + 'amount': 5, + 'project_id': 'fake', + 'affinity': None, + 'start_date': datetime.datetime(2030, 1, 1, 8, 00), + 'end_date': datetime.datetime(2030, 1, 1, 12, 00) + } + mock_get_flavor.return_value = ({"PCPU": 2}, {}, + {"flavor_id": "fake"}) + + self.assertRaises(mgr_exceptions.NotEnoughHostsAvailable, + plugin.allocation_candidates, + reservation) + + @mock.patch.object(flavor_plugin.FlavorPlugin, '_create_resources') + @mock.patch.object(flavor_plugin.FlavorPlugin, '_get_flavor_details') + def test_allocation_candidates_avoids_reservations(self, mock_get_flavor, + mock_create): + self._create_fake_host() + fake_inventory_values = { + 'computehost_id': 123, + 'resource_class': 'PCPU', + 'total': 10, + 'reserved': 2, + 'min_unit': 1, + 'max_unit': 10, + 'step_size': 1, + 'allocation_ratio': 1.0 + } + db_api.host_resource_inventory_create(fake_inventory_values) + plugin = flavor_plugin.FlavorPlugin() + new_reservation = { + 'flavor_id': "34eb7166-0e9b-432c-96fd-dff37f22e36e", + 'amount': 3, + 'project_id': 'fake', + 'affinity': None, + 'start_date': datetime.datetime(2030, 1, 1, 8, 00), + 'end_date': datetime.datetime(2030, 1, 1, 12, 00) + } + fake_flavor = { + "disk": 0, # GiB + "OS-FLV-EXT-DATA:ephemeral": 0, # GiB + "id": "34eb7166-0e9b-432c-96fd-dff37f22e36e", + "name": "test1", + "ram": 0, # MB + "swap": 0, + "vcpus": 2, + "extra_specs": {'hw:cpu_policy': 'dedicated'} + } + mock_get_flavor.return_value = ({"PCPU": 2}, {}, + fake_flavor) + old_reservation = new_reservation.copy() + old_reservation['amount'] = 2 + fake_phys_reservation = new_reservation.copy() + fake_phys_reservation['id'] = 345 + fake_start_event = { + 'id': 123, + 'lease_id': 1234, + 'event_type': "start_lease", + 'time': datetime.datetime(2030, 1, 1, 8, 00), + 'status': "fake", + } + fake_lease = { + 'id': 1234, + 'name': "fakelease", + 'user_id': 'fake', + 'project_id': 'fake', + 'start_date': datetime.datetime(2030, 1, 1, 8, 00), + 'end_date': datetime.datetime(2030, 1, 1, 12, 00), + 'trust': 'trust', + 'reservations': [fake_phys_reservation], + 'events': [fake_start_event] + } + db_api.lease_create(fake_lease) + mock_create.return_value = ("flavor_id", "aggregate_id") + # create a reservation to avoid + plugin.reserve_resource("345", old_reservation) + + # Host as 10 PCPUs, 2 are reserved, leaving 8 PCPUs available + # Old reservation is for 2 flavors needing 2 each, so 4 left + # So there should be space for 2 lots of 2 PCPUs + new_reservation['amount'] = 2 + result = plugin.allocation_candidates(new_reservation) + self.assertEqual(2, len(result)) + + # there should not be space for 3 lots of 2 PCPUs + new_reservation['amount'] = 3 + self.assertRaises(mgr_exceptions.NotEnoughHostsAvailable, + plugin.allocation_candidates, + new_reservation) + + @mock.patch.object(flavors.FlavorManager, 'get') + def test__get_flavor_details(self, mock_get): + plugin = flavor_plugin.FlavorPlugin() + mock_flavor = mock.Mock() + mock_get.return_value = mock_flavor + fake_flavor = { + "disk": 10, # GiB + "OS-FLV-EXT-DATA:ephemeral": 0, # GiB + "id": "34eb7166-0e9b-432c-96fd-dff37f22e36e", + "name": "test1", + "ram": 1024, # MB + "swap": 0, + "vcpus": 1, + } + mock_flavor.to_dict.return_value = fake_flavor + fake_extra_specs = { + 'hw:cpu_policy': 'dedicated', + 'trait:HW_CPU_X86_AVX': 'required', + 'resources:VGPU': '1', + } + mock_flavor.get_keys.return_value = fake_extra_specs + + resource_request, resource_traits, source_flavor = \ + plugin._get_flavor_details("34eb7166-0e9b-432c-96fd-dff37f22e36e") + + self.assertDictEqual({ + 'DISK_GB': 10, 'MEMORY_MB': 1024, 'PCPU': 1, 'VCPU': 0, 'VGPU': 1 + }, resource_request) + self.assertDictEqual({'HW_CPU_X86_AVX': 'required'}, resource_traits) + expected = fake_flavor.copy() + expected["extra_specs"] = fake_extra_specs + self.assertDictEqual(expected, source_flavor) + + @mock.patch.object(flavor_plugin.FlavorPlugin, '_create_resources') + @mock.patch.object(flavor_plugin.FlavorPlugin, '_pick_hosts') + def test_reserve_resource(self, mock_pick, mock_create): + plugin = flavor_plugin.FlavorPlugin() + reservation = { + 'flavor_id': "34eb7166-0e9b-432c-96fd-dff37f22e36e", + 'amount': 5, + 'affinity': None, + 'start_date': datetime.datetime(2030, 1, 1, 8, 00), + 'end_date': datetime.datetime(2030, 1, 1, 12, 00) + } + fake_flavor = { + "disk": 10, # GiB + "OS-FLV-EXT-DATA:ephemeral": 0, # GiB + "id": "34eb7166-0e9b-432c-96fd-dff37f22e36e", + "name": "test1", + "ram": 1024, # MB + "swap": 0, + "vcpus": 1, + } + mock_pick.return_value = (['123', '124'], fake_flavor) + mock_create.return_value = ("flavor_id", "aggregate_id") + + id = plugin.reserve_resource("345", reservation) + + allocations = db_api.host_allocation_get_all_by_values( + reservation_id="345") + self.assertEqual(2, len(allocations)) + reservation = db_api.instance_reservation_get(id) + self.assertEqual("345", reservation["reservation_id"]) + self.assertEqual("flavor_id", reservation["flavor_id"]) + self.assertEqual("aggregate_id", reservation["aggregate_id"]) + + @mock.patch.object(nova.ReservationPool, 'create') + @mock.patch.object(context.BlazarContext, 'current') + @mock.patch.object(placement.BlazarPlacementClient, + 'create_reservation_class') + @mock.patch.object(flavor_plugin.FlavorPlugin, '_create_flavor') + def test_create_resources(self, mock_create_flavor, + mock_reservation_create, + mock_current, mock_pool_create): + plugin = flavor_plugin.FlavorPlugin() + fake_flavor = { + "disk": 10, # GiB + "OS-FLV-EXT-DATA:ephemeral": 100, # GiB + "id": "34eb7166-0e9b-432c-96fd-dff37f22e36e", + "name": "test1", + "ram": 1024, # MB + "swap": 0, + "vcpus": 2, + "extra_specs": {'hw:cpu_policy': 'dedicated'} + } + fake_reservation = { + 'reservation_id': "12345", + 'vcpus': fake_flavor["vcpus"], + 'memory_mb': fake_flavor["ram"], + 'disk_gb': fake_flavor["disk"], + 'amount': 2, + 'affinity': None, + 'resource_properties': json.dumps(fake_flavor) + } + mock_context = mock.Mock() + mock_context.project_id = "fake-project-id" + mock_current.return_value = mock_context + mock_aggregate = mock.Mock() + mock_aggregate.id = "aggregate_id" + mock_pool_create.return_value = mock_aggregate + mock_flavor = mock.Mock() + mock_flavor.id = "flavor_id" + mock_create_flavor.return_value = mock_flavor + + fid, aid = plugin._create_resources(fake_reservation) + + self.assertEqual(fid, "flavor_id") + self.assertEqual(aid, "aggregate_id") + mock_create_flavor.assert_called_once_with(fake_reservation) + mock_reservation_create.assert_called_once_with("12345") + mock_current.assert_called_once_with() + mock_pool_create.assert_called_once_with( + name="12345", + metadata={'reservation': '12345', + 'filter_tenant_id': 'fake-project-id'} + ), + + @mock.patch.object(flavors.FlavorManager, 'create') + def test_create_flavor(self, mock_create): + plugin = flavor_plugin.FlavorPlugin() + fake_flavor = { + "disk": 10, # GiB + "OS-FLV-EXT-DATA:ephemeral": 100, # GiB + "id": "34eb7166-0e9b-432c-96fd-dff37f22e36e", + "name": "test1", + "ram": 1024, # MB + "swap": 0, + "vcpus": 2, + "extra_specs": {'hw:cpu_policy': 'dedicated'} + } + fake_reservation = { + 'reservation_id': "12345", + 'vcpus': fake_flavor["vcpus"], + 'memory_mb': fake_flavor["ram"], + 'disk_gb': fake_flavor["disk"], + 'amount': 2, + 'affinity': None, + 'resource_properties': json.dumps(fake_flavor) + } + mock_flavor = mock.Mock() + mock_create.return_value = mock_flavor + + plugin._create_flavor(fake_reservation) + + mock_flavor.set_keys.assert_called_once_with({ + 'hw:cpu_policy': 'dedicated', + 'aggregate_instance_extra_specs:reservation': '12345', + 'resources:CUSTOM_RESERVATION_12345': '1', + }) + mock_create.assert_called_once_with( + flavorid='12345', name='reservation:12345', vcpus=2, ram=1024, + disk=10, is_public=False) + + def test__query_available_hosts(self): + get_reservations = self.patch(db_utils, + 'get_reservations_by_host_id') + get_reservations.return_value = [] + plugin = flavor_plugin.FlavorPlugin() + + # Check that we can fit 4 VCPU resource requests + self._create_fake_host() + fake_inventory_values = { + 'computehost_id': 123, + 'resource_class': 'VCPU', + 'total': 4, + 'reserved': 0, + 'min_unit': 1, + 'max_unit': 4, + 'step_size': 1, + 'allocation_ratio': 1.0 + } + db_api.host_resource_inventory_create(fake_inventory_values) + + query_params = { + 'start_date': datetime.datetime(2020, 7, 7, 18, 0), + 'end_date': datetime.datetime(2020, 7, 7, 19, 0), + 'resource_request': { + 'VCPU': 1, + }, + 'resource_traits': {}, + 'project_id': 'fake', + } + ret = plugin._query_available_hosts(**query_params) + self.assertEqual(4, len(ret)) + + # Only 2 * 1024 MB requests fit when we add a MEMORY_MB inventory + fake_inventory_values = { + 'computehost_id': 123, + 'resource_class': 'MEMORY_MB', + 'total': 2048, + 'reserved': 0, + 'min_unit': 1, + 'max_unit': 2048, + 'step_size': 1, + 'allocation_ratio': 1.0 + } + db_api.host_resource_inventory_create(fake_inventory_values) + + query_params = { + 'start_date': datetime.datetime(2020, 7, 7, 18, 0), + 'end_date': datetime.datetime(2020, 7, 7, 19, 0), + 'resource_request': { + 'VCPU': 1, + 'MEMORY_MB': 1024 + }, + 'resource_traits': {}, + 'project_id': 'fake', + } + ret = plugin._query_available_hosts(**query_params) + self.assertEqual(2, len(ret)) + + def test__query_available_hosts_excludes_ironic_hosts(self): + get_reservations = self.patch(db_utils, + 'get_reservations_by_host_id') + get_reservations.return_value = [] + plugin = flavor_plugin.FlavorPlugin() + + # A QEMU (virtual) host and an ironic (baremetal) host, both + # reservable with identical VCPU inventory. + for host_id, hv_type in ((123, 'QEMU'), (456, 'ironic')): + host_values = fake._get_fake_host_values(id=host_id) + host_values['reservable'] = 1 + host_values['hypervisor_type'] = hv_type + db_api.host_create(host_values) + db_api.host_resource_inventory_create({ + 'computehost_id': host_id, + 'resource_class': 'VCPU', + 'total': 4, + 'reserved': 0, + 'min_unit': 1, + 'max_unit': 4, + 'step_size': 1, + 'allocation_ratio': 1.0, + }) + + query_params = { + 'start_date': datetime.datetime(2020, 7, 7, 18, 0), + 'end_date': datetime.datetime(2020, 7, 7, 19, 0), + 'resource_request': {'VCPU': 1}, + 'resource_traits': {}, + 'project_id': 'fake', + } + + # Default (filter_ironic_hosts=True): the ironic host is excluded, so + # only the QEMU host's 4 slots are returned. + ret = plugin._query_available_hosts(**query_params) + self.assertEqual(4, len(ret)) + self.assertTrue(all(h['hypervisor_type'] != 'ironic' for h in ret)) + + # When disabled, both hosts contribute (4 + 4 slots). + cfg.CONF.set_override('filter_ironic_hosts', False, + group=plugin.resource_type) + self.addCleanup(cfg.CONF.clear_override, 'filter_ironic_hosts', + group=plugin.resource_type) + ret = plugin._query_available_hosts(**query_params) + self.assertEqual(8, len(ret)) + + def test__query_available_hosts_honors_project_restriction(self): + get_reservations = self.patch(db_utils, + 'get_reservations_by_host_id') + get_reservations.return_value = [] + plugin = flavor_plugin.FlavorPlugin() + + # A host restricted to project "proj-a" via the authorized_projects + # extra capability. + self._create_fake_host() + db_api.host_extra_capability_create( + fake._get_fake_host_extra_capabilities( + computehost_id=123, name='authorized_projects', + value='proj-a')) + db_api.host_resource_inventory_create({ + 'computehost_id': 123, + 'resource_class': 'VCPU', + 'total': 4, + 'reserved': 0, + 'min_unit': 1, + 'max_unit': 4, + 'step_size': 1, + 'allocation_ratio': 1.0, + }) + + query_params = { + 'start_date': datetime.datetime(2020, 7, 7, 18, 0), + 'end_date': datetime.datetime(2020, 7, 7, 19, 0), + 'resource_request': {'VCPU': 1}, + 'resource_traits': {}, + } + + # Authorized project sees the host's 4 slots. + ret = plugin._query_available_hosts(project_id='proj-a', **query_params) + self.assertEqual(4, len(ret)) + + # Unauthorized project sees none. + ret = plugin._query_available_hosts(project_id='proj-b', **query_params) + self.assertEqual(0, len(ret)) + + def _fake_flavor_details(self): + return ( + {"VCPU": 1}, + {}, + { + "vcpus": 1, + "ram": 1024, + "disk": 10, + "OS-FLV-EXT-DATA:ephemeral": 0, + }, + ) + + def _create_host_with_vcpu(self, host_id): + self._create_fake_host( + id=host_id, hypervisor_hostname=host_id, vcpus=1) + db_api.host_resource_inventory_create({ + 'computehost_id': host_id, + 'resource_class': 'VCPU', + 'total': 1, + 'reserved': 0, + 'min_unit': 1, + 'max_unit': 1, + 'step_size': 1, + 'allocation_ratio': 1.0, + }) + + def test_update_reservation_extension_should_pass_with_full_host(self): + # Scenario: Host has 1 VCPU. + # Reservation consumes 1 VCPU. + # User extends Reservation. + # Should succeed because the reservation's own usage should be excluded. + + plugin = flavor_plugin.FlavorPlugin() + + # 1. Setup Host + self._create_host_with_vcpu('host1') + + # 2. Setup Existing Reservation + self._create_lease_and_reservation( + lease_id='lease-1', + start_date=datetime.datetime(2030, 1, 1, 10, 0), + end_date=datetime.datetime(2030, 1, 1, 11, 0), + host_id='host1', + reservation_id='res-1' + ) + + new_values = { + 'start_date': datetime.datetime(2030, 1, 1, 10, 0), + 'end_date': datetime.datetime(2030, 1, 1, 12, 0), + 'project_id': 'proj1', + 'amount': 1, + 'flavor_id': 'flavor1', + } + + # Mock Nova/Flavor/Placement calls to avoid external calls + # We need _get_flavor_details to return our VCPU requirement + with mock.patch.object(plugin, '_get_flavor_details') as mock_get_flavor: + mock_get_flavor.return_value = self._fake_flavor_details() + plugin.update_reservation('res-1', new_values) + + def test_update_reservation_extension_should_fail_if_blocked(self): + # Scenario: Host has 1 VCPU. + # Lease 1: 10:00-11:00 (1 VCPU). + # Lease 2: 11:00-12:00 (1 VCPU). + # User extends Lease 1 to 12:00. + # Should FAIL because Lease 2 consumes the resource 11:00-12:00. + + plugin = flavor_plugin.FlavorPlugin() + + # 1. Setup Host + self._create_host_with_vcpu('host1') + + # 2. Setup Lease 1 (The one we update) + self._create_lease_and_reservation( + lease_id='lease-1', + start_date=datetime.datetime(2030, 1, 1, 10, 0), + end_date=datetime.datetime(2030, 1, 1, 11, 0), + host_id='host1', + reservation_id='res-1') + + # 3. Setup Lease 2 (Blocking lease) + self._create_lease_and_reservation( + lease_id='lease-2', + start_date=datetime.datetime(2030, 1, 1, 11, 0), + end_date=datetime.datetime(2030, 1, 1, 12, 0), + host_id='host1', + reservation_id='res-2') + + new_values = { + 'start_date': datetime.datetime(2030, 1, 1, 10, 0), + 'end_date': datetime.datetime(2030, 1, 1, 12, 0), + 'project_id': 'proj1', + 'amount': 1, + 'flavor_id': 'flavor1', + } + + with mock.patch.object(plugin, '_get_flavor_details') as mock_get_flavor: + mock_get_flavor.return_value = self._fake_flavor_details() + self.assertRaises(mgr_exceptions.NotEnoughHostsAvailable, + plugin.update_reservation, + 'res-1', new_values) + + def test_update_reservation_extension_mixed_hosts(self): + # Scenario: + # Host 1 (1 VCPU). Lease 1 (1 VCPU) 10:00-11:00. + # Host 2 (1 VCPU). Lease 2 (1 VCPU) 10:00-12:00. + # User extends Lease 1 to 12:00. + # Host 1 should be available (exclude self). + # Host 2 should be unavailable (Lease 2). + + plugin = flavor_plugin.FlavorPlugin() + + # 1. Setup Hosts + self._create_host_with_vcpu('host1') + self._create_host_with_vcpu('host2') + + # 2. Setup Lease 1 on Host 1 + self._create_lease_and_reservation( + lease_id='lease-1', + start_date=datetime.datetime(2030, 1, 1, 10, 0), + end_date=datetime.datetime(2030, 1, 1, 11, 0), + host_id='host1', + reservation_id='res-1' + ) + + # 3. Setup Lease 2 on Host 2 (Blocking) + self._create_lease_and_reservation( + lease_id='lease-2', + start_date=datetime.datetime(2030, 1, 1, 10, 0), + end_date=datetime.datetime(2030, 1, 1, 12, 0), + host_id='host2', + reservation_id='res-2' + ) + + new_values = { + 'start_date': datetime.datetime(2030, 1, 1, 10, 0), + 'end_date': datetime.datetime(2030, 1, 1, 12, 0), + 'project_id': 'proj1', + 'amount': 1, + 'flavor_id': 'flavor1', + } + + with mock.patch.object(plugin, '_get_flavor_details') as mock_get_flavor: + mock_get_flavor.return_value = self._fake_flavor_details() + plugin.update_reservation('res-1', new_values) diff --git a/blazar/tests/plugins/floatingips/test_floatingip_plugin.py b/blazar/tests/plugins/floatingips/test_floatingip_plugin.py index 351f15ca4..adf3fee39 100644 --- a/blazar/tests/plugins/floatingips/test_floatingip_plugin.py +++ b/blazar/tests/plugins/floatingips/test_floatingip_plugin.py @@ -16,6 +16,7 @@ from unittest import mock from oslo_config import cfg +from oslo_config import fixture as conf_fixture from blazar import context from blazar.db import api as db_api @@ -24,9 +25,10 @@ from blazar.plugins import floatingips as plugin from blazar.plugins.floatingips import floatingip_plugin from blazar import tests -from blazar.utils.openstack import exceptions as opst_exceptions +from blazar.utils.openstack import exceptions as utils_exceptions from blazar.utils.openstack import neutron - +from blazar.utils.openstack import base +from blazar import status CONF = cfg.CONF @@ -77,11 +79,11 @@ def test_create_floatingip(self): def test_create_floatingip_with_invalid_ip(self): m = mock.MagicMock() - m.fetch_subnet.side_effect = opst_exceptions.NeutronUsesFloatingIP() + m.fetch_subnet.side_effect = utils_exceptions.NeutronUsesFloatingIP() self.fip_pool.return_value = m fip_plugin = floatingip_plugin.FloatingIpPlugin() - self.assertRaises(opst_exceptions.NeutronUsesFloatingIP, + self.assertRaises(utils_exceptions.NeutronUsesFloatingIP, fip_plugin.create_floatingip, {'floating_ip_address': 'invalid-ip', 'floating_network_id': 'id'}) @@ -386,6 +388,7 @@ def test_update_active_reservation_increase_amount_fips_available(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'id': '018c1b43-e69e-4aef-a543-09681539cf4c', + 'project_id': 'fake-project-id', 'start_date': datetime.datetime(2013, 12, 19, 20, 0), 'end_date': datetime.datetime(2013, 12, 19, 21, 0), } @@ -421,7 +424,6 @@ def test_update_active_reservation_increase_amount_fips_available(self): 'subnet_id': 'subnet-id', 'floating_ip_address': '172.2.24.100' } - self.set_context(context.BlazarContext(project_id='fake-project-id')) m = mock.MagicMock() self.fip_pool.return_value = m fip_allocation_create = self.patch( @@ -433,8 +435,7 @@ def test_update_active_reservation_increase_amount_fips_available(self): values) fip_reservation_update.assert_called_once_with( 'fip_resv_id1', {'amount': 2}) - self.fip_pool.assert_called_once_with( - 'f548089e-fb3e-4013-a043-c5ed809c7a67') + self.fip_pool.assert_any_call('f548089e-fb3e-4013-a043-c5ed809c7a67') m.create_reserved_floatingip.assert_called_once_with( 'subnet-id', '172.2.24.100', @@ -460,6 +461,7 @@ def test_update_active_reservation_fip_creation_failure(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'id': '018c1b43-e69e-4aef-a543-09681539cf4c', + 'project_id': 'fake-project-id', 'start_date': datetime.datetime(2013, 12, 19, 20, 0), 'end_date': datetime.datetime(2013, 12, 19, 21, 0), } @@ -500,7 +502,6 @@ def test_update_active_reservation_fip_creation_failure(self): 'floating_ip_address': '172.2.24.101' } ) - self.set_context(context.BlazarContext(project_id='fake-project-id')) m = mock.MagicMock() m.create_reserved_floatingip.side_effect = (None, Exception()) self.fip_pool.return_value = m @@ -749,12 +750,11 @@ def test_on_start(self): 'subnet_id': 'subnet-id', 'floating_ip_address': '172.2.24.100' } - self.set_context(context.BlazarContext(project_id='fake-project-id')) m = mock.MagicMock() self.fip_pool.return_value = m - + fake_lease = {'project_id': 'fake-project-id'} fip_plugin = floatingip_plugin.FloatingIpPlugin() - fip_plugin.on_start('resource-id1') + fip_plugin.on_start('resource-id1', lease=fake_lease) self.fip_pool.assert_called_once_with('network-id1') m.create_reserved_floatingip.assert_called_once_with('subnet-id', @@ -768,10 +768,16 @@ def test_on_end(self): 'reservation_id': 'reservation-id1', 'network_id': 'network-id1' } + reservation_get = self.patch(self.db_api, 'reservation_get') + # only if reservation is active, FloatingIPPool is called in deallocate + reservation_get.return_value = { + 'status': status.reservation.ACTIVE, + } fip_allocation_get_all_by_values = self.patch( self.db_api, 'fip_allocation_get_all_by_values' ) fip_allocation_get_all_by_values.return_value = [{ + 'id': 'alloc-id1', 'floatingip_id': 'fip-id', }] fip_get = self.patch(self.db_api, 'floatingip_get') @@ -783,12 +789,15 @@ def test_on_end(self): self.set_context(context.BlazarContext(project_id='fake-project-id')) m = mock.MagicMock() self.fip_pool.return_value = m + patch_fip_allocation_destroy = self.patch( + db_api, 'fip_allocation_destroy') fip_plugin = floatingip_plugin.FloatingIpPlugin() fip_plugin.on_end('resource-id1') self.fip_pool.assert_called_once_with('network-id1') m.delete_reserved_floatingip.assert_called_once_with('172.2.24.100') + patch_fip_allocation_destroy.assert_called_once_with('alloc-id1') def test_matching_fips_not_allocated_fips(self): def fip_allocation_get_all_by_values(**kwargs): @@ -920,3 +929,185 @@ def test_matching_fips_not_matching(self): 'network-id', [], 2, datetime.datetime(2013, 12, 19, 20, 0), datetime.datetime(2013, 12, 19, 21, 0)) + + +class FloatingIpMonitorPluginTestCase(tests.TestCase): + + def setUp(self): + super(FloatingIpMonitorPluginTestCase, self).setUp() + self.db_api = db_api + self.db_utils = db_utils + self.fip_monitor_plugin = floatingip_plugin.FloatingIpMonitorPlugin() + + def test_poll_fip_with_fip_in_pool(self): + def fake_fetch_subnet(*args, **kwargs): + raise utils_exceptions.NeutronUsesFloatingIP(floatingip="1", subnet="2") + + fips = [ + {'id': '1', + 'floating_network_id': 'network1', + 'floating_ip_address': "10.10.10.1"}, + {'id': '2', + 'floating_network_id': 'network2', + 'floating_ip_address': "10.10.10.2"}, + ] + + fip_list = self.patch(db_api, 'floatingip_list') + fip_list.return_value = fips + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_fip_id') + get_reservations.side_effect = [ + {'id': "1", 'status': status.reservation.ERROR}, + {'id': "2", 'status': status.reservation.ERROR} + ] + neutron_client_patch = self.patch(neutron.neutron_client.Client, 'show_network') + neutron_client_patch.show_network.return_value = {"subnets": "1"} + fetch_subnet = self.patch(neutron.FloatingIPPool, 'fetch_subnet') + fetch_subnet.side_effect = fake_fetch_subnet + result = self.fip_monitor_plugin.poll_resource_failures() + self.assertEqual(result, (fips, [])) + + def test_poll_fip_with_fip_active_reservation(self): + def fake_fetch_subnet(*args, **kwargs): + return None + + fips = [ + {'id': '1', + 'floating_network_id': 'network1', + 'floating_ip_address': "10.10.10.1"}, + ] + + fip_list = self.patch(db_api, 'floatingip_list') + fip_list.return_value = fips + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_fip_id') + get_reservations.side_effect = [ + {'id': "1", 'status': status.reservation.ACTIVE} + ] + result = self.fip_monitor_plugin.poll_resource_failures() + self.assertEqual(result, ([], [])) + + def test_poll_fip_with_fip_not_in_subnet(self): + def fake_fetch_subnet(*args, **kwargs): + raise utils_exceptions.FloatingIPSubnetNotFound(floatingip="1", subnet="2") + + fips = [ + {'id': '1', + 'floating_network_id': 'network1', + 'floating_ip_address': "10.10.10.1"}, + ] + + fip_list = self.patch(db_api, 'floatingip_list') + fip_list.return_value = fips + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_fip_id') + get_reservations.side_effect = [ + {'id': "1", 'status': status.reservation.ERROR}, + ] + neutron_client_patch = self.patch(neutron.neutron_client.Client, 'show_network') + neutron_client_patch.show_network.return_value = {"subnets": "1"} + fetch_subnet = self.patch(neutron.FloatingIPPool, 'fetch_subnet') + fetch_subnet.side_effect = fake_fetch_subnet + result = self.fip_monitor_plugin.poll_resource_failures() + self.assertEqual(result, ([], [])) + + def test_poll_fip_without_reservation_tags(self): + def fake_fetch_subnet(*args, **kwargs): + return + def fake_get_reservation(res_id): + if res_id == "1": + return { + 'id': '1', + 'status': status.reservation.DELETED + } + elif res_id == "2": + return { + 'id': '2', + 'status': status.reservation.ERROR + } + return + def fake_show_fip(fip_addr): + if fip_addr == "1": + return { + 'tags':[ + 'blazar', + f'reservation:{fip_addr}' + ] + } + else: + return {'tags':[]} + + fips = [ + {'id': '1', + 'floating_network_id': 'network1', + 'floating_ip_address': "1"}, + {'id': '2', + 'floating_network_id': 'network2', + 'floating_ip_address': "2"}, + ] + + fip_list = self.patch(db_api, 'floatingip_list') + fip_list.return_value = fips + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_fip_id') + get_reservations.side_effect = fake_get_reservation + neutron_client_patch = self.patch(neutron.neutron_client.Client, 'show_network') + neutron_client_patch.show_network.return_value = {"subnets": "1"} + fetch_subnet = self.patch(neutron.FloatingIPPool, 'fetch_subnet') + fetch_subnet.side_effect = fake_fetch_subnet + show_fip_patch = self.patch(neutron.FloatingIPPool, 'show_floatingip') + show_fip_patch.side_effect = fake_show_fip + fip_reservation_get_patch = self.patch(db_api, 'reservation_get') + fip_reservation_get_patch.side_effect = fake_get_reservation + delete_reserved_patch = self.patch(neutron.FloatingIPPool, 'delete_reserved_floatingip') + + result = self.fip_monitor_plugin.poll_resource_failures() + self.assertTrue(delete_reserved_patch.called) + self.assertEqual(result, ([], [fips[0]])) + + + def test_poll_fip_with_reservation_status_deleted(self): + def fake_fetch_subnet(*args, **kwargs): + return + def fake_get_reservation(res_id): + if res_id == "1": + return { + 'id': '1', + 'status': status.reservation.DELETED + } + elif res_id == "2": + return { + 'id': '2', + 'status': status.reservation.ACTIVE + } + return + def fake_show_fip(fip_addr): + return { + 'tags':[ + 'blazar', + f'reservation:{fip_addr}' + ] + } + + fips = [ + {'id': '1', + 'floating_network_id': 'network1', + 'floating_ip_address': "1"}, + {'id': '2', + 'floating_network_id': 'network2', + 'floating_ip_address': "2"}, + ] + + fip_list = self.patch(db_api, 'floatingip_list') + fip_list.return_value = fips + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_fip_id') + get_reservations.side_effect = fake_get_reservation + neutron_client_patch = self.patch(neutron.neutron_client.Client, 'show_network') + neutron_client_patch.show_network.return_value = {"subnets": "1"} + fetch_subnet = self.patch(neutron.FloatingIPPool, 'fetch_subnet') + fetch_subnet.side_effect = fake_fetch_subnet + show_fip_patch = self.patch(neutron.FloatingIPPool, 'show_floatingip') + show_fip_patch.side_effect = fake_show_fip + fip_reservation_get_patch = self.patch(db_api, 'reservation_get') + fip_reservation_get_patch.side_effect = fake_get_reservation + delete_reserved_patch = self.patch(neutron.FloatingIPPool, 'delete_reserved_floatingip') + + result = self.fip_monitor_plugin.poll_resource_failures() + self.assertTrue(delete_reserved_patch.called) + self.assertEqual(result, ([], [fips[0]])) diff --git a/blazar/tests/plugins/instances/test_instance_plugin.py b/blazar/tests/plugins/instances/test_instance_plugin.py index ed9001bfa..5acf47baa 100644 --- a/blazar/tests/plugins/instances/test_instance_plugin.py +++ b/blazar/tests/plugins/instances/test_instance_plugin.py @@ -30,7 +30,6 @@ from blazar import tests from blazar.utils.openstack import nova from oslo_config import cfg -from oslo_config import fixture as conf_fixture CONF = cfg.CONF @@ -41,20 +40,6 @@ class TestVirtualInstancePlugin(tests.TestCase): def setUp(self): super(TestVirtualInstancePlugin, self).setUp() - def test_configuration(self): - self.cfg = self.useFixture(conf_fixture.Config(CONF)) - self.cfg.config(os_admin_username='fake-user') - self.cfg.config(os_admin_password='fake-passwd') - self.cfg.config(os_admin_user_domain_name='fake-user-domain') - self.cfg.config(os_admin_project_name='fake-pj-name') - self.cfg.config(os_admin_project_domain_name='fake-pj-domain') - plugin = instance_plugin.VirtualInstancePlugin() - self.assertEqual("fake-user", plugin.username) - self.assertEqual("fake-passwd", plugin.password) - self.assertEqual("fake-user-domain", plugin.user_domain_name) - self.assertEqual("fake-pj-name", plugin.project_name) - self.assertEqual("fake-pj-domain", plugin.project_domain_name) - def get_input_values(self, vcpus, memory, disk, amount, affinity, start, end, lease_id, resource_properties): values = {'vcpus': vcpus, 'memory_mb': memory, 'disk_gb': disk, @@ -119,6 +104,8 @@ def test_reserve_resource(self): expected_ret = 'instance-reservation-id1' + ctx = context.BlazarContext(project_id='fake-project') + self.set_context(ctx) ret = plugin.reserve_resource('res_id1', inputs) self.assertEqual(expected_ret, ret) @@ -135,7 +122,7 @@ def test_reserve_resource(self): mock_alloc_create.assert_any_call({'compute_host_id': 'host2', 'reservation_id': 'res_id1'}) mock_create_resources.assert_called_once_with( - fake_instance_reservation) + ctx, fake_instance_reservation) mock_inst_update.assert_called_once_with('instance-reservation-id1', {'flavor_id': 1, 'server_group_id': 2, @@ -147,6 +134,7 @@ def test_affinity_error(self, value): inputs = self.get_input_values(2, 4018, 10, 1, value, '2030-01-01 08:00', '2030-01-01 08:00', 'lease-1', '') + self.set_context(context.BlazarContext(project_id='fake-project')) self.assertRaises(mgr_exceptions.MalformedParameter, plugin.reserve_resource, 'reservation_id', inputs) self.assertRaises(mgr_exceptions.MalformedParameter, @@ -158,6 +146,7 @@ def test_error_with_amount(self, value): inputs = self.get_input_values(2, 4018, 10, value, False, '2030-01-01 08:00', '2030-01-01 08:00', 'lease-1', '') + self.set_context(context.BlazarContext(project_id='fake-project')) self.assertRaises(mgr_exceptions.MalformedParameter, plugin.reserve_resource, 'reservation_id', inputs) self.assertRaises(mgr_exceptions.MalformedParameter, @@ -171,6 +160,7 @@ def test_create_reservation_with_missing_param(self, missing_param): '2030-01-01 08:00', '2030-01-01 08:00', 'lease-1', '') del inputs[missing_param] + self.set_context(context.BlazarContext(project_id='fake-project')) self.assertRaises(mgr_exceptions.MissingParameter, plugin.reserve_resource, 'reservation_id', inputs) @@ -410,6 +400,61 @@ def fake_get_reservation_by_host(host_id, start, end): expected_query = ['vcpus >= 2', 'memory_mb >= 2048', 'local_gb >= 100'] mock_host_get_query.assert_called_once_with(expected_query) + def test_pickup_host_with_anti_affinity(self): + def fake_get_reservation_by_host(host_id, start, end): + if host_id in ['host-1', 'host-3']: + return [ + {'id': '1', + 'resource_type': instances.RESOURCE_TYPE}, + {'id': '2', + 'resource_type': instances.RESOURCE_TYPE} + ] + else: + return [] + + plugin = instance_plugin.VirtualInstancePlugin() + + mock_host_allocation_get = self.patch( + db_api, 'host_allocation_get_all_by_values') + mock_host_allocation_get.return_value = [] + + mock_host_get_query = self.patch(db_api, + 'reservable_host_get_all_by_queries') + hosts_list = [self.generate_host_info('host-1', 8, 8192, 1000), + self.generate_host_info('host-2', 2, 2048, 500)] + mock_host_get_query.return_value = hosts_list + + mock_get_reservations = self.patch(db_utils, + 'get_reservations_by_host_id') + + mock_get_reservations.side_effect = fake_get_reservation_by_host + + mock_max_usages = self.patch(plugin, 'max_usages') + mock_max_usages.return_value = (0, 0, 0) + + mock_reservation_get = self.patch(db_api, 'reservation_get') + mock_reservation_get.return_value = { + 'status': 'pending' + } + + params = { + 'vcpus': 2, + 'memory_mb': 2048, + 'disk_gb': 100, + 'amount': 2, + 'affinity': False, + 'resource_properties': '', + 'start_date': datetime.datetime(2030, 1, 1, 8, 00), + 'end_date': datetime.datetime(2030, 1, 1, 12, 00) + } + + expected = {'added': ['host-1', 'host-2'], 'removed': []} + ret = plugin.pickup_hosts('reservation-id1', params) + + self.assertEqual(expected, ret) + expected_query = ['vcpus >= 2', 'memory_mb >= 2048', 'local_gb >= 100'] + mock_host_get_query.assert_called_once_with(expected_query) + @ddt.data('None', 'none', None) def test_pickup_host_with_no_affinity(self, value): def fake_get_reservation_by_host(host_id, start, end): @@ -681,8 +726,9 @@ def test_create_resources(self): fake_client.nova.server_groups.create.return_value = \ fake_server_group - self.set_context(context.BlazarContext(project_id='fake-project', - auth_token='fake-token')) + ctx = context.BlazarContext(project_id='fake-project', + auth_token='fake-token') + self.set_context(ctx) fake_flavor = mock.MagicMock(method='set_keys', flavorid='reservation-id1') mock_nova = mock.MagicMock() @@ -700,7 +746,7 @@ def test_create_resources(self): expected = (fake_flavor, fake_server_group, fake_agg) - ret = plugin._create_resources(instance_reservation) + ret = plugin._create_resources(ctx, instance_reservation) self.assertEqual(expected, ret) @@ -718,7 +764,8 @@ def test_create_resources(self): name='reservation-id1', metadata={'reservation': 'reservation-id1', 'filter_tenant_id': 'fake-project', - 'affinity_id': 'server_group_id1'}) + 'affinity_id': 'server_group_id1'}, + project_id='fake-project') mock_create_reservation_class.assert_called_once_with( 'reservation-id1') @@ -1108,7 +1155,7 @@ def fake_host_get(host_id): mock_host_get = self.patch(db_api, 'host_get') mock_host_get.side_effect = fake_host_get - plugin.on_start('resource-id1') + plugin.on_start('resource-id1', lease={"project_id": "fake-project"}) mock_nova.flavor_access.add_tenant_access.assert_called_once_with( 'reservation-id1', 'fake-project') @@ -1167,7 +1214,7 @@ def test_on_end(self): 404, "The server doesn't exist in Nova"), Exception('Unknown'), None, None, None] - plugin.on_end('resource-id1') + plugin.on_end('resource-id1', lease={"project_id": "fake-project-id"}) mock_nova.flavor_access.remove_tenant_access.assert_called_once_with( 'reservation-id1', 'fake-project-id') @@ -1659,3 +1706,33 @@ def test_reallocate_missing_resources_with_affinity(self): destroy_calls = [mock.call('alloc-1'), mock.call('alloc-2')] alloc_destroy.assert_has_calls(destroy_calls) self.assertEqual(False, result) + + @ddt.data(False, True, None) + def test_cleanup_resources(self, affinity): + instance_reservation = { + 'reservation_id': 'reservation-id1', + 'vcpus': 2, + 'memory_mb': 1024, + 'disk_gb': 20, + 'affinity': affinity + } + + # Set server_group_id according to the affinity value + server_group_id = 'group-1' if affinity is not None else None + instance_reservation['server_group_id'] = server_group_id + + mock_nova_client = self.patch(nova, 'NovaClientWrapper') + mock_nova_client.return_value = mock.MagicMock() + mock_nova_pool = self.patch(nova, 'ReservationPool') + mock_nova_pool.return_value = mock.MagicMock() + plugin = instance_plugin.VirtualInstancePlugin() + mock_nova = mock.MagicMock() + type(plugin).nova = mock_nova + + plugin.cleanup_resources(instance_reservation) + + if affinity is not None: + mock_nova.nova.server_groups.delete.assert_called_once_with( + 'group-1') + mock_nova.nova.flavors.delete.assert_called_once_with( + 'reservation-id1') diff --git a/blazar/tests/plugins/networks/__init__.py b/blazar/tests/plugins/networks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/blazar/tests/plugins/networks/test_network_plugin.py b/blazar/tests/plugins/networks/test_network_plugin.py new file mode 100644 index 000000000..fc3c2f9a5 --- /dev/null +++ b/blazar/tests/plugins/networks/test_network_plugin.py @@ -0,0 +1,1375 @@ +# Copyright (c) 2018 StackHPC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import uuid + +from oslo_config import cfg +from unittest import mock + +from blazar import context +from blazar.db import api as db_api +from blazar.db import exceptions as db_exceptions +from blazar.db import utils as db_utils +from blazar.manager import exceptions as manager_exceptions +from blazar.manager import service +from blazar.plugins import networks as plugin +from blazar.plugins.networks import network_plugin +from blazar import status +from blazar import tests +from blazar.utils.openstack import base +from blazar.utils.openstack import ironic +from blazar.utils.openstack import neutron +from blazar.utils.openstack import nova +from blazar.utils import trusts + +CONF = cfg.CONF + + +class NetworkPluginTestCase(tests.TestCase): + + def setUp(self): + super(NetworkPluginTestCase, self).setUp() + self.cfg = cfg + self.context = context + self.patch(self.context, 'BlazarContext') + + self.ironic_client = ( + self.patch(ironic, 'BlazarIronicClient').return_value) + self.neutron_client = ( + self.patch(neutron, 'BlazarNeutronClient').return_value) + + self.service = service + self.manager = self.service.ManagerService() + + self.fake_network_id = 'e3ed59f3-27e6-48df-b8bd-2a397aeb57dc' + self.fake_network_values = { + 'network_type': 'vlan', + 'physical_network': 'physnet1', + 'segment_id': 1234 + } + self.fake_network = self.fake_network_values.copy() + self.fake_network['id'] = self.fake_network_id + + self.patch(base, 'url_for').return_value = 'http://foo.bar' + self.network_plugin = network_plugin + self.fake_network_plugin = self.network_plugin.NetworkPlugin() + self.db_api = db_api + self.db_utils = db_utils + + self.db_network_get = self.patch(self.db_api, 'network_get') + self.db_network_get.return_value = self.fake_network + self.db_network_list = self.patch(self.db_api, 'network_list') + self.db_network_create = self.patch(self.db_api, 'network_create') + self.db_network_update = self.patch(self.db_api, 'network_update') + self.db_network_destroy = self.patch(self.db_api, 'network_destroy') + + self.db_network_extra_capability_get_all_per_network = self.patch( + self.db_api, 'network_extra_capability_get_all_per_network') + + self.db_network_extra_capability_get_all_per_name = self.patch( + self.db_api, 'network_extra_capability_get_all_per_name') + + self.db_network_extra_capability_create = self.patch( + self.db_api, 'network_extra_capability_create') + + self.db_network_extra_capability_update = self.patch( + self.db_api, 'network_extra_capability_update') + + self.get_extra_capabilities = self.patch( + self.fake_network_plugin, '_get_extra_capabilities') + + self.get_extra_capabilities.return_value = { + 'foo': 'bar', + 'buzz': 'word', + } + self.fake_network_plugin.setup(None) + + self.trusts = trusts + self.trust_ctx = self.patch(self.trusts, 'create_ctx_from_trust') + self.trust_create = self.patch(self.trusts, 'create_trust') + + self.ServerManager = nova.ServerManager + + def test_get_network(self): + network = self.fake_network_plugin.get_network(self.fake_network_id) + self.db_network_get.assert_called_once_with(self.fake_network_id) + expected = self.fake_network.copy() + expected.update({'foo': 'bar', 'buzz': 'word'}) + self.assertEqual(expected, network) + + def test_get_network_without_extracapabilities(self): + self.get_extra_capabilities.return_value = {} + network = self.fake_network_plugin.get_network(self.fake_network_id) + self.db_network_get.assert_called_once_with(self.fake_network_id) + self.assertEqual(self.fake_network, network) + + def test_list_networks(self): + self.fake_network_plugin.list_networks() + self.db_network_list.assert_called_once_with() + + def test_create_network_without_extra_capabilities(self): + network_values = { + 'network_type': 'vlan', + 'physical_network': 'physnet1', + 'segment_id': 1234 + } + expected_network_values = network_values.copy() + self.get_extra_capabilities.return_value = {} + network = self.fake_network_plugin.create_network(network_values) + self.db_network_create.assert_called_once_with(expected_network_values) + self.assertEqual(network, self.fake_network) + + def test_create_network_with_extra_capabilities(self): + fake_network = self.fake_network.copy() + fake_network.update({'foo': 'bar'}) + # NOTE(sbauza): 'id' will be pop'd, we need to keep track of it + fake_request = fake_network.copy() + fake_capa = {'network_id': self.fake_network_id, + 'capability_name': 'foo', + 'capability_value': 'bar', + } + self.get_extra_capabilities.return_value = {'foo': 'bar'} + self.db_network_create.return_value = self.fake_network + network = self.fake_network_plugin.create_network(fake_request) + self.db_network_create.assert_called_once_with( + self.fake_network_values) + self.db_network_extra_capability_create.assert_called_once_with( + fake_capa) + self.assertEqual(network, fake_network) + + def test_create_network_with_capabilities_too_long(self): + fake_network = self.fake_network_values.copy() + fake_network.update({'foo': 'bar'}) + # NOTE(sbauza): 'id' will be pop'd, we need to keep track of it + fake_request = fake_network.copy() + long_key = "" + for i in range(65): + long_key += "0" + fake_request[long_key] = "foo" + self.db_network_create.return_value = self.fake_network + self.assertRaises(manager_exceptions.ExtraCapabilityTooLong, + self.fake_network_plugin.create_network, + fake_request) + + def test_create_network_without_required_params(self): + self.assertRaises(manager_exceptions.MissingParameter, + self.fake_network_plugin.create_network, + {'network_type': 'vlan', + 'physical_network': 'physnet1'}) + + def test_create_network_with_invalid_segment_id(self): + self.assertRaises(manager_exceptions.MalformedParameter, + self.fake_network_plugin.create_network, + {'network_type': 'vlan', + 'physical_network': 'physnet1', + 'segment_id': 0}) + self.assertRaises(manager_exceptions.MalformedParameter, + self.fake_network_plugin.create_network, + {'network_type': 'vlan', + 'physical_network': 'physnet1', + 'segment_id': 4095}) + + def test_create_network_issuing_rollback(self): + def fake_db_network_create(*args, **kwargs): + raise db_exceptions.BlazarDBException + self.db_network_create.side_effect = fake_db_network_create + self.assertRaises(db_exceptions.BlazarDBException, + self.fake_network_plugin.create_network, + self.fake_network) + + def test_create_duplicate_network(self): + def fake_db_network_create(*args, **kwargs): + raise db_exceptions.BlazarDBDuplicateEntry + self.db_network_create.side_effect = fake_db_network_create + self.assertRaises(db_exceptions.BlazarDBDuplicateEntry, + self.fake_network_plugin.create_network, + self.fake_network) + + def test_create_network_having_issue_when_storing_extra_capability(self): + def fake_db_network_extra_capability_create(*args, **kwargs): + raise db_exceptions.BlazarDBException + fake_network = self.fake_network_values.copy() + fake_network.update({'foo': 'bar'}) + fake_request = fake_network.copy() + self.get_extra_capabilities.return_value = {'foo': 'bar'} + self.db_network_create.return_value = self.fake_network + fake = self.db_network_extra_capability_create + fake.side_effect = fake_db_network_extra_capability_create + self.assertRaises(manager_exceptions.CantAddExtraCapability, + self.fake_network_plugin.create_network, + fake_request) + + def test_update_network(self): + network_values = {'segment_id': 2345} + self.fake_network_plugin.update_network(self.fake_network_id, + network_values) + self.db_network_update.assert_called_once_with( + self.fake_network_id, network_values) + + def test_update_network_extra_capabilities(self): + network_values = {'foo': 'baz'} + + self.db_network_extra_capability_get_all_per_name.return_value = [ + ({'id': 'extra_id1', + 'network_id': self.fake_network_id, + 'capability_value': 'bar'}, + 'foo') + ] + + self.get_reservations_by_network = self.patch( + self.db_utils, 'get_reservations_by_network_id') + self.get_reservations_by_network.return_value = [] + + self.fake_network_plugin.update_network(self.fake_network_id, + network_values) + self.db_network_extra_capability_update.assert_called_once_with( + 'extra_id1', {'capability_name': 'foo', 'capability_value': 'baz'}) + + def test_update_network_having_issue_when_storing_extra_capability(self): + def fake_db_network_extra_capability_update(*args, **kwargs): + raise RuntimeError + network_values = {'foo': 'baz'} + self.get_reservations_by_network = self.patch( + self.db_utils, 'get_reservations_by_network_id') + self.get_reservations_by_network.return_value = [] + self.db_network_extra_capability_get_all_per_name.return_value = [ + ({'id': 'extra_id1', + 'network_id': self.fake_network_id, + 'capability_value': 'bar'}, + 'foo') + ] + fake = self.db_network_extra_capability_update + fake.side_effect = fake_db_network_extra_capability_update + self.assertRaises(manager_exceptions.CantAddExtraCapability, + self.fake_network_plugin.update_network, + self.fake_network_id, network_values) + + def test_update_network_with_new_extra_capability(self): + network_values = {'qux': 'word'} + + self.db_network_extra_capability_get_all_per_network.return_value = [] + self.fake_network_plugin.update_network(self.fake_network_id, + network_values) + self.db_network_extra_capability_create.assert_called_once_with({ + 'network_id': self.fake_network_id, + 'capability_name': 'qux', + 'capability_value': 'word' + }) + + def test_update_network_with_used_capability(self): + network_values = {'foo': 'buzz'} + + self.db_network_extra_capability_get_all_per_name.return_value = [ + ({'id': 'extra_id1', + 'network_id': self.fake_network_id, + 'capability_value': 'bar'}, + 'foo') + ] + fake_network_reservation = { + 'resource_type': plugin.RESOURCE_TYPE, + 'resource_id': 'resource-1', + } + + fake_get_reservations = self.patch(self.db_utils, + 'get_reservations_by_network_id') + fake_get_reservations.return_value = [fake_network_reservation] + + fake_get_plugin_reservation = self.patch(self.db_utils, + 'get_plugin_reservation') + fake_get_plugin_reservation.return_value = { + 'resource_properties': '["==", "$foo", "bar"]' + } + self.assertRaises(manager_exceptions.CantAddExtraCapability, + self.fake_network_plugin.update_network, + self.fake_network_id, network_values) + fake_get_plugin_reservation.assert_called_once_with( + plugin.RESOURCE_TYPE, 'resource-1') + + def test_delete_network(self): + network_allocation_get_all = self.patch( + self.db_api, + 'network_allocation_get_all_by_values') + network_allocation_get_all.return_value = [] + self.fake_network_plugin.delete_network(self.fake_network_id) + + self.db_network_destroy.assert_called_once_with(self.fake_network_id) + + def test_delete_reserved_network(self): + network_allocation_get_all = self.patch( + self.db_api, + 'network_allocation_get_all_by_values') + network_allocation_get_all.return_value = [ + { + 'id': u'dd305477-4df8-4547-87f6-69069ee546a6', + 'network_id': self.fake_network_id + } + ] + + self.assertRaises(manager_exceptions.CantDeleteNetwork, + self.fake_network_plugin.delete_network, + self.fake_network_id) + + def test_delete_network_not_existing_in_db(self): + self.db_network_get.return_value = None + self.assertRaises(manager_exceptions.NetworkNotFound, + self.fake_network_plugin.delete_network, + self.fake_network_id) + + def test_delete_network_issuing_rollback(self): + def fake_db_network_destroy(*args, **kwargs): + raise db_exceptions.BlazarDBException + network_allocation_get_all = self.patch( + self.db_api, + 'network_allocation_get_all_by_values') + network_allocation_get_all.return_value = [] + self.db_network_destroy.side_effect = fake_db_network_destroy + self.assertRaises(manager_exceptions.CantDeleteNetwork, + self.fake_network_plugin.delete_network, + self.fake_network_id) + + def generate_event(self, id, lease_id, event_type, time, status='UNDONE'): + return { + 'id': id, + 'lease_id': lease_id, + 'event_type': event_type, + 'time': time, + 'status': status + } + + def get_uuid(self): + return str(uuid.uuid4()) + + def generate_basic_events(self, lease_id, start, before_end, end): + return [ + self.generate_event(self.get_uuid(), lease_id, 'start_lease', + datetime.datetime.strptime(start, + '%Y-%m-%d %H:%M')), + self.generate_event(self.get_uuid(), lease_id, 'before_end_lease', + datetime.datetime.strptime(before_end, + '%Y-%m-%d %H:%M')), + self.generate_event(self.get_uuid(), lease_id, 'end_lease', + datetime.datetime.strptime(end, + '%Y-%m-%d %H:%M')), + ] + + def test_create_reservation_no_network_available(self): + now = datetime.datetime.utcnow() + lease = { + 'id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + 'user_id': '123', + 'project_id': '456', + } + values = { + 'lease_id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + 'start_date': now, + 'end_date': now + datetime.timedelta(hours=1), + 'resource_type': plugin.RESOURCE_TYPE, + 'network_name': 'foo-net', + 'network_properties': '', + 'resource_properties': '', + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = lease + network_reservation_create = self.patch(self.db_api, + 'network_reservation_create') + matching_networks = self.patch(self.fake_network_plugin, + '_matching_networks') + matching_networks.return_value = [] + self.assertRaises(manager_exceptions.NotEnoughNetworksAvailable, + self.fake_network_plugin.reserve_resource, + u'f9894fcf-e2ed-41e9-8a4c-92fac332608e', + values) + network_reservation_create.assert_not_called() + + def test_create_reservation_networks_available(self): + lease = { + 'id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + 'user_id': '123', + 'project_id': '456', + } + values = { + 'lease_id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + 'network_properties': '', + 'resource_properties': '', + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'resource_type': plugin.RESOURCE_TYPE, + 'network_name': 'foo-net', + 'network_description': '' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = lease + network_reservation_create = self.patch(self.db_api, + 'network_reservation_create') + matching_networks = self.patch(self.fake_network_plugin, + '_matching_networks') + matching_networks.return_value = ['network1', 'network2'] + network_allocation_create = self.patch( + self.db_api, + 'network_allocation_create') + self.fake_network_plugin.reserve_resource( + u'441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + network_values = { + 'reservation_id': u'441c1476-9f8f-4700-9f30-cd9b6fef3509', + 'network_properties': '', + 'resource_properties': '', + 'status': 'pending', + 'before_end': 'default', + 'network_name': 'foo-net', + 'network_description': '', + } + network_reservation_create.assert_called_once_with(network_values) + calls = [ + mock.call( + {'network_id': 'network1', + 'reservation_id': u'441c1476-9f8f-4700-9f30-cd9b6fef3509', + }), + ] + network_allocation_create.assert_has_calls(calls) + + def test_create_reservation_with_missing_param_properties(self): + values = { + 'lease_id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + 'start_date': datetime.datetime(2017, 3, 1, 20, 00), + 'end_date': datetime.datetime(2017, 3, 2, 20, 00), + 'resource_type': plugin.RESOURCE_TYPE, + } + self.assertRaises( + manager_exceptions.MissingParameter, + self.fake_network_plugin.reserve_resource, + u'441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + + def test_create_reservation_with_invalid_param_before_end(self): + values = { + 'lease_id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + 'before_end': 'invalid', + 'start_date': datetime.datetime(2017, 3, 1, 20, 00), + 'end_date': datetime.datetime(2017, 3, 2, 20, 00), + 'resource_type': plugin.RESOURCE_TYPE, + 'network_name': 'foo-net', + 'network_properties': '', + 'resource_properties': '', + } + self.assertRaises( + manager_exceptions.MalformedParameter, + self.fake_network_plugin.reserve_resource, + u'441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + + def test_update_reservation_shorten(self): + values = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 30), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': u'10870923-6d56-45c9-b592-f788053f5baa', + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + } + + network_reservation_get = self.patch( + self.db_api, 'network_reservation_get') + self.patch(self.db_api, 'network_allocation_get_all_by_values') + + self.fake_network_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + network_reservation_get.assert_not_called() + + def test_update_reservation_extend(self): + values = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 30) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': u'10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': u'91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + } + network_reservation_get = self.patch( + self.db_api, 'network_reservation_get') + network_reservation_get.return_value = { + 'network_properties': '["=", "$network_type", "vlan"]', + 'resource_properties': '' + } + network_allocation_get_all = self.patch( + self.db_api, + 'network_allocation_get_all_by_values') + network_allocation_get_all.return_value = [ + { + 'id': u'dd305477-4df8-4547-87f6-69069ee546a6', + 'network_id': 'network1' + } + ] + network_get_all_by_queries = self.patch( + self.db_api, 'network_get_all_by_queries') + network_get_all_by_queries.return_value = [{'id': 'network1'}] + get_reserved_periods = self.patch(self.db_utils, + 'get_reserved_periods') + get_reserved_periods.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)) + ] + network_allocation_create = self.patch( + self.db_api, + 'network_allocation_create') + network_allocation_destroy = self.patch( + self.db_api, + 'network_allocation_destroy') + + self.fake_network_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + network_allocation_create.assert_not_called() + network_allocation_destroy.assert_not_called() + + def test_update_reservation_move_failure(self): + values = { + 'start_date': datetime.datetime(2013, 12, 20, 20, 00), + 'end_date': datetime.datetime(2013, 12, 20, 21, 30) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': u'10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': u'91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'active' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + } + network_reservation_get = self.patch( + self.db_api, + 'network_reservation_get') + network_reservation_get.return_value = { + 'network_properties': '["=", "$network_type", "vlan"]', + 'resource_properties': '' + } + network_allocation_get_all = self.patch( + self.db_api, + 'network_allocation_get_all_by_values') + network_allocation_get_all.return_value = [ + { + 'id': u'dd305477-4df8-4547-87f6-69069ee546a6', + 'network_id': 'network1' + } + ] + network_get_all_by_queries = self.patch(self.db_api, + 'network_get_all_by_queries') + network_get_all_by_queries.return_value = [{'id': 'network1'}] + get_reserved_periods = self.patch(self.db_utils, + 'get_reserved_periods') + get_reserved_periods.return_value = [ + (datetime.datetime(2013, 12, 20, 20, 30), + datetime.datetime(2013, 12, 20, 21, 00)) + ] + matching_networks = self.patch( + self.fake_network_plugin, '_matching_networks') + matching_networks.return_value = [] + self.assertRaises( + manager_exceptions.NotEnoughNetworksAvailable, + self.fake_network_plugin.update_reservation, + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + reservation_get.assert_called() + + def test_update_reservation_move_overlap(self): + values = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 30), + 'end_date': datetime.datetime(2013, 12, 19, 21, 30) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': u'10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': u'91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + } + network_reservation_get = self.patch( + self.db_api, + 'network_reservation_get') + network_reservation_get.return_value = { + 'network_properties': '["=", "$network_type", "vlan"]', + 'resource_properties': '' + } + network_allocation_get_all = self.patch( + self.db_api, + 'network_allocation_get_all_by_values') + network_allocation_get_all.return_value = [ + { + 'id': u'dd305477-4df8-4547-87f6-69069ee546a6', + 'network_id': 'network1' + } + ] + network_get_all_by_queries = self.patch(self.db_api, + 'network_get_all_by_queries') + network_get_all_by_queries.return_value = [{'id': 'network1'}] + get_reserved_periods = self.patch(self.db_utils, + 'get_reserved_periods') + get_reserved_periods.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 30), + datetime.datetime(2013, 12, 19, 21, 00)) + ] + network_allocation_create = self.patch( + self.db_api, + 'network_allocation_create') + network_allocation_destroy = self.patch( + self.db_api, + 'network_allocation_destroy') + + self.fake_network_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + network_allocation_create.assert_not_called() + network_allocation_destroy.assert_not_called() + + def test_update_reservation_move_realloc(self): + values = { + 'start_date': datetime.datetime(2013, 12, 20, 20, 00), + 'end_date': datetime.datetime(2013, 12, 20, 21, 30) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': u'10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': u'91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + } + network_reservation_get = self.patch( + self.db_api, + 'network_reservation_get') + network_reservation_get.return_value = { + 'network_properties': '["=", "$network_type", "vlan"]', + 'resource_properties': '' + } + network_allocation_get_all = self.patch( + self.db_api, + 'network_allocation_get_all_by_values') + network_allocation_get_all.return_value = [ + { + 'id': u'dd305477-4df8-4547-87f6-69069ee546a6', + 'network_id': 'network1' + } + ] + network_get_all_by_queries = self.patch(self.db_api, + 'network_get_all_by_queries') + network_get_all_by_queries.return_value = [{'id': 'network1'}, + {'id': 'network2'}] + network_allocation_create = self.patch( + self.db_api, + 'network_allocation_create') + network_allocation_destroy = self.patch( + self.db_api, + 'network_allocation_destroy') + get_reserved_periods = self.patch(self.db_utils, + 'get_reserved_periods') + get_reserved_periods.return_value = [ + (datetime.datetime(2013, 12, 20, 20, 30), + datetime.datetime(2013, 12, 20, 21, 00)) + ] + matching_networks = self.patch( + self.fake_network_plugin, '_matching_networks') + matching_networks.return_value = ['network2'] + self.fake_network_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + network_reservation_get.assert_called_with( + u'91253650-cc34-4c4f-bbe8-c943aa7d0c9b') + network_allocation_destroy.assert_called_with( + 'dd305477-4df8-4547-87f6-69069ee546a6') + network_allocation_create.assert_called_with( + { + 'network_id': 'network2', + 'reservation_id': '706eb3bc-07ed-4383-be93-b32845ece672' + } + ) + + def test_update_reservation_realloc_with_properties_change(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'network_properties': '["=", "$network_type", "vlan"]', + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00) + } + network_reservation_get = self.patch( + self.db_api, 'network_reservation_get') + network_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'network_properties': '["=", "$network_type", "vlan"]', + 'resource_properties': '' + } + network_allocation_get_all = self.patch( + self.db_api, 'network_allocation_get_all_by_values') + network_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'network_id': 'network1' + } + ] + network_get_all_by_queries = self.patch(self.db_api, + 'network_get_all_by_queries') + network_get_all_by_queries.return_value = [{'id': 'network2'}] + matching_networks = self.patch( + self.fake_network_plugin, '_matching_networks') + matching_networks.return_value = ['network2'] + network_allocation_create = self.patch(self.db_api, + 'network_allocation_create') + network_allocation_destroy = self.patch(self.db_api, + 'network_allocation_destroy') + network_reservation_update = self.patch(self.db_api, + 'network_reservation_update') + + self.fake_network_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + network_reservation_get.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b') + matching_networks.assert_called_with( + '["=", "$network_type", "vlan"]', + '', + datetime.datetime(2017, 7, 12, 20, 00), + datetime.datetime(2017, 7, 12, 21, 00) + ) + network_allocation_create.assert_called_with( + { + 'network_id': 'network2', + 'reservation_id': '706eb3bc-07ed-4383-be93-b32845ece672' + } + ) + network_allocation_destroy.assert_called_with( + 'dd305477-4df8-4547-87f6-69069ee546a6' + ) + network_reservation_update.assert_called_with( + '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + {'network_properties': '["=", "$network_type", "vlan"]'} + ) + + def test_update_reservation_no_requested_networks_available(self): + values = { + 'start_date': datetime.datetime(2017, 7, 12, 20, 00), + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'resource_properties': '["=", "$segment_id", "2345"]' + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + } + network_reservation_get = self.patch( + self.db_api, 'network_reservation_get') + network_reservation_get.return_value = { + 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'network_properties': '["=", "$network_type", "vlan"]', + 'resource_properties': '' + } + network_allocation_get_all = self.patch( + self.db_api, 'network_allocation_get_all_by_values') + network_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'network_id': 'network1' + } + ] + network_get_all_by_queries = self.patch(self.db_api, + 'network_get_all_by_queries') + network_get_all_by_queries.return_value = [] + matching_networks = self.patch( + self.fake_network_plugin, '_matching_networks') + matching_networks.return_value = [] + + self.assertRaises( + manager_exceptions.NotEnoughNetworksAvailable, + self.fake_network_plugin.update_reservation, + '441c1476-9f8f-4700-9f30-cd9b6fef3509', + values) + + def test_on_start(self): + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + 'project_id': '456' + } + reservation_get = self.patch( + self.db_api, 'reservation_get') + reservation_get.return_value = { + 'id': u'593e7028-c0d1-4d76-8642-2ffd890b324c', + 'lease_id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + } + network_reservation_get = self.patch( + self.db_api, 'network_reservation_get') + network_reservation_get.return_value = { + 'id': '04de74e8-193a-49d2-9ab8-cba7b49e45e8', + 'network_id': None, + 'network_name': 'foo-net', + 'network_description': None, + 'reservation_id': u'593e7028-c0d1-4d76-8642-2ffd890b324c' + } + network_allocation_get_all_by_values = self.patch( + self.db_api, 'network_allocation_get_all_by_values') + network_allocation_get_all_by_values.return_value = [ + {'network_id': 'network1'}, + ] + network_get = self.patch(self.db_api, 'network_get') + network_get.return_value = { + 'network_id': 'network1', + 'network_type': 'vlan', + 'physical_network': 'physnet1', + 'segment_id': 1234 + } + create_network = self.patch(self.neutron_client, 'create_network') + create_network.return_value = { + 'network': { + 'id': '69cab064-0e60-4efb-a503-b42dde0fb3f2', + 'name': 'foo-net' + } + } + network_reservation_update = self.patch( + self.db_api, + 'network_reservation_update') + + self.fake_network_plugin.on_start( + u'04de74e8-193a-49d2-9ab8-cba7b49e45e8') + create_network.assert_called_with( + body={ + 'network': { + 'provider:segmentation_id': 1234, + 'name': 'foo-net', + 'project_id': '456', + 'provider:physical_network': 'physnet1', + 'provider:network_type': 'vlan'}}) + network_reservation_update.assert_called_with( + '04de74e8-193a-49d2-9ab8-cba7b49e45e8', + {'network_id': '69cab064-0e60-4efb-a503-b42dde0fb3f2'}) + + def test_on_start_failure(self): + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + 'project_id': '456' + } + reservation_get = self.patch( + self.db_api, 'reservation_get') + reservation_get.return_value = { + 'id': u'593e7028-c0d1-4d76-8642-2ffd890b324c', + 'lease_id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + } + network_reservation_get = self.patch( + self.db_api, 'network_reservation_get') + network_reservation_get.return_value = { + 'id': '04de74e8-193a-49d2-9ab8-cba7b49e45e8', + 'network_id': None, + 'network_name': 'foo-net', + 'network_description': None, + 'reservation_id': u'593e7028-c0d1-4d76-8642-2ffd890b324c' + } + network_allocation_get_all_by_values = self.patch( + self.db_api, 'network_allocation_get_all_by_values') + network_allocation_get_all_by_values.return_value = [ + {'network_id': 'network1'}, + ] + network_get = self.patch(self.db_api, 'network_get') + network_get.return_value = { + 'network_id': 'network1', + 'network_type': 'vlan', + 'physical_network': 'physnet1', + 'segment_id': 1234 + } + + def fake_create_network(*args, **kwargs): + raise manager_exceptions.NetworkCreationFailed + create_network = self.patch(self.neutron_client, 'create_network') + create_network.side_effect = fake_create_network + + self.assertRaises(manager_exceptions.NetworkCreationFailed, + self.fake_network_plugin.on_start, + '04de74e8-193a-49d2-9ab8-cba7b49e45e8') + + def test_on_end(self): + network_reservation_get = self.patch( + self.db_api, 'network_reservation_get') + network_reservation_get.return_value = { + 'id': '04de74e8-193a-49d2-9ab8-cba7b49e45e8', + 'network_id': '69cab064-0e60-4efb-a503-b42dde0fb3f2', + 'network_name': 'foo-net', + 'reservation_id': u'593e7028-c0d1-4d76-8642-2ffd890b324c' + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'id': u'593e7028-c0d1-4d76-8642-2ffd890b324c', + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'status': 'active' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + 'trust_id': 'exxee111qwwwwe' + } + network_reservation_update = self.patch( + self.db_api, + 'network_reservation_update') + network_allocation_get_all_by_values = self.patch( + self.db_api, + 'network_allocation_get_all_by_values') + network_allocation_get_all_by_values.return_value = [ + {'id': u'bfa9aa0b-8042-43eb-a4e6-4555838bf64f', + 'network_id': u'cdae2a65-236f-475a-977d-f6ad82f828b7', + }, + ] + network_allocation_destroy = self.patch( + self.db_api, + 'network_allocation_destroy') + delete_network = self.patch(self.neutron_client, 'delete_network') + delete_network.return_value = None + + self.fake_network_plugin.on_end( + u'04de74e8-193a-49d2-9ab8-cba7b49e45e8') + network_reservation_update.assert_called_with( + u'04de74e8-193a-49d2-9ab8-cba7b49e45e8', {'status': 'completed'}) + network_allocation_destroy.assert_called_with( + u'bfa9aa0b-8042-43eb-a4e6-4555838bf64f') + delete_network.assert_called_with( + '69cab064-0e60-4efb-a503-b42dde0fb3f2') + + def test_on_end_failure(self): + network_reservation_get = self.patch( + self.db_api, 'network_reservation_get') + network_reservation_get.return_value = { + 'id': '04de74e8-193a-49d2-9ab8-cba7b49e45e8', + 'network_id': '69cab064-0e60-4efb-a503-b42dde0fb3f2', + 'network_name': 'foo-net', + 'reservation_id': u'593e7028-c0d1-4d76-8642-2ffd890b324c' + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'id': u'593e7028-c0d1-4d76-8642-2ffd890b324c', + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'status': 'active' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'id': u'018c1b43-e69e-4aef-a543-09681539cf4c', + 'trust_id': 'exxee111qwwwwe' + } + network_reservation_update = self.patch( + self.db_api, + 'network_reservation_update') + network_allocation_get_all_by_values = self.patch( + self.db_api, + 'network_allocation_get_all_by_values') + network_allocation_get_all_by_values.return_value = [ + {'id': u'bfa9aa0b-8042-43eb-a4e6-4555838bf64f', + 'network_id': u'cdae2a65-236f-475a-977d-f6ad82f828b7', + }, + ] + network_allocation_destroy = self.patch( + self.db_api, + 'network_allocation_destroy') + + def fake_delete_network(*args, **kwargs): + raise manager_exceptions.NetworkDeletionFailed + delete_network = self.patch(self.neutron_client, 'delete_network') + delete_network.side_effect = fake_delete_network + + self.assertRaises(manager_exceptions.NetworkDeletionFailed, + self.fake_network_plugin.on_end, + '04de74e8-193a-49d2-9ab8-cba7b49e45e8') + network_reservation_update.assert_called_with( + u'04de74e8-193a-49d2-9ab8-cba7b49e45e8', {'status': 'completed'}) + network_allocation_destroy.assert_called_with( + u'bfa9aa0b-8042-43eb-a4e6-4555838bf64f') + delete_network.assert_called_with( + '69cab064-0e60-4efb-a503-b42dde0fb3f2') + + def test_list_resource_properties(self): + self.db_list_resource_properties = self.patch( + self.db_api, 'resource_properties_list') + + # Expecting a list of (Reservation, Allocation) + self.db_list_resource_properties.return_value = [ + ('prop1', False, 'aaa', False), + ('prop1', False, 'bbb', False), + ('prop2', False, 'aaa', False), + ('prop2', False, 'aaa', False), + ('prop3', True, 'aaa', False) + ] + + expected = [ + {'property': 'prop1'}, + {'property': 'prop2'} + ] + + ret = self.fake_network_plugin.list_resource_properties(query=None) + + # Sort returned value to use assertListEqual + ret.sort(key=lambda x: x['property']) + + self.assertListEqual(expected, ret) + self.db_list_resource_properties.assert_called_once_with( + 'network') + + def test_list_resource_properties_with_detail(self): + self.db_list_resource_properties = self.patch( + self.db_api, 'resource_properties_list') + + # Expecting a list of (Reservation, Allocation) + self.db_list_resource_properties.return_value = [ + ('prop1', False, 'aaa', False), + ('prop1', False, 'bbb', False), + ('prop2', False, 'ccc', False), + ('prop3', True, 'aaa', False) + ] + + expected = [ + {'property': 'prop1', 'private': False, 'values': ['aaa', 'bbb'], 'is_unique': False}, + {'property': 'prop2', 'private': False, 'values': ['ccc'], 'is_unique': False} + ] + + ret = self.fake_network_plugin.list_resource_properties( + query={'detail': True}) + + # Sort returned value to use assertListEqual + ret.sort(key=lambda x: x['property']) + + self.assertListEqual(expected, ret) + self.db_list_resource_properties.assert_called_once_with( + 'network') + + def test_update_resource_property(self): + resource_property_values = { + 'resource_type': 'network', + 'private': False} + + db_resource_property_update = self.patch( + self.db_api, 'resource_property_update') + + self.fake_network_plugin.update_resource_property( + 'foo', resource_property_values) + db_resource_property_update.assert_called_once_with( + 'network', 'foo', resource_property_values) + + +class NetworkMonitorPluginTestCase(tests.TestCase): + + def setUp(self): + super(NetworkMonitorPluginTestCase, self).setUp() + self.db_api = db_api + self.db_utils = db_utils + self.network_plugin = network_plugin + self.fake_network_monitor_plugin = self.network_plugin.NetworkMonitorPlugin() + self.cfg = cfg + self.cfg.CONF.set_override('enable_polling_monitor_dry_run', 'false', group='network') + + def test_network_stuck_in_errored_lease(self): + networks_from_blazar = [ + { + 'id': 'network1', + 'segment_id': 'segment1' + } + ] + fake_ports = { + 'ports': [{"id":"port1", "fixed_ips": [{"subnet_id": "subnet1"}], + "device_id": "router1", + "device_owner": "network:router_interface"}] + } + fake_neutron_networks = { + 'networks': [{"id": "neutron1", "provider:segmentation_id": "segment1"}] + } + fake_subnets = {'subnets': [{"id": "subnet1"}]} + network_list = self.patch(db_api, 'network_list') + network_list.return_value = networks_from_blazar + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_network_id') + get_reservations.side_effect = [ + {'id': "1", 'status': status.reservation.ERROR}, + ] + neutron_list_networks_patch = self.patch(neutron.neutron_client.Client, 'list_networks') + neutron_list_networks_patch.return_value = fake_neutron_networks + neutron_list_ports_patch = self.patch(neutron.neutron_client.Client, 'list_ports') + neutron_list_ports_patch.return_value = fake_ports + neutron_list_subnets_patch = self.patch(neutron.neutron_client.Client, 'list_subnets') + neutron_list_subnets_patch.return_value = fake_subnets + neutron_show_subnet_patch = self.patch(neutron.neutron_client.Client, 'show_subnet') + neutron_show_subnet_patch.return_value = {'subnet': {'id': 'subnet1', 'cidr': '192.168.1.0/24'}} + neutron_show_router_patch = self.patch(neutron.neutron_client.Client, 'show_router') + neutron_show_router_patch.return_value = {'router': {'id': 'router1', 'routes': [{'gateway': '192.168.1.1'}]}} + neutron_remove_route_patch = self.patch(neutron.neutron_client.Client, 'remove_extra_routes_from_router') + neutron_delete_port_patch = self.patch(neutron.neutron_client.Client, 'delete_port') + neutron_remove_interface_patch = self.patch(neutron.neutron_client.Client, 'remove_interface_router') + neutron_delete_subnet_patch = self.patch(neutron.neutron_client.Client, 'delete_subnet') + neutron_delete_network_patch = self.patch(neutron.neutron_client.Client, 'delete_network') + result = self.fake_network_monitor_plugin.poll_resource_failures() + neutron_remove_interface_patch.assert_called_once_with( + "router1", { + 'subnet_id': "subnet1" + } + ) + neutron_delete_subnet_patch.assert_called_once_with("subnet1") + neutron_delete_network_patch.assert_called_once_with("neutron1") + neutron_remove_route_patch.assert_called_once_with( + 'router1', + {'router': {'routes': [{'gateway': '192.168.1.1'}]}} + ) + self.assertEqual(result, ([], [])) + + def test_network_stuck_in_active_lease(self): + networks_from_blazar = [ + { + 'id': 'network1', + 'segment_id': 'segment1' + } + ] + fake_ports = { + 'ports': [{"id":"port1", "fixed_ips": [{"subnet_id": "subnet1"}], + "device_id": "router1", + "device_owner": "network:router_interface"}] + } + network_list = self.patch(db_api, 'network_list') + network_list.return_value = networks_from_blazar + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_network_id') + get_reservations.side_effect = [ + {'id': "1", 'status': status.reservation.ACTIVE}, + ] + neutron_remove_interface_patch = self.patch(neutron.neutron_client.Client, 'remove_interface_router') + neutron_delete_subnet_patch = self.patch(neutron.neutron_client.Client, 'delete_subnet') + neutron_delete_network_patch = self.patch(neutron.neutron_client.Client, 'delete_network') + result = self.fake_network_monitor_plugin.poll_resource_failures() + self.assertFalse(neutron_remove_interface_patch.called) + self.assertFalse(neutron_delete_subnet_patch.called) + self.assertFalse(neutron_delete_network_patch.called) + self.assertEqual(result, ([], [])) + + def test_network_stuck_in_errored_lease_dry_run(self): + self.cfg.CONF.set_override('enable_polling_monitor_dry_run', 'True', group='network') + networks_from_blazar = [ + { + 'id': 'network1', + 'segment_id': 'segment1' + } + ] + fake_ports = { + 'ports': [{"id":"port1", "fixed_ips": [{"subnet_id": "subnet1"}], + "device_id": "router1", + "device_owner": "network:router_interface"}] + } + network_list = self.patch(db_api, 'network_list') + network_list.return_value = networks_from_blazar + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_network_id') + get_reservations.side_effect = [ + {'id': "1", 'status': status.reservation.ACTIVE}, + ] + neutron_remove_interface_patch = self.patch(neutron.neutron_client.Client, 'remove_interface_router') + neutron_delete_subnet_patch = self.patch(neutron.neutron_client.Client, 'delete_subnet') + neutron_delete_network_patch = self.patch(neutron.neutron_client.Client, 'delete_network') + result = self.fake_network_monitor_plugin.poll_resource_failures() + self.assertFalse(neutron_remove_interface_patch.called) + self.assertFalse(neutron_delete_subnet_patch.called) + self.assertFalse(neutron_delete_network_patch.called) + self.assertEqual(result, ([], [])) + + def test_network_stuck_in_errored_lease_network_not_found(self): + networks_from_blazar = [ + { + 'id': 'network1', + 'segment_id': 'segment1' + } + ] + fake_ports = { + 'ports': [{"id":"port1", "fixed_ips": [{"subnet_id": "subnet1"}], + "device_id": "router1", + "device_owner": "network:router_interface"}] + } + network_list = self.patch(db_api, 'network_list') + network_list.return_value = networks_from_blazar + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_network_id') + get_reservations.side_effect = [ + {'id': "1", 'status': status.reservation.ERROR}, + ] + neutron_list_networks_patch = self.patch(neutron.neutron_client.Client, 'list_networks') + neutron_list_networks_patch.return_value = {'networks': []} + neutron_remove_interface_patch = self.patch(neutron.neutron_client.Client, 'remove_interface_router') + neutron_delete_subnet_patch = self.patch(neutron.neutron_client.Client, 'delete_subnet') + neutron_delete_network_patch = self.patch(neutron.neutron_client.Client, 'delete_network') + result = self.fake_network_monitor_plugin.poll_resource_failures() + self.assertFalse(neutron_remove_interface_patch.called) + self.assertFalse(neutron_delete_subnet_patch.called) + self.assertFalse(neutron_delete_network_patch.called) + self.assertEqual(result, ([], [])) + + def test_network_stuck_in_errored_lease_no_router_ports(self): + networks_from_blazar = [ + { + 'id': 'network1', + 'segment_id': 'segment1' + } + ] + fake_ports = { + 'ports': [{"id":"port1", "fixed_ips": [{"subnet_id": "subnet1"}], + "device_id": "dhcp1", + "device_owner": "network:dhcp"}] + } + fake_neutron_networks = { + 'networks': [{"id": "neutron1", "provider:segmentation_id": "segment1"}] + } + fake_subnets = {'subnets': [{"id": "subnet1"}]} + network_list = self.patch(db_api, 'network_list') + network_list.return_value = networks_from_blazar + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_network_id') + get_reservations.side_effect = [ + {'id': "1", 'status': status.reservation.ERROR}, + ] + neutron_list_networks_patch = self.patch(neutron.neutron_client.Client, 'list_networks') + neutron_list_networks_patch.return_value = fake_neutron_networks + neutron_list_ports_patch = self.patch(neutron.neutron_client.Client, 'list_ports') + neutron_list_ports_patch.return_value = fake_ports + neutron_list_subnets_patch = self.patch(neutron.neutron_client.Client, 'list_subnets') + neutron_list_subnets_patch.return_value = fake_subnets + neutron_remove_interface_patch = self.patch(neutron.neutron_client.Client, 'remove_interface_router') + neutron_delete_subnet_patch = self.patch(neutron.neutron_client.Client, 'delete_subnet') + neutron_delete_network_patch = self.patch(neutron.neutron_client.Client, 'delete_network') + neutron_delete_port_patch = self.patch(neutron.neutron_client.Client, 'delete_port') + result = self.fake_network_monitor_plugin.poll_resource_failures() + self.assertFalse(neutron_remove_interface_patch.called) + neutron_delete_subnet_patch.assert_called_once_with("subnet1") + neutron_delete_network_patch.assert_called_once_with("neutron1") + neutron_delete_port_patch.assert_called() + self.assertEqual(result, ([], [])) + + def test_network_stuck_in_errored_lease_subnet_routes(self): + networks_from_blazar = [ + { + 'id': 'network1', + 'segment_id': 'segment1' + } + ] + fake_ports = { + "ports": [{"id":"port1", "fixed_ips": [{"subnet_id": "subnet1"}], + "device_id": "router1", + "device_owner": "network:router_interface"}], + } + fake_neutron_networks = { + 'networks': [{"id": "neutron1", "provider:segmentation_id": "segment1"}] + } + fake_subnets = {'subnets': [{"id": "subnet1"}]} + network_list = self.patch(db_api, 'network_list') + network_list.return_value = networks_from_blazar + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_network_id') + get_reservations.side_effect = [ + {'id': "1", 'status': status.reservation.ERROR}, + ] + neutron_list_networks_patch = self.patch(neutron.neutron_client.Client, 'list_networks') + neutron_list_networks_patch.return_value = fake_neutron_networks + neutron_list_ports_patch = self.patch(neutron.neutron_client.Client, 'list_ports') + neutron_list_ports_patch.return_value = fake_ports + neutron_list_subnets_patch = self.patch(neutron.neutron_client.Client, 'list_subnets') + neutron_list_subnets_patch.return_value = fake_subnets + neutron_show_subnet_patch = self.patch(neutron.neutron_client.Client, 'show_subnet') + neutron_show_subnet_patch.return_value = {'subnet': {'id': 'subnet1', 'cidr': '192.168.1.0/24'}} + neutron_show_router_patch = self.patch(neutron.neutron_client.Client, 'show_router') + neutron_show_router_patch.return_value = {'router': {'id': 'router1', 'routes': [{'gateway': '192.168.1.1'}]}} + neutron_remove_route_patch = self.patch(neutron.neutron_client.Client, 'remove_extra_routes_from_router') + neutron_remove_interface_patch = self.patch(neutron.neutron_client.Client, 'remove_interface_router') + neutron_delete_subnet_patch = self.patch(neutron.neutron_client.Client, 'delete_subnet') + neutron_delete_network_patch = self.patch(neutron.neutron_client.Client, 'delete_network') + neutron_delete_port_patch = self.patch(neutron.neutron_client.Client, 'delete_port') + result = self.fake_network_monitor_plugin.poll_resource_failures() + neutron_remove_interface_patch.assert_called_once_with( + "router1", { + 'subnet_id': "subnet1" + } + ) + neutron_delete_subnet_patch.assert_called_once_with("subnet1") + neutron_delete_network_patch.assert_called_once_with("neutron1") + neutron_delete_port_patch.assert_called_once_with("port1") + neutron_remove_route_patch.assert_called_once_with( + 'router1', + {'router': {'routes': [{'gateway': '192.168.1.1'}]}} + ) + self.assertEqual(result, ([], [])) + + +class TestRemoveSubnetRouteFromRouter(tests.TestCase): + def test_remove_subnet_route_from_router_same_cidr(self): + neutron_show_subnet_patch = self.patch(neutron.neutron_client.Client, 'show_subnet') + neutron_show_subnet_patch.return_value = {'subnet': {'id': 'subnet1', 'cidr': '192.168.1.0/24'}} + neutron_show_router_patch = self.patch(neutron.neutron_client.Client, 'show_router') + neutron_show_router_patch.return_value = {'router': {'id': 'router1', 'routes': [{'gateway': '192.168.1.1'}]}} + neutron_remove_route_patch = self.patch(neutron.neutron_client.Client, 'remove_extra_routes_from_router') + + router_id = 'router123' + subnet_id = 'subnet456' + dry_run = False + + network_plugin.remove_subnet_route_from_router(router_id, subnet_id, dry_run) + + neutron_show_subnet_patch.assert_called_once_with(subnet_id) + neutron_show_router_patch.assert_called_once_with(router_id) + neutron_remove_route_patch.assert_called_once_with( + router_id, + {'router': {'routes': [{'gateway': '192.168.1.1'}]}} + ) + + def test_remove_subnet_route_from_router_diff_cidr(self): + neutron_show_subnet_patch = self.patch(neutron.neutron_client.Client, 'show_subnet') + neutron_show_subnet_patch.return_value = {'subnet': {'id': 'subnet1', 'cidr': '192.168.1.0/24'}} + neutron_show_router_patch = self.patch(neutron.neutron_client.Client, 'show_router') + neutron_show_router_patch.return_value = {'router': {'id': 'router1', 'routes': [{'gateway': '192.168.2.1'}]}} + neutron_remove_route_patch = self.patch(neutron.neutron_client.Client, 'remove_extra_routes_from_router') + + router_id = 'router123' + subnet_id = 'subnet456' + dry_run = False + + network_plugin.remove_subnet_route_from_router(router_id, subnet_id, dry_run) + + neutron_show_subnet_patch.assert_called_once_with(subnet_id) + neutron_show_router_patch.assert_called_once_with(router_id) + neutron_remove_route_patch.assert_not_called() diff --git a/blazar/tests/plugins/oshosts/test_physical_host_plugin.py b/blazar/tests/plugins/oshosts/test_physical_host_plugin.py index 79e131ca3..41bafdb57 100644 --- a/blazar/tests/plugins/oshosts/test_physical_host_plugin.py +++ b/blazar/tests/plugins/oshosts/test_physical_host_plugin.py @@ -13,17 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections import datetime from unittest import mock import ddt +from unittest import skip from novaclient import client as nova_client from novaclient import exceptions as nova_exceptions from oslo_config import cfg from oslo_config import fixture as conf_fixture +import random import testtools -from blazar import context +from blazar import context, policy +from blazar import status from blazar.db import api as db_api from blazar.db import exceptions as db_exceptions from blazar.db import utils as db_utils @@ -39,15 +43,6 @@ CONF = cfg.CONF - -class AggregateFake(object): - - def __init__(self, i, name, hosts): - self.id = i - self.name = name - self.hosts = hosts - - class PhysicalHostPluginSetupOnlyTestCase(tests.TestCase): def setUp(self): @@ -71,28 +66,14 @@ def setUp(self): self.db_host_extra_capability_get_all_per_host = ( self.patch(self.db_api, 'host_extra_capability_get_all_per_host')) - def test_configuration(self): - self.assertEqual("fake-user", self.fake_phys_plugin.username) - self.assertEqual("fake-passwd", self.fake_phys_plugin.password) - self.assertEqual("fake-user-domain", - self.fake_phys_plugin.user_domain_name) - self.assertEqual("fake-pj-name", self.fake_phys_plugin.project_name) - self.assertEqual("fake-pj-domain", - self.fake_phys_plugin.project_domain_name) - def test__get_extra_capabilities_with_values(self): + ComputeHostExtraCapability = collections.namedtuple( + 'ComputeHostExtraCapability', + ['id', 'property_id', 'capability_value', 'computehost_id']) self.db_host_extra_capability_get_all_per_host.return_value = [ - {'id': 1, - 'capability_name': 'foo', - 'capability_value': 'bar', - 'other': 'value', - 'computehost_id': 1 - }, - {'id': 2, - 'capability_name': 'buzz', - 'capability_value': 'word', - 'computehost_id': 1 - }] + (ComputeHostExtraCapability(1, 'foo', 'bar', 1), 'foo'), + (ComputeHostExtraCapability(2, 'buzz', 'word', 1), 'buzz')] + res = self.fake_phys_plugin._get_extra_capabilities(1) self.assertEqual({'foo': 'bar', 'buzz': 'word'}, res) @@ -138,7 +119,7 @@ def setUp(self): self.db_utils = db_utils self.db_host_get = self.patch(self.db_api, 'host_get') - self.db_host_get.return_value = self.fake_host + self.db_host_get.return_value = self.fake_host.copy() self.db_host_list = self.patch(self.db_api, 'host_list') self.db_host_create = self.patch(self.db_api, 'host_create') self.db_host_update = self.patch(self.db_api, 'host_update') @@ -156,6 +137,9 @@ def setUp(self): self.db_host_extra_capability_update = self.patch( self.db_api, 'host_extra_capability_update') + self.db_host_extra_capability_destroy = self.patch( + self.db_api, 'host_extra_capability_destroy') + self.nova = nova self.rp_create = self.patch(self.nova.ReservationPool, 'create') self.patch(self.nova.ReservationPool, 'get_aggregate_from_name_or_id') @@ -165,7 +149,7 @@ def setUp(self): 'remove_computehost') self.get_host_details = self.patch(self.nova.NovaInventory, 'get_host_details') - self.get_host_details.return_value = self.fake_host + self.get_host_details.return_value = self.fake_host.copy() self.get_servers_per_host = self.patch( self.nova.NovaInventory, 'get_servers_per_host') self.get_servers_per_host.return_value = None @@ -197,6 +181,14 @@ def setUp(self): self.ServerManager = nova.ServerManager + def reservation_allocation_dict(self, r_id, l_id, p_id, h_ids): + return { + 'id': r_id, 'status': 'active', 'lease_id': l_id, + 'start_date': datetime.datetime(2015, 1, 1, 0, 0), + 'end_date': datetime.datetime(2015, 1, 2, 0, 0), + 'lease_name': l_id, 'project_id': p_id, + 'host_ids': h_ids} + def test_get_host(self): host = self.fake_phys_plugin.get_computehost(self.fake_host_id) self.db_host_get.assert_called_once_with('1') @@ -210,35 +202,138 @@ def test_get_host_without_extracapabilities(self): self.db_host_get.assert_called_once_with('1') self.assertEqual(self.fake_host, host) - @testtools.skip('incorrect decorator') def test_list_hosts(self): self.fake_phys_plugin.list_computehosts({}) self.db_host_list.assert_called_once_with() - del self.service_utils - def test_create_host_without_extra_capabilities(self): + @mock.patch.object(db_api, 'host_trait_create') + @mock.patch.object(db_api, 'host_resource_inventory_create') + @mock.patch.object(placement.BlazarPlacementClient, 'get_traits') + @mock.patch.object(placement.BlazarPlacementClient, 'get_inventory') + @mock.patch.object(placement.BlazarPlacementClient, + 'get_resource_provider') + def test_create_host_without_extra_capabilities( + self, mock_get_rp, mock_get_inventory, mock_get_traits, + mock_db_inv, mock_db_trait): + + mock_get_rp.return_value = {"uuid": "fake_rp_uuid"} + mock_get_traits.return_value = ["CUSTOM_HW_FPGA_CLASS1"] + mock_get_inventory.return_value = { + "inventories": { + "MEMORY_MB": { + "allocation_ratio": 1.5, + "max_unit": 5825, + "min_unit": 1, + "reserved": 512, + "step_size": 1, + "total": 5825 + }, + "VCPU": { + "allocation_ratio": 16.0, + "max_unit": 4, + "min_unit": 1, + "reserved": 0, + "step_size": 1, + "total": 4 + } + }, + "resource_provider_generation": 7 + } self.get_extra_capabilities.return_value = {} - host = self.fake_phys_plugin.create_computehost(self.fake_host) - self.db_host_create.assert_called_once_with(self.fake_host) - self.prov_create.assert_called_once_with('hypvsr1') + fake_request = self.fake_host.copy() + self.db_host_create.return_value = self.fake_host + + host = self.fake_phys_plugin.create_computehost(fake_request) + self.assertEqual(self.fake_host, host) + fake_db_return = self.fake_host.copy() + fake_db_return.pop("id") + self.db_host_create.assert_called_once_with(fake_db_return) + self.prov_create.assert_called_once_with('hypvsr1') + mock_get_inventory.assert_called_once_with("fake_rp_uuid") + mock_get_traits.assert_called_once_with("fake_rp_uuid") + mock_db_trait.assert_called_once_with({ + 'computehost_id': '1', + 'trait': "CUSTOM_HW_FPGA_CLASS1", + }) + mock_db_inv.assert_any_call({ + 'computehost_id': '1', + 'resource_class': "MEMORY_MB", + "allocation_ratio": 1.5, + "max_unit": 5825, + "min_unit": 1, + "reserved": 512, + "step_size": 1, + "total": 5825 + }) - def test_create_host_with_extra_capabilities(self): + @mock.patch.object(db_api, 'host_trait_create') + @mock.patch.object(db_api, 'host_resource_inventory_create') + @mock.patch.object(placement.BlazarPlacementClient, 'get_traits') + @mock.patch.object(placement.BlazarPlacementClient, 'get_inventory') + @mock.patch.object(placement.BlazarPlacementClient, + 'get_resource_provider') + def test_create_host_with_extra_capabilities( + self, mock_get_rp, mock_get_inventory, mock_get_traits, + mock_db_inv, mock_db_trait): + + mock_get_rp.return_value = {"uuid": "fake_rp_uuid"} + mock_get_traits.return_value = [] + mock_get_inventory.return_value = { + "inventories": { + "MEMORY_MB": { + "allocation_ratio": 1.5, + "max_unit": 5825, + "min_unit": 1, + "reserved": 512, + "step_size": 1, + "total": 5825 + } + }, + "resource_provider_generation": 7 + } fake_host = self.fake_host.copy() fake_host.update({'foo': 'bar'}) # NOTE(sbauza): 'id' will be pop'd, we need to keep track of it fake_request = fake_host.copy() fake_capa = {'computehost_id': '1', - 'capability_name': 'foo', + 'property_name': 'foo', 'capability_value': 'bar', } self.get_extra_capabilities.return_value = {'foo': 'bar'} - self.db_host_create.return_value = self.fake_host + self.db_host_create.return_value = fake_host host = self.fake_phys_plugin.create_computehost(fake_request) - self.db_host_create.assert_called_once_with(self.fake_host) + + self.assertEqual(fake_host, host) + expected_fake_host = self.fake_host.copy() + # the returned host will not have id + expected_fake_host.pop("id") + self.db_host_create.assert_called_once_with(expected_fake_host) self.prov_create.assert_called_once_with('hypvsr1') self.db_host_extra_capability_create.assert_called_once_with(fake_capa) self.assertEqual(fake_host, host) + mock_get_inventory.assert_called_once_with("fake_rp_uuid") + mock_get_traits.assert_called_once_with("fake_rp_uuid") + mock_db_trait.assert_not_called() + mock_db_inv.assert_called_once_with({ + 'computehost_id': '1', + 'resource_class': "MEMORY_MB", + "allocation_ratio": 1.5, + "max_unit": 5825, + "min_unit": 1, + "step_size": 1, + "reserved": 512, + "total": 5825 + }) + + @mock.patch.object(placement.BlazarPlacementClient, + 'get_resource_provider') + def test_create_host_without_resource_provider(self, mock_get_rp): + mock_get_rp.return_value = None + fake_host = self.fake_host.copy() + self.assertRaises(manager_exceptions.ResourceProviderNotFound, + self.fake_phys_plugin.create_computehost, + fake_host) def test_create_host_with_capabilities_too_long(self): fake_host = self.fake_host.copy() @@ -278,6 +373,14 @@ def fake_db_host_create(*args, **kwargs): self.prov_create.assert_called_once_with('hypvsr1') self.prov_delete.assert_called_once_with('hypvsr1') + def test_create_duplicate_host(self): + def fake_db_host_create(*args, **kwargs): + raise db_exceptions.BlazarDBDuplicateEntry + self.db_host_create.side_effect = fake_db_host_create + self.assertRaises(db_exceptions.BlazarDBDuplicateEntry, + self.fake_phys_plugin.create_computehost, + self.fake_host) + def test_create_host_having_issue_when_storing_extra_capability(self): def fake_db_host_extra_capability_create(*args, **kwargs): raise db_exceptions.BlazarDBException @@ -285,7 +388,7 @@ def fake_db_host_extra_capability_create(*args, **kwargs): fake_host.update({'foo': 'bar'}) fake_request = fake_host.copy() self.get_extra_capabilities.return_value = {'foo': 'bar'} - self.db_host_create.return_value = self.fake_host + self.db_host_create.return_value = fake_host fake = self.db_host_extra_capability_create fake.side_effect = fake_db_host_extra_capability_create self.assertRaises(manager_exceptions.CantAddExtraCapability, @@ -296,11 +399,10 @@ def test_update_host(self): host_values = {'foo': 'baz'} self.db_host_extra_capability_get_all_per_name.return_value = [ - {'id': 'extra_id1', - 'computehost_id': self.fake_host_id, - 'capability_name': 'foo', - 'capability_value': 'bar' - }, + ({'id': 'extra_id1', + 'computehost_id': self.fake_host_id, + 'capability_value': 'bar'}, + 'foo'), ] self.get_reservations_by_host = self.patch( @@ -310,7 +412,7 @@ def test_update_host(self): self.fake_phys_plugin.update_computehost(self.fake_host_id, host_values) self.db_host_extra_capability_update.assert_called_once_with( - 'extra_id1', {'capability_name': 'foo', 'capability_value': 'baz'}) + 'extra_id1', {'capability_value': 'baz'}) def test_update_host_having_issue_when_storing_extra_capability(self): def fake_db_host_extra_capability_update(*args, **kwargs): @@ -320,11 +422,10 @@ def fake_db_host_extra_capability_update(*args, **kwargs): self.db_utils, 'get_reservations_by_host_id') self.get_reservations_by_host.return_value = [] self.db_host_extra_capability_get_all_per_name.return_value = [ - {'id': 'extra_id1', - 'computehost_id': self.fake_host_id, - 'capability_name': 'foo', - 'capability_value': 'bar' - }, + ({'id': 'extra_id1', + 'computehost_id': self.fake_host_id, + 'capability_value': 'bar'}, + 'foo'), ] fake = self.db_host_extra_capability_update fake.side_effect = fake_db_host_extra_capability_update @@ -340,19 +441,37 @@ def test_update_host_with_new_extra_capability(self): host_values) self.db_host_extra_capability_create.assert_called_once_with({ 'computehost_id': '1', - 'capability_name': 'qux', + 'property_name': 'qux', 'capability_value': 'word' }) + def test_update_host_with_removed_capability(self): + host_values = {'foo': None} + + self.db_host_extra_capability_get_all_per_name.return_value = [ + ({'id': 'extra_id1', + 'computehost_id': self.fake_host_id, + 'capability_value': 'bar'}, + 'foo'), + ] + + self.get_reservations_by_host = self.patch( + self.db_utils, 'get_reservations_by_host_id') + self.get_reservations_by_host.return_value = [] + + self.fake_phys_plugin.update_computehost(self.fake_host_id, + host_values) + self.db_host_extra_capability_destroy.assert_called_once_with( + 'extra_id1') + def test_update_host_with_used_capability(self): host_values = {'foo': 'buzz'} self.db_host_extra_capability_get_all_per_name.return_value = [ - {'id': 'extra_id1', - 'computehost_id': self.fake_host_id, - 'capability_name': 'foo', - 'capability_value': 'bar' - }, + ({'id': 'extra_id1', + 'computehost_id': self.fake_host_id, + 'capability_value': 'bar'}, + 'foo'), ] fake_phys_reservation = { 'resource_type': plugin.RESOURCE_TYPE, @@ -437,32 +556,47 @@ def test_list_allocations(self): # Expecting a list of (Reservation, Allocation) self.db_get_reserv_allocs.return_value = [ - ('reservation-1', 'lease-1', 'host-1'), - ('reservation-1', 'lease-1', 'host-2'), - ('reservation-2', 'lease-1', 'host-2'), - ('reservation-2', 'lease-1', 'host-3'), - ('reservation-3', 'lease-2', 'host-1'), - ] + self.reservation_allocation_dict(*r) for r + in [ + ('reservation-1', 'lease-1', + 'project-1', ['host-1', 'host-2']), + ('reservation-2', 'lease-1', + 'project-1', ['host-2', 'host-3']), + ('reservation-3', 'lease-2', 'project-2', ['host-1'])]] + + self.db_host_list = self.patch(self.db_api, 'host_list') + self.db_host_list.return_value = [ + {'id': 'host-1'}, {'id': 'host-2'}, {'id': 'host-3'}] expected = [ { 'resource_id': 'host-1', 'reservations': [ - {'id': 'reservation-1', 'lease_id': 'lease-1'}, - {'id': 'reservation-3', 'lease_id': 'lease-2'}, + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + {'id': 'reservation-3', + 'lease_id': 'lease-2', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, ] }, { 'resource_id': 'host-2', 'reservations': [ - {'id': 'reservation-1', 'lease_id': 'lease-1'}, - {'id': 'reservation-2', 'lease_id': 'lease-1'}, + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + {'id': 'reservation-2', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, ] }, { 'resource_id': 'host-3', 'reservations': [ - {'id': 'reservation-2', 'lease_id': 'lease-1'}, + {'id': 'reservation-2', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, ] } ] @@ -481,30 +615,43 @@ def test_list_allocations_with_lease_id(self): # Expecting a list of (Reservation, Allocation) self.db_get_reserv_allocs.return_value = [ - ('reservation-1', 'lease-1', 'host-1'), - ('reservation-1', 'lease-1', 'host-2'), - ('reservation-2', 'lease-1', 'host-2'), - ('reservation-2', 'lease-1', 'host-3'), - ] + self.reservation_allocation_dict(*r) for r + in [ + ('reservation-1', 'lease-1', + 'project-1', ['host-1', 'host-2']), + ('reservation-2', 'lease-1', + 'project-1', ['host-2', 'host-3'])]] + + self.db_host_list = self.patch(self.db_api, 'host_list') + self.db_host_list.return_value = [ + {'id': 'host-1'}, {'id': 'host-2'}, {'id': 'host-3'}] expected = [ { 'resource_id': 'host-1', 'reservations': [ - {'id': 'reservation-1', 'lease_id': 'lease-1'}, + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, ] }, { 'resource_id': 'host-2', 'reservations': [ - {'id': 'reservation-1', 'lease_id': 'lease-1'}, - {'id': 'reservation-2', 'lease_id': 'lease-1'}, + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + {'id': 'reservation-2', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, ] }, { 'resource_id': 'host-3', 'reservations': [ - {'id': 'reservation-2', 'lease_id': 'lease-1'}, + {'id': 'reservation-2', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, ] } ] @@ -523,21 +670,29 @@ def test_list_allocations_with_reservation_id(self): # Expecting a list of (Reservation, Allocation) self.db_get_reserv_allocs.return_value = [ - ('reservation-1', 'lease-1', 'host-1'), - ('reservation-1', 'lease-1', 'host-2'), - ] + self.reservation_allocation_dict(*r) for r + in [ + ('reservation-1', 'lease-1', + 'project-1', ['host-1', 'host-2'])]] + + self.db_host_list = self.patch(self.db_api, 'host_list') + self.db_host_list.return_value = [{'id': 'host-1'}, {'id': 'host-2'}] expected = [ { 'resource_id': 'host-1', 'reservations': [ - {'id': 'reservation-1', 'lease_id': 'lease-1'}, + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, ] }, { 'resource_id': 'host-2', 'reservations': [ - {'id': 'reservation-1', 'lease_id': 'lease-1'}, + {'id': 'reservation-1', + 'lease_id': 'lease-1', 'extras': {}, + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, ] }, ] @@ -557,18 +712,25 @@ def test_get_allocations(self): # Expecting a list of (Reservation, Allocation) self.db_get_reserv_allocs.return_value = [ - ('reservation-1', 'lease-1', 'host-1'), - ('reservation-1', 'lease-1', 'host-2'), - ('reservation-2', 'lease-1', 'host-2'), - ('reservation-2', 'lease-1', 'host-3'), - ('reservation-3', 'lease-2', 'host-1'), - ] + self.reservation_allocation_dict(*r) for r + in [ + ('reservation-1', 'lease-1', + 'project-1', ['host-1', 'host-2']), + ('reservation-2', 'lease-1', + 'project-1', ['host-2', 'host-3']), + ('reservation-3', 'lease-2', 'project-2', ['host-1'])]] + + self.db_host_list = self.patch(self.db_api, 'host_list') + self.db_host_list.return_value = [ + {'id': 'host-1'}, {'id': 'host-2'}, {'id': 'host-3'}] expected = { 'resource_id': 'host-1', 'reservations': [ - {'id': 'reservation-1', 'lease_id': 'lease-1'}, - {'id': 'reservation-3', 'lease_id': 'lease-2'}, + {'id': 'reservation-1', 'lease_id': 'lease-1', + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, + {'id': 'reservation-3', 'lease_id': 'lease-2', + 'start_date': datetime.datetime(2015, 1, 1), 'end_date': datetime.datetime(2015, 1, 2)}, ] } ret = self.fake_phys_plugin.get_allocations('host-1', {}) @@ -584,15 +746,19 @@ def test_get_allocations_with_lease_id(self): # Expecting a list of (Reservation, Allocation) self.db_get_reserv_allocs.return_value = [ - ('reservation-1', 'lease-1', 'host-1'), + self.reservation_allocation_dict( + 'reservation-1', 'lease-1', 'project-1', ['host-1']), ] + self.db_host_list = self.patch(self.db_api, 'host_list') + self.db_host_list.return_value = [{'id': 'host-1'}] + expected = { 'resource_id': 'host-1', 'reservations': [ - {'id': 'reservation-1', 'lease_id': 'lease-1'}, - ] - } + {'id': 'reservation-1', 'lease_id': 'lease-1', + 'start_date': datetime.datetime(2015, 1, 1, 0, 0), 'end_date': datetime.datetime(2015, 1, 2, 0, 0)}]} + ret = self.fake_phys_plugin.get_allocations('host-1', {'lease_id': 'lease-1'}) @@ -607,15 +773,18 @@ def test_get_allocations_with_reservation_id(self): # Expecting a list of (Reservation, Allocation) self.db_get_reserv_allocs.return_value = [ - ('reservation-1', 'lease-1', 'host-1'), - ] + self.reservation_allocation_dict( + 'reservation-1', 'lease-1', 'project-1', ['host-1'])] + + self.db_host_list = self.patch(self.db_api, 'host_list') + self.db_host_list.return_value = [{'id': 'host-1'}] expected = { 'resource_id': 'host-1', 'reservations': [ - {'id': 'reservation-1', 'lease_id': 'lease-1'}, - ] - } + {'id': 'reservation-1', 'lease_id': 'lease-1', + 'start_date': datetime.datetime(2015, 1, 1, 0, 0), 'end_date': datetime.datetime(2015, 1, 2, 0, 0)}]} + ret = self.fake_phys_plugin.get_allocations( 'host-1', {'reservation_id': 'reservation-1'}) @@ -630,12 +799,18 @@ def test_get_allocations_with_invalid_host(self): # Expecting a list of (Reservation, Allocation) self.db_get_reserv_allocs.return_value = [ - ('reservation-1', 'lease-1', 'host-1'), - ('reservation-1', 'lease-1', 'host-2'), - ('reservation-2', 'lease-1', 'host-2'), - ('reservation-2', 'lease-1', 'host-3'), - ('reservation-3', 'lease-2', 'host-1'), - ] + self.reservation_allocation_dict(*r) for r + in [ + ('reservation-1', 'lease-1', + 'project-1', ['host-1', 'host-2']), + ('reservation-2', 'lease-1', + 'project-1', ['host-2', 'host-3']), + ('reservation-3', 'lease-2', 'project-2', ['host-1'])]] + + self.db_host_list = self.patch(self.db_api, 'host_list') + self.db_host_list.return_value = [ + {'id': 'host-1'}, {'id': 'host-2'}, {'id': 'host-3'}] + expected = {'resource_id': 'no-reserved-host', 'reservations': []} ret = self.fake_phys_plugin.get_allocations('no-reserved-host', {}) @@ -652,6 +827,7 @@ def test_create_reservation_no_hosts_available(self): 'start_date': now, 'end_date': now + datetime.timedelta(hours=1), 'resource_type': plugin.RESOURCE_TYPE, + 'project_id': 'fake-project' } host_reservation_create = self.patch(self.db_api, 'host_reservation_create') @@ -674,6 +850,7 @@ def test_create_reservation_hosts_available(self): 'start_date': datetime.datetime(2013, 12, 19, 20, 00), 'end_date': datetime.datetime(2013, 12, 19, 21, 00), 'resource_type': plugin.RESOURCE_TYPE, + 'project_id': 'fake-project' } self.rp_create.return_value = mock.MagicMock(id=1) host_reservation_create = self.patch(self.db_api, @@ -693,7 +870,8 @@ def test_create_reservation_hosts_available(self): 'hypervisor_properties': '["=", "$memory_mb", "256"]', 'count_range': '1-1', 'status': 'pending', - 'before_end': 'default' + 'before_end': 'default', + 'on_start': 'default' } host_reservation_create.assert_called_once_with(host_values) calls = [ @@ -708,24 +886,55 @@ def test_create_reservation_hosts_available(self): ] host_allocation_create.assert_has_calls(calls) - @ddt.data("min", "max", "hypervisor_properties", "resource_properties") - def test_create_reservation_with_missing_param(self, missing_param): + def test_create_reservation_hosts_non_reservable(self): values = { 'lease_id': '018c1b43-e69e-4aef-a543-09681539cf4c', 'min': 1, - 'max': 2, - 'before_end': 'default', + 'max': 1, 'hypervisor_properties': '["=", "$memory_mb", "256"]', 'resource_properties': '', - 'start_date': datetime.datetime(2017, 3, 1, 20, 00), - 'end_date': datetime.datetime(2017, 3, 2, 20, 00), - 'resource_type': plugin.RESOURCE_TYPE} - del values[missing_param] - self.assertRaises( - manager_exceptions.MissingParameter, - self.fake_phys_plugin.reserve_resource, + 'start_date': datetime.datetime(2013, 12, 19, 20, 00), + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'resource_type': plugin.RESOURCE_TYPE, + 'project_id': 'fake-project' + } + self.rp_create.return_value = mock.MagicMock(id=1) + host_reservation_create = self.patch(self.db_api, + 'host_reservation_create') + matching_hosts = self.patch(self.fake_phys_plugin, '_matching_hosts') + matching_hosts.return_value = ['host1', 'host2'] + host_allocation_create = self.patch( + self.db_api, + 'host_allocation_create') + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + self.fake_phys_plugin.reserve_resource( '441c1476-9f8f-4700-9f30-cd9b6fef3509', values) + host_values = { + 'reservation_id': '441c1476-9f8f-4700-9f30-cd9b6fef3509', + 'aggregate_id': 1, + 'resource_properties': '', + 'hypervisor_properties': '["=", "$memory_mb", "256"]', + 'count_range': '1-1', + 'status': 'pending', + 'before_end': 'default', + 'on_start': 'default' + } + host_reservation_create.assert_called_once_with(host_values) + calls = [ + mock.call( + {'compute_host_id': 'host1', + 'reservation_id': '441c1476-9f8f-4700-9f30-cd9b6fef3509', + }), + mock.call( + {'compute_host_id': 'host2', + 'reservation_id': '441c1476-9f8f-4700-9f30-cd9b6fef3509', + }), + ] + host_allocation_create.assert_has_calls(calls) @ddt.data({"params": {'max': 0}}, {"params": {'max': -1}}, @@ -748,6 +957,7 @@ def test_create_reservation_with_invalid_param(self, params): 'resource_type': plugin.RESOURCE_TYPE} for key, value in params.items(): values[key] = value + self.patch(db_api, 'host_allocation_get_all_by_values') self.assertRaises( manager_exceptions.MalformedParameter, self.fake_phys_plugin.reserve_resource, @@ -783,6 +993,7 @@ def test_update_reservation_with_invalid_param(self, params): } for key, value in params.items(): values[key] = value + self.patch(db_api, 'host_allocation_get_all_by_values') self.assertRaises( manager_exceptions.MalformedParameter, self.fake_phys_plugin.update_reservation, @@ -802,6 +1013,7 @@ def test_create_update_reservation_with_invalid_range(self): } self.patch(self.db_api, 'reservation_get') self.patch(self.db_api, 'lease_get') + self.patch(db_api, 'host_allocation_get_all_by_values') host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { @@ -855,7 +1067,8 @@ def test_update_reservation_extend(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2013, 12, 19, 20, 00), - 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { @@ -897,7 +1110,8 @@ def test_update_reservation_extend(self): def test_update_reservation_move_failure(self): values = { 'start_date': datetime.datetime(2013, 12, 20, 20, 00), - 'end_date': datetime.datetime(2013, 12, 20, 21, 30) + 'end_date': datetime.datetime(2013, 12, 20, 21, 30), + 'project_id': 'fake-project' } reservation_get = self.patch(self.db_api, 'reservation_get') reservation_get.return_value = { @@ -908,7 +1122,8 @@ def test_update_reservation_move_failure(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2013, 12, 19, 20, 00), - 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch( self.db_api, @@ -962,7 +1177,8 @@ def test_update_reservation_move_overlap(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2013, 12, 19, 20, 00), - 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch( self.db_api, @@ -1003,6 +1219,158 @@ def test_update_reservation_move_overlap(self): host_allocation_create.assert_not_called() host_allocation_destroy.assert_not_called() + def test_update_reservation_move_with_cleaning_time_overlap(self): + self.cfg.CONF.set_override('cleaning_time', 5) + # Lease 1 is 20:00 to 21:00. + # Lease 2 is 22:00 to 23:00, moved backwards to 21:00 to 22:00 + values = { + 'start_date': datetime.datetime(2013, 12, 19, 21, 00), + 'end_date': datetime.datetime(2013, 12, 19, 22, 00) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' # pending so it can fail with NotEnoughHostsAvailable if no resources + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 22, 00), + 'end_date': datetime.datetime(2013, 12, 19, 23, 00), + 'project_id': 'fake-project' + } + host_reservation_get = self.patch( + self.db_api, + 'host_reservation_get') + host_reservation_get.return_value = { + 'count_range': '1-1', + 'hypervisor_properties': '["=", "$memory_mb", "256"]', + 'resource_properties': '' + } + host_allocation_get_all = self.patch( + self.db_api, + 'host_allocation_get_all_by_values') + host_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'compute_host_id': 'host1' + } + ] + host_get_all_by_queries = self.patch(self.db_api, + 'host_get_all_by_queries') + host_get_all_by_queries.return_value = [{'id': 'host1'}] + # When get_reserved_periods is called, we will simulate what it returns + # based on the arguments. + # If it's called with the START margin (20:55 to 22:05), it should see Lease 1 + # which overlaps at 20:55 to 21:00. + def fake_get_reserved_periods(compute_host_id, start_date, end_date, duration): + # Lease 1: 20:00 to 21:00 + # Lease 2: 22:00 to 23:00 (original) + periods = [] + if start_date < datetime.datetime(2013, 12, 19, 21, 00): + # Lease 1 overlaps + periods.append(( + max(start_date, datetime.datetime(2013, 12, 19, 20, 00)), + min(end_date, datetime.datetime(2013, 12, 19, 21, 00)) + )) + # Lease 2 intersection (itself, which we are supposed to ignore or whatever + # - actually self-intersect is handled by comparing to max_start/min_end) + periods.append(( + max(start_date, datetime.datetime(2013, 12, 19, 22, 00)), + min(end_date, datetime.datetime(2013, 12, 19, 23, 00)) + )) + # filter out invalid ones + return [p for p in periods if p[0] < p[1]] + + get_reserved_periods = self.patch(self.db_utils, + 'get_reserved_periods') + get_reserved_periods.side_effect = fake_get_reserved_periods + + matching_hosts = self.patch(self.fake_phys_plugin, '_matching_hosts') + matching_hosts.return_value = [] + + self.assertRaises( + manager_exceptions.NotEnoughHostsAvailable, + self.fake_phys_plugin.update_reservation, + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + + def test_update_reservation_move_with_cleaning_time_success(self): + self.cfg.CONF.set_override('cleaning_time', 5) + # Lease 1 is 20:00 to 20:55. + # Lease 2 is 22:00 to 23:00, moved backwards to 21:00 to 22:00. + # The cleaning time brings Lease 2's start to 20:55 and end to 22:05, + # which does NOT overlap with Lease 1. + values = { + 'start_date': datetime.datetime(2013, 12, 19, 21, 00), + 'end_date': datetime.datetime(2013, 12, 19, 22, 00) + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = { + 'lease_id': '10870923-6d56-45c9-b592-f788053f5baa', + 'resource_id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', + 'status': 'pending' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = { + 'start_date': datetime.datetime(2013, 12, 19, 22, 00), + 'end_date': datetime.datetime(2013, 12, 19, 23, 00), + 'project_id': 'fake-project' + } + host_reservation_get = self.patch( + self.db_api, + 'host_reservation_get') + host_reservation_get.return_value = { + 'count_range': '1-1', + 'hypervisor_properties': '["=", "$memory_mb", "256"]', + 'resource_properties': '' + } + host_allocation_get_all = self.patch( + self.db_api, + 'host_allocation_get_all_by_values') + host_allocation_get_all.return_value = [ + { + 'id': 'dd305477-4df8-4547-87f6-69069ee546a6', + 'compute_host_id': 'host1' + } + ] + host_get_all_by_queries = self.patch(self.db_api, + 'host_get_all_by_queries') + host_get_all_by_queries.return_value = [{'id': 'host1'}] + # If the bug is fixed, we call with margin: start=20:55, end=22:05. + def fake_get_reserved_periods(compute_host_id, start_date, end_date, duration): + # Lease 1: 20:00 to 20:55 + # Lease 2: 22:00 to 23:00 (itself) + periods = [] + if start_date < datetime.datetime(2013, 12, 19, 20, 55): + periods.append(( + max(start_date, datetime.datetime(2013, 12, 19, 20, 00)), + min(end_date, datetime.datetime(2013, 12, 19, 20, 55)) + )) + periods.append(( + max(start_date, datetime.datetime(2013, 12, 19, 22, 00)), + min(end_date, datetime.datetime(2013, 12, 19, 23, 00)) + )) + return [p for p in periods if p[0] < p[1]] + + get_reserved_periods = self.patch(self.db_utils, + 'get_reserved_periods') + get_reserved_periods.side_effect = fake_get_reserved_periods + + matching_hosts = self.patch(self.fake_phys_plugin, '_matching_hosts') + matching_hosts.return_value = [] + + host_allocation_create = self.patch(self.db_api, 'host_allocation_create') + host_allocation_destroy = self.patch(self.db_api, 'host_allocation_destroy') + + # Should not raise any exception! + self.fake_phys_plugin.update_reservation( + '706eb3bc-07ed-4383-be93-b32845ece672', + values) + + host_allocation_create.assert_not_called() + host_allocation_destroy.assert_not_called() + def test_update_reservation_move_realloc(self): values = { 'start_date': datetime.datetime(2013, 12, 20, 20, 00), @@ -1017,7 +1385,8 @@ def test_update_reservation_move_realloc(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2013, 12, 19, 20, 00), - 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch( self.db_api, @@ -1084,14 +1453,15 @@ def test_update_reservation_min_increase_success(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2017, 7, 12, 20, 00), - 'end_date': datetime.datetime(2017, 7, 12, 21, 00) + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', 'count_range': '2-3', 'hypervisor_properties': '["=", "$memory_mb", "16384"]', - 'resource_properties': '' + 'resource_properties': '', } host_allocation_get_all = self.patch( self.db_api, 'host_allocation_get_all_by_values') @@ -1131,7 +1501,8 @@ def test_update_reservation_min_increase_success(self): '', '1-1', datetime.datetime(2017, 7, 12, 20, 00), - datetime.datetime(2017, 7, 12, 21, 00) + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' ) host_allocation_destroy.assert_not_called() host_allocation_create.assert_called_with( @@ -1160,7 +1531,8 @@ def test_update_reservation_min_increase_fail(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2017, 7, 12, 20, 00), - 'end_date': datetime.datetime(2017, 7, 12, 21, 00) + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { @@ -1200,7 +1572,8 @@ def test_update_reservation_min_increase_fail(self): '', '1-1', datetime.datetime(2017, 7, 12, 20, 00), - datetime.datetime(2017, 7, 12, 21, 00) + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' ) def test_update_reservation_min_decrease(self): @@ -1218,7 +1591,8 @@ def test_update_reservation_min_decrease(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2017, 7, 12, 20, 00), - 'end_date': datetime.datetime(2017, 7, 12, 21, 00) + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { @@ -1279,7 +1653,8 @@ def test_update_reservation_max_increase_alloc(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2017, 7, 12, 20, 00), - 'end_date': datetime.datetime(2017, 7, 12, 21, 00) + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { @@ -1326,7 +1701,8 @@ def test_update_reservation_max_increase_alloc(self): '', '0-1', datetime.datetime(2017, 7, 12, 20, 00), - datetime.datetime(2017, 7, 12, 21, 00) + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' ) host_allocation_destroy.assert_not_called() host_allocation_create.assert_called_with( @@ -1355,7 +1731,8 @@ def test_update_active_reservation_max_increase_alloc(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2017, 7, 12, 20, 00), - 'end_date': datetime.datetime(2017, 7, 12, 21, 00) + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { @@ -1392,7 +1769,10 @@ def test_update_active_reservation_max_increase_alloc(self): matching_hosts = self.patch(self.fake_phys_plugin, '_matching_hosts') matching_hosts.return_value = ['host3'] host_get = self.patch(self.db_api, 'host_get') - host_get.return_value = {'service_name': 'host3_hostname'} + host_get.return_value = { + 'hypervisor_hostname': 'host3_hostname', + 'service_name': 'service1', + } add_computehost = self.patch( self.nova.ReservationPool, 'add_computehost') host_reservation_update = self.patch(self.db_api, @@ -1408,7 +1788,8 @@ def test_update_active_reservation_max_increase_alloc(self): '', '0-1', datetime.datetime(2017, 7, 12, 20, 00), - datetime.datetime(2017, 7, 12, 21, 00) + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' ) host_allocation_destroy.assert_not_called() host_allocation_create.assert_called_with( @@ -1438,7 +1819,8 @@ def test_update_reservation_max_increase_noalloc(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2017, 7, 12, 20, 00), - 'end_date': datetime.datetime(2017, 7, 12, 21, 00) + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { @@ -1480,7 +1862,8 @@ def test_update_reservation_max_increase_noalloc(self): '', '0-1', datetime.datetime(2017, 7, 12, 20, 00), - datetime.datetime(2017, 7, 12, 21, 00) + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' ) host_reservation_update.assert_called_with( '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', @@ -1502,14 +1885,16 @@ def test_update_reservation_max_decrease(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2017, 7, 12, 20, 00), - 'end_date': datetime.datetime(2017, 7, 12, 21, 00) + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { 'id': '91253650-cc34-4c4f-bbe8-c943aa7d0c9b', 'count_range': '1-2', 'hypervisor_properties': '["=", "$memory_mb", "16384"]', - 'resource_properties': '' + 'resource_properties': '', + 'project_id': 'fake-project' } host_allocation_get_all = self.patch( self.db_api, 'host_allocation_get_all_by_values') @@ -1561,7 +1946,8 @@ def test_update_reservation_realloc_with_properties_change(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2017, 7, 12, 20, 00), - 'end_date': datetime.datetime(2017, 7, 12, 21, 00) + 'end_date': datetime.datetime(2017, 7, 12, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { @@ -1600,7 +1986,8 @@ def test_update_reservation_realloc_with_properties_change(self): '', '1-1', datetime.datetime(2017, 7, 12, 20, 00), - datetime.datetime(2017, 7, 12, 21, 00) + datetime.datetime(2017, 7, 12, 21, 00), + 'fake-project' ) host_allocation_create.assert_called_with( { @@ -1631,7 +2018,8 @@ def test_update_reservation_no_requested_hosts_available(self): lease_get = self.patch(self.db_api, 'lease_get') lease_get.return_value = { 'start_date': datetime.datetime(2013, 12, 19, 20, 00), - 'end_date': datetime.datetime(2013, 12, 19, 21, 00) + 'end_date': datetime.datetime(2013, 12, 19, 21, 00), + 'project_id': 'fake-project' } host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = { @@ -1672,7 +2060,7 @@ def test_on_start(self): {'compute_host_id': 'host1'}, ] host_get = self.patch(self.db_api, 'host_get') - host_get.return_value = {'service_name': 'host1_hostname'} + host_get.return_value = {'hypervisor_hostname': 'host1_hostname'} add_computehost = self.patch( self.nova.ReservationPool, 'add_computehost') @@ -1684,8 +2072,9 @@ def test_before_end_with_no_action(self): host_reservation_get = self.patch(self.db_api, 'host_reservation_get') host_reservation_get.return_value = {'before_end': ''} reservationpool = self.patch(self.nova, 'ReservationPool') + fake_lease = {'project_id': 'fake-project'} self.fake_phys_plugin.before_end( - '04de74e8-193a-49d2-9ab8-cba7b49e45e8') + '04de74e8-193a-49d2-9ab8-cba7b49e45e8', lease=fake_lease) reservationpool.assert_not_called() def test_before_end_with_snapshot(self): @@ -1700,8 +2089,9 @@ def test_before_end_with_snapshot(self): list_servers = self.patch(self.ServerManager, 'list') list_servers.return_value = ['server1', 'server2'] create_image = self.patch(self.ServerManager, 'create_image') + fake_lease = {'project_id': 'fake-project'} self.fake_phys_plugin.before_end( - '04de74e8-193a-49d2-9ab8-cba7b49e45e8') + '04de74e8-193a-49d2-9ab8-cba7b49e45e8', lease=fake_lease) create_image.assert_any_call(server='server1') create_image.assert_any_call(server='server2') @@ -1743,7 +2133,7 @@ def test_on_end_with_instances(self): '04de74e8-193a-49d2-9ab8-cba7b49e45e8', {'status': 'completed'}) host_allocation_destroy.assert_called_with( 'bfa9aa0b-8042-43eb-a4e6-4555838bf64f') - list_servers.assert_called_with(search_opts={'host': 'host', + list_servers.assert_called_with(search_opts={'node': 'host', 'all_tenants': 1}) delete_server.assert_any_call(server='server1') delete_server.assert_any_call(server='server2') @@ -1785,6 +2175,7 @@ def test_on_end_without_instances(self): delete_server.assert_not_called() delete_pool.assert_called_with(1) + @skip # these tests pass when ran individually def test_heal_reservations_before_start_and_resources_changed(self): failed_host = {'id': '1'} dummy_reservation = { @@ -1803,25 +2194,68 @@ def test_heal_reservations_before_start_and_resources_changed(self): get_reservations = self.patch(self.db_utils, 'get_reservations_by_host_ids') get_reservations.return_value = [dummy_reservation] - reallocate = self.patch(self.fake_phys_plugin, '_reallocate') + reallocate = self.patch(self.fake_phys_plugin.monitor, '_reallocate') reallocate.return_value = True + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation - result = self.fake_phys_plugin.heal_reservations( - [failed_host], - datetime.datetime(2020, 1, 1, 12, 00), - datetime.datetime(2020, 1, 1, 13, 00)) - reallocate.assert_called_once_with( - dummy_reservation['computehost_allocations'][0]) - self.assertEqual({}, result) + dummy_host_reservation = { + 'aggregate_id': 1 + } + host_reservation_get = self.patch(self.db_api, 'host_reservation_get') + host_reservation_get.return_value = dummy_host_reservation - def test_heal_reservations_before_start_and_missing_resources(self): - failed_host = {'id': '1'} - dummy_reservation = { - 'id': 'rsrv-1', - 'resource_type': plugin.RESOURCE_TYPE, - 'lease_id': 'lease-1', - 'status': 'pending', - 'hypervisor_properties': [], + host_get = self.patch( + self.db_api, + 'reservable_host_get_all_by_queries') + host_get.return_value = [ + {'id': 'host1'}, + {'id': 'host2'}, + {'id': 'host3'}, + ] + + host_get = self.patch( + self.db_api, + 'host_allocation_get_all_by_values') + + def host_allocation_get_all_by_values(**kwargs): + if kwargs['compute_host_id'] == 'host1': + return True + host_get.side_effect = host_allocation_get_all_by_values + host_get = self.patch( + self.db_utils, + 'get_free_periods') + host_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + self.patch(self.db_api, 'host_allocation_update') + + result = self.fake_phys_plugin.monitor.heal_reservations( + [failed_host], + datetime.datetime(2020, 1, 1, 12, 00), + datetime.datetime(2020, 1, 1, 13, 00)) + reallocate.assert_called_once_with( + dummy_reservation['computehost_allocations'][0]) + self.assertEqual({}, result) + + @skip + def test_heal_reservations_before_start_and_missing_resources(self): + failed_host = {'id': '1'} + dummy_reservation = { + 'id': 'rsrv-1', + 'resource_type': plugin.RESOURCE_TYPE, + 'lease_id': 'lease-1', + 'status': 'pending', + 'hypervisor_properties': [], 'resource_properties': [], 'resource_id': 'resource-1', 'computehost_allocations': [{ @@ -1832,10 +2266,52 @@ def test_heal_reservations_before_start_and_missing_resources(self): get_reservations = self.patch(self.db_utils, 'get_reservations_by_host_ids') get_reservations.return_value = [dummy_reservation] - reallocate = self.patch(self.fake_phys_plugin, '_reallocate') + reallocate = self.patch(self.fake_phys_plugin.monitor, '_reallocate') reallocate.return_value = False + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + + dummy_host_reservation = { + 'aggregate_id': 1 + } + host_reservation_get = self.patch(self.db_api, 'host_reservation_get') + host_reservation_get.return_value = dummy_host_reservation + + host_get = self.patch( + self.db_api, + 'reservable_host_get_all_by_queries') + host_get.return_value = [ + {'id': 'host1'}, + {'id': 'host2'}, + {'id': 'host3'}, + ] + + host_get = self.patch( + self.db_api, + 'host_allocation_get_all_by_values') + + def host_allocation_get_all_by_values(**kwargs): + if kwargs['compute_host_id'] == 'host1': + return True + host_get.side_effect = host_allocation_get_all_by_values + host_get = self.patch( + self.db_utils, + 'get_free_periods') + host_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + self.patch(self.db_api, 'host_allocation_update') - result = self.fake_phys_plugin.heal_reservations( + result = self.fake_phys_plugin.monitor.heal_reservations( [failed_host], datetime.datetime(2020, 1, 1, 12, 00), datetime.datetime(2020, 1, 1, 13, 00)) @@ -1863,18 +2339,62 @@ def test_heal_active_reservations_and_resources_changed(self): get_reservations = self.patch(self.db_utils, 'get_reservations_by_host_ids') get_reservations.return_value = [dummy_reservation] - reallocate = self.patch(self.fake_phys_plugin, '_reallocate') + + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + + dummy_host_reservation = { + 'aggregate_id': 1 + } + host_reservation_get = self.patch(self.db_api, 'host_reservation_get') + host_reservation_get.return_value = dummy_host_reservation + + host_get = self.patch( + self.db_api, + 'reservable_host_get_all_by_queries') + host_get.return_value = [ + {'id': 'host1'}, + {'id': 'host2'}, + {'id': 'host3'}, + ] + + host_get = self.patch( + self.db_api, + 'host_allocation_get_all_by_values') + + def host_allocation_get_all_by_values(**kwargs): + if kwargs['compute_host_id'] == 'host1': + return True + host_get.side_effect = host_allocation_get_all_by_values + host_get = self.patch( + self.db_utils, + 'get_free_periods') + host_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + + reallocate = self.patch(self.fake_phys_plugin.monitor, '_reallocate') reallocate.return_value = True - result = self.fake_phys_plugin.heal_reservations( + self.patch(self.db_api, 'host_allocation_update') + + result = self.fake_phys_plugin.monitor.heal_reservations( [failed_host], datetime.datetime(2020, 1, 1, 12, 00), datetime.datetime(2020, 1, 1, 13, 00)) - reallocate.assert_called_once_with( - dummy_reservation['computehost_allocations'][0]) - self.assertEqual( - {dummy_reservation['id']: {'resources_changed': True}}, - result) + # skipped for leases which are active d285bc9 + reallocate.assert_not_called() + self.assertEqual({}, result) def test_heal_active_reservations_and_missing_resources(self): failed_host = {'id': '1'} @@ -1894,18 +2414,55 @@ def test_heal_active_reservations_and_missing_resources(self): get_reservations = self.patch(self.db_utils, 'get_reservations_by_host_ids') get_reservations.return_value = [dummy_reservation] - reallocate = self.patch(self.fake_phys_plugin, '_reallocate') + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + reallocate = self.patch(self.fake_phys_plugin.monitor, '_reallocate') reallocate.return_value = False + dummy_host_reservation = { + 'aggregate_id': 1 + } + host_reservation_get = self.patch(self.db_api, 'host_reservation_get') + host_reservation_get.return_value = dummy_host_reservation + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1' + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + host_get = self.patch( + self.db_api, + 'reservable_host_get_all_by_queries') + host_get.return_value = [ + {'id': 'host1'}, + {'id': 'host2'}, + {'id': 'host3'}, + ] + host_get = self.patch( + self.db_api, + 'host_allocation_get_all_by_values') - result = self.fake_phys_plugin.heal_reservations( + def host_allocation_get_all_by_values(**kwargs): + if kwargs['compute_host_id'] == 'host1': + return True + host_get.side_effect = host_allocation_get_all_by_values + host_get = self.patch( + self.db_utils, + 'get_free_periods') + host_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + self.patch(self.db_api, 'host_allocation_update') + + result = self.fake_phys_plugin.monitor.heal_reservations( [failed_host], datetime.datetime(2020, 1, 1, 12, 00), datetime.datetime(2020, 1, 1, 13, 00)) - reallocate.assert_called_once_with( - dummy_reservation['computehost_allocations'][0]) - self.assertEqual( - {dummy_reservation['id']: {'missing_resources': True}}, - result) + # skipped for leases which are active d285bc9 + reallocate.assert_not_called() + self.assertEqual({}, result) def test_reallocate_before_start(self): failed_host = {'id': '1'} @@ -1913,7 +2470,7 @@ def test_reallocate_before_start(self): dummy_allocation = { 'id': 'alloc-1', 'compute_host_id': failed_host['id'], - 'reservation_id': 'rsrv-1' + 'reservation_id': 'rsrv-1', } dummy_reservation = { 'id': 'rsrv-1', @@ -1931,7 +2488,8 @@ def test_reallocate_before_start(self): 'name': 'lease-name', 'start_date': datetime.datetime(2020, 1, 1, 12, 00), 'end_date': datetime.datetime(2020, 1, 2, 12, 00), - 'trust_id': 'trust-1' + 'trust_id': 'trust-1', + 'project_id': 'fake-project' } reservation_get = self.patch(self.db_api, 'reservation_get') reservation_get.return_value = dummy_reservation @@ -1948,12 +2506,15 @@ def test_reallocate_before_start(self): mock.Mock(wraps=datetime.datetime)) as patched: patched.utcnow.return_value = datetime.datetime( 2020, 1, 1, 11, 00) - result = self.fake_phys_plugin._reallocate(dummy_allocation) + result = self.fake_phys_plugin._reallocate(dummy_allocation, force=True) matching_hosts.assert_called_once_with( dummy_reservation['hypervisor_properties'], dummy_reservation['resource_properties'], - '1-1', dummy_lease['start_date'], dummy_lease['end_date']) + '1-1', dummy_lease['start_date'], dummy_lease['end_date'], + dummy_lease['project_id'], + allow_unreservable=False, + ) alloc_update.assert_called_once_with( dummy_allocation['id'], {'compute_host_id': new_host['id']}) @@ -1961,9 +2522,9 @@ def test_reallocate_before_start(self): def test_reallocate_active(self): failed_host = {'id': '1', - 'service_name': 'compute-1'} + 'hypervisor_hostname': 'compute-1'} new_host = {'id': '2', - 'service_name': 'compute-2'} + 'hypervisor_hostname': 'compute-2'} dummy_allocation = { 'id': 'alloc-1', 'compute_host_id': failed_host['id'], @@ -1985,7 +2546,8 @@ def test_reallocate_active(self): 'name': 'lease-name', 'start_date': datetime.datetime(2020, 1, 1, 12, 00), 'end_date': datetime.datetime(2020, 1, 2, 12, 00), - 'trust_id': 'trust-1' + 'trust_id': 'trust-1', + 'project_id': 'fake-project' } reservation_get = self.patch(self.db_api, 'reservation_get') reservation_get.return_value = dummy_reservation @@ -2004,22 +2566,26 @@ def test_reallocate_active(self): mock.Mock(wraps=datetime.datetime)) as patched: patched.utcnow.return_value = datetime.datetime( 2020, 1, 1, 13, 00) - result = self.fake_phys_plugin._reallocate(dummy_allocation) + self.nova_client.client.get.return_value = None, {'servers': []} + result = self.fake_phys_plugin._reallocate(dummy_allocation, force=True) self.remove_compute_host.assert_called_once_with( dummy_host_reservation['aggregate_id'], - failed_host['service_name']) + failed_host['hypervisor_hostname']) matching_hosts.assert_called_once_with( dummy_reservation['hypervisor_properties'], dummy_reservation['resource_properties'], '1-1', datetime.datetime(2020, 1, 1, 13, 00), - dummy_lease['end_date']) + dummy_lease['end_date'], + 'fake-project', + allow_unreservable=False, + ) alloc_update.assert_called_once_with( dummy_allocation['id'], {'compute_host_id': new_host['id']}) self.add_compute_host( dummy_host_reservation['aggregate_id'], - new_host['service_name']) + new_host['hypervisor_hostname']) self.assertEqual(True, result) def test_reallocate_missing_resources(self): @@ -2045,7 +2611,8 @@ def test_reallocate_missing_resources(self): 'name': 'lease-name', 'start_date': datetime.datetime(2020, 1, 1, 12, 00), 'end_date': datetime.datetime(2020, 1, 2, 12, 00), - 'trust_id': 'trust-1' + 'trust_id': 'trust-1', + 'project_id': 'fake-project' } reservation_get = self.patch(self.db_api, 'reservation_get') reservation_get.return_value = dummy_reservation @@ -2062,15 +2629,72 @@ def test_reallocate_missing_resources(self): mock.Mock(wraps=datetime.datetime)) as patched: patched.utcnow.return_value = datetime.datetime( 2020, 1, 1, 11, 00) - result = self.fake_phys_plugin._reallocate(dummy_allocation) + result = self.fake_phys_plugin._reallocate(dummy_allocation, force=True) matching_hosts.assert_called_once_with( dummy_reservation['hypervisor_properties'], dummy_reservation['resource_properties'], - '1-1', dummy_lease['start_date'], dummy_lease['end_date']) + '1-1', dummy_lease['start_date'], dummy_lease['end_date'], + dummy_lease['project_id'], + allow_unreservable=False, + ) alloc_destroy.assert_called_once_with(dummy_allocation['id']) self.assertEqual(False, result) + def test_reallocate_missing_resources_no_force(self): + failed_host = {'id': '1'} + dummy_allocation = { + 'id': 'alloc-1', + 'compute_host_id': failed_host['id'], + 'reservation_id': 'rsrv-1' + } + dummy_reservation = { + 'id': 'rsrv-1', + 'resource_type': plugin.RESOURCE_TYPE, + 'lease_id': 'lease-1', + 'status': 'pending', + 'hypervisor_properties': [], + 'resource_properties': [], + 'resource_id': 'resource-1' + } + dummy_host_reservation = { + 'aggregate_id': 1 + } + dummy_lease = { + 'name': 'lease-name', + 'start_date': datetime.datetime(2020, 1, 1, 12, 00), + 'end_date': datetime.datetime(2020, 1, 2, 12, 00), + 'trust_id': 'trust-1', + 'project_id': 'fake-project' + } + reservation_get = self.patch(self.db_api, 'reservation_get') + reservation_get.return_value = dummy_reservation + host_reservation_get = self.patch(self.db_api, 'host_reservation_get') + host_reservation_get.return_value = dummy_host_reservation + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = dummy_lease + matching_hosts = self.patch(host_plugin.PhysicalHostPlugin, + '_matching_hosts') + matching_hosts.return_value = [] + alloc_destroy = self.patch(self.db_api, 'host_allocation_destroy') + + with mock.patch.object(datetime, 'datetime', + mock.Mock(wraps=datetime.datetime)) as patched: + patched.utcnow.return_value = datetime.datetime( + 2020, 1, 1, 11, 00) + result = self.fake_phys_plugin._reallocate(dummy_allocation, force=False) + + matching_hosts.assert_called_once_with( + dummy_reservation['hypervisor_properties'], + dummy_reservation['resource_properties'], + '1-1', dummy_lease['start_date'], dummy_lease['end_date'], + dummy_lease['project_id'], + allow_unreservable=False, + ) + alloc_destroy.assert_not_called() + self.assertEqual(False, result) + + def test_matching_hosts_not_allocated_hosts(self): def host_allocation_get_all_by_values(**kwargs): if kwargs['compute_host_id'] == 'host1': @@ -2094,11 +2718,17 @@ def host_allocation_get_all_by_values(**kwargs): (datetime.datetime(2013, 12, 19, 20, 00), datetime.datetime(2013, 12, 19, 21, 00)), ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False result = self.fake_phys_plugin._matching_hosts( '[]', '[]', '1-3', datetime.datetime(2013, 12, 19, 20, 00), - datetime.datetime(2013, 12, 19, 21, 00)) - self.assertEqual(['host2', 'host3'], result) + datetime.datetime(2013, 12, 19, 21, 00), + 'fake-project' + ) + self.assertEqual(set(['host2', 'host3']), set(result)) def test_matching_hosts_allocated_hosts(self): def host_allocation_get_all_by_values(**kwargs): @@ -2123,11 +2753,17 @@ def host_allocation_get_all_by_values(**kwargs): (datetime.datetime(2013, 12, 19, 20, 00), datetime.datetime(2013, 12, 19, 21, 00)), ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False result = self.fake_phys_plugin._matching_hosts( '[]', '[]', '3-3', datetime.datetime(2013, 12, 19, 20, 00), - datetime.datetime(2013, 12, 19, 21, 00)) - self.assertEqual(['host1', 'host2', 'host3'], result) + datetime.datetime(2013, 12, 19, 21, 00), + None + ) + self.assertEqual(set(['host1', 'host2', 'host3']), set(result)) def test_matching_hosts_allocated_hosts_with_cleaning_time(self): def host_allocation_get_all_by_values(**kwargs): @@ -2155,21 +2791,154 @@ def host_allocation_get_all_by_values(**kwargs): datetime.datetime(2013, 12, 19, 21, 00) + datetime.timedelta(minutes=5)) ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False result = self.fake_phys_plugin._matching_hosts( '[]', '[]', '3-3', datetime.datetime(2013, 12, 19, 20, 00), - datetime.datetime(2013, 12, 19, 21, 00)) + datetime.datetime(2013, 12, 19, 21, 00), + None) + self.addCleanup(CONF.clear_override, 'cleaning_time') self.assertEqual(['host1', 'host2', 'host3'], result) + @mock.patch.object(random, "shuffle") + def test_random_matching_hosts_not_allocated_hosts(self, mock_shuffle): + def host_allocation_get_all_by_values(**kwargs): + if kwargs['compute_host_id'] == 'host1': + return True + self.cfg.CONF.set_override('randomize_host_selection', True, + group=plugin.RESOURCE_TYPE) + host_get = self.patch( + self.db_api, + 'reservable_host_get_all_by_queries') + host_get.return_value = [ + {'id': 'host1'}, + {'id': 'host2'}, + {'id': 'host3'}, + ] + host_get = self.patch( + self.db_api, + 'host_allocation_get_all_by_values') + host_get.side_effect = host_allocation_get_all_by_values + host_get = self.patch( + self.db_utils, + 'get_free_periods') + host_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + self.fake_phys_plugin._matching_hosts( + '[]', '[]', '1-3', + datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00), + None) + self.addCleanup(CONF.clear_override, 'randomize_host_selection', + group=plugin.RESOURCE_TYPE) + mock_shuffle.assert_called_once_with(['host2', 'host3']) + + @mock.patch.object(random, "shuffle") + def test_random_matching_hosts_allocated_hosts(self, mock_shuffle): + def host_allocation_get_all_by_values(**kwargs): + if kwargs['compute_host_id'] == 'host1': + return True + self.cfg.CONF.set_override('randomize_host_selection', True, + group=plugin.RESOURCE_TYPE) + host_get = self.patch( + self.db_api, + 'reservable_host_get_all_by_queries') + host_get.return_value = [ + {'id': 'host1'}, + {'id': 'host2'}, + {'id': 'host3'}, + ] + host_get = self.patch( + self.db_api, + 'host_allocation_get_all_by_values') + host_get.side_effect = host_allocation_get_all_by_values + host_get = self.patch( + self.db_utils, + 'get_free_periods') + host_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00)), + ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + self.fake_phys_plugin._matching_hosts( + '[]', '[]', '3-3', + datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00), + None) + self.addCleanup(CONF.clear_override, 'randomize_host_selection', + group=plugin.RESOURCE_TYPE) + mock_shuffle.assert_called_once_with(['host1', 'host2', 'host3']) + + @mock.patch.object(random, "shuffle") + def test_random_matching_hosts_allocated_cleaning_time(self, mock_shuffle): + def host_allocation_get_all_by_values(**kwargs): + if kwargs['compute_host_id'] == 'host1': + return True + self.cfg.CONF.set_override('randomize_host_selection', True, + group=plugin.RESOURCE_TYPE) + self.cfg.CONF.set_override('cleaning_time', '5') + host_get = self.patch( + self.db_api, + 'reservable_host_get_all_by_queries') + host_get.return_value = [ + {'id': 'host1'}, + {'id': 'host2'}, + {'id': 'host3'}, + ] + host_get = self.patch( + self.db_api, + 'host_allocation_get_all_by_values') + host_get.side_effect = host_allocation_get_all_by_values + host_get = self.patch( + self.db_utils, + 'get_free_periods') + host_get.return_value = [ + (datetime.datetime(2013, 12, 19, 20, 00) + - datetime.timedelta(minutes=5), + datetime.datetime(2013, 12, 19, 21, 00) + + datetime.timedelta(minutes=5)) + ] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + self.fake_phys_plugin._matching_hosts( + '[]', '[]', '3-3', + datetime.datetime(2013, 12, 19, 20, 00), + datetime.datetime(2013, 12, 19, 21, 00), + None) + self.addCleanup(CONF.clear_override, 'randomize_host_selection', + group=plugin.RESOURCE_TYPE) + self.addCleanup(CONF.clear_override, 'cleaning_time') + mock_shuffle.assert_called_once_with(['host1', 'host2', 'host3']) + def test_matching_hosts_not_matching(self): host_get = self.patch( self.db_api, 'reservable_host_get_all_by_queries') host_get.return_value = [] + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False result = self.fake_phys_plugin._matching_hosts( '["=", "$memory_mb", "2048"]', '[]', '1-1', datetime.datetime(2013, 12, 19, 20, 00), - datetime.datetime(2013, 12, 19, 21, 00)) + datetime.datetime(2013, 12, 19, 21, 00), + None + ) self.assertEqual([], result) def test_check_params_with_valid_before_end(self): @@ -2206,35 +2975,120 @@ def test_check_params_without_before_end(self): } self.fake_phys_plugin._check_params(values) self.assertEqual(values['before_end'], 'default') + self.addCleanup(self.cfg.CONF.clear_override, 'before_end', + group='physical:host') + + def test_list_resource_properties(self): + self.db_list_resource_properties = self.patch( + self.db_api, 'resource_properties_list') + + # Expecting a list of (Reservation, Allocation) + self.db_list_resource_properties.return_value = [ + ('prop1', False, 'aaa', False), + ('prop1', False, 'bbb', False), + ('prop2', False, 'aaa', False), + ('prop2', False, 'aaa', False), + ('prop3', True, 'aaa', False) + ] + + expected = [ + {'property': 'prop1',}, + {'property': 'prop2',} + ] + + ret = self.fake_phys_plugin.list_resource_properties( + query={'detail': False}) + + # Sort returned value to use assertListEqual + ret.sort(key=lambda x: x['property']) + + self.assertListEqual(expected, ret) + self.db_list_resource_properties.assert_called_once_with( + 'physical:host') + + def test_list_resource_properties_with_detail(self): + self.db_list_resource_properties = self.patch( + self.db_api, 'resource_properties_list') + + # Expecting a list of (Reservation, Allocation) + self.db_list_resource_properties.return_value = [ + ('prop1', False, 'aaa', False), + ('prop1', False, 'bbb', False), + ('prop2', False, 'ccc', False), + ('prop3', True, 'aaa', False) + ] + + expected = [ + {'property': 'prop1', 'private': False, 'values': ['aaa', 'bbb'], 'is_unique': False}, + {'property': 'prop2', 'private': False, 'values': ['ccc'], 'is_unique': False} + ] + + ret = self.fake_phys_plugin.list_resource_properties( + query={'detail': True}) + + # Sort returned value to use assertListEqual + ret.sort(key=lambda x: x['property']) + + self.assertListEqual(expected, ret) + self.db_list_resource_properties.assert_called_once_with( + 'physical:host') + + def test_update_resource_property(self): + resource_property_values = { + 'resource_type': 'physical:host', + 'private': False} + + db_resource_property_update = self.patch( + self.db_api, 'resource_property_update') + + self.fake_phys_plugin.update_resource_property( + 'foo', resource_property_values) + db_resource_property_update.assert_called_once_with( + 'physical:host', 'foo', resource_property_values) + + def test_update_disabled_property_as_admin(self): + resource_property_values = {'disabled': 'true'} + host_get = self.patch(self.db_api, 'host_get') + host_get.return_value = {'id': 'host1', 'hypervisor_hostname': 'host1'} + host_update = self.patch(self.db_api, 'host_update') + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = True + + self.assertRaises(manager_exceptions.MissingParameter, + self.fake_phys_plugin.update_computehost, + 'host1', {'disabled': 'true', 'reservable': 'true'}) + + self.fake_phys_plugin.update_computehost( + 'host1', {'disabled': 'true', "disabled_reason": "maintenance"}) + host_update.assert_called_once_with('host1', {"disabled": True, "reservable": False, "disabled_reason": "maintenance"}) + + def test_update_disabled_property_as_user(self): + resource_property_values = {'disabled': 'true'} + host_get = self.patch(self.db_api, 'host_get') + host_get.return_value = {'id': 'host1', 'hypervisor_hostname': 'host1'} + host_update = self.patch(self.db_api, 'host_update') + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = False + self.fake_phys_plugin.update_computehost( + 'host1', {'disabled': 'true', "disabled_reason": "maintenance"}) + self.assertFalse(host_update.called) class PhysicalHostMonitorPluginTestCase(tests.TestCase): def setUp(self): super(PhysicalHostMonitorPluginTestCase, self).setUp() + self.cfg = cfg + self.cfg.CONF.set_override('enable_polling_monitor_dry_run', 'false', group='physical:host') self.patch(nova_client, 'Client') self.host_monitor_plugin = host_plugin.PhysicalHostMonitorPlugin() - def test_configuration(self): - # reset the singleton at first - host_plugin.PhysicalHostMonitorPlugin._instance = None - self.cfg = self.useFixture(conf_fixture.Config(CONF)) - self.cfg.config(os_admin_username='fake-user') - self.cfg.config(os_admin_password='fake-passwd') - self.cfg.config(os_admin_user_domain_name='fake-user-domain') - self.cfg.config(os_admin_project_name='fake-pj-name') - self.cfg.config(os_admin_project_domain_name='fake-pj-domain') - self.host_monitor_plugin = host_plugin.PhysicalHostMonitorPlugin() - self.assertEqual('fake-user', self.host_monitor_plugin.username) - self.assertEqual("fake-passwd", self.host_monitor_plugin.password) - self.assertEqual("fake-user-domain", - self.host_monitor_plugin.user_domain_name) - self.assertEqual("fake-pj-name", self.host_monitor_plugin.project_name) - self.assertEqual("fake-pj-domain", - self.host_monitor_plugin.project_domain_name) - def test_notification_callback_disabled_true(self): - failed_host = {'hypervisor_hostname': 'hypvsr1'} + failed_host = {'hypervisor_hostname': 'hypvsr1', 'id': '1', 'disabled': False} event_type = 'service.update' payload = { 'nova_object.namespace': 'nova', @@ -2257,15 +3111,12 @@ def test_notification_callback_disabled_true(self): host_get_all = self.patch(db_api, 'reservable_host_get_all_by_queries') host_get_all.return_value = [failed_host] - handle_failures = self.patch(self.host_monitor_plugin, - '_handle_failures') - handle_failures.return_value = {'rsrv-1': {'missing_resources': True}} + self.patch(db_api, 'host_update') result = self.host_monitor_plugin.notification_callback(event_type, payload) host_get_all.assert_called_once_with( ['hypervisor_hostname == ' + payload['nova_object.data']['host']]) - self.assertEqual({'rsrv-1': {'missing_resources': True}}, result) def test_notification_callback_no_failure(self): event_type = 'service.update' @@ -2289,15 +3140,12 @@ def test_notification_callback_no_failure(self): } host_get_all = self.patch(db_api, 'host_get_all_by_queries') host_get_all.return_value = [] - handle_failures = self.patch(self.host_monitor_plugin, - '_handle_failures') result = self.host_monitor_plugin.notification_callback(event_type, payload) host_get_all.assert_called_once_with( ['reservable == 0', 'hypervisor_hostname == ' + payload['nova_object.data']['host']]) - handle_failures.assert_not_called() self.assertEqual({}, result) def test_notification_callback_recover(self): @@ -2323,8 +3171,6 @@ def test_notification_callback_recover(self): } host_get_all = self.patch(db_api, 'host_get_all_by_queries') host_get_all.return_value = [recovered_host] - handle_failures = self.patch(self.host_monitor_plugin, - '_handle_failures') host_update = self.patch(db_api, 'host_update') result = self.host_monitor_plugin.notification_callback(event_type, @@ -2334,7 +3180,42 @@ def test_notification_callback_recover(self): 'hypervisor_hostname == ' + payload['nova_object.data']['host']]) host_update.assert_called_once_with(recovered_host['id'], {'reservable': True}) - handle_failures.assert_not_called() + self.assertEqual({}, result) + + def test_notification_callback_not_recover_with_blazar_disabled(self): + recovered_host = {'hypervisor_hostname': 'hypvsr1', 'id': 1, 'disabled': True} + event_type = 'service.update' + payload = { + 'nova_object.namespace': 'nova', + 'nova_object.name': 'ServiceStatusPayload', + 'nova_object.version': '1.1', + 'nova_object.data': { + 'host': 'compute-1', + 'disabled': False, + 'last_seen_up': '2012-10-29T13:42:05Z', + 'binary': 'nova-compute', + 'topic': 'compute', + 'disabled_reason': None, + 'report_count': 1, + 'forced_down': False, + 'version': 22, + 'availability_zone': None, + 'uuid': 'fa69c544-906b-4a6a-a9c6-c1f7a8078c73' + } + } + host_get_all = self.patch(db_api, 'host_get_all_by_queries') + host_get_all.return_value = [recovered_host] + host_update = self.patch(db_api, 'host_update') + is_admin = self.patch( + policy, 'enforce' + ) + is_admin.return_value = True + result = self.host_monitor_plugin.notification_callback(event_type, + payload) + host_get_all.assert_called_once_with( + ['reservable == 0', + 'hypervisor_hostname == ' + payload['nova_object.data']['host']]) + self.assertFalse(host_update.called) self.assertEqual({}, result) def test_poll_resource_failures_state_down(self): @@ -2350,14 +3231,20 @@ def test_poll_resource_failures_state_down(self): host_get_all = self.patch(db_api, 'host_get_all_by_filters') host_get_all.return_value = hosts + host_update = self.patch(db_api, 'host_update') hypervisors_list = self.patch( self.host_monitor_plugin.nova.hypervisors, 'list') hypervisors_list.return_value = [ - mock.MagicMock(id=1, state='down', status='enabled'), - mock.MagicMock(id=2, state='down', status='enabled')] + mock.MagicMock(id=1, state='down', status='enabled', hypervisor_hostname="hypvsr1"), + mock.MagicMock(id=2, state='down', status='enabled', hypervisor_hostname="hypvsr2")] - result = self.host_monitor_plugin._poll_resource_failures() + result = self.host_monitor_plugin.poll_resource_failures() self.assertEqual((hosts, []), result) + host_update.assert_has_calls( + [mock.call('1', {"last_error": "Hypervisor status is down and enabled"}), + mock.call('2', {'last_error': "Hypervisor status is down and enabled"})], + any_order=True) + def test_poll_resource_failures_status_disabled(self): hosts = [ @@ -2372,14 +3259,19 @@ def test_poll_resource_failures_status_disabled(self): host_get_all = self.patch(db_api, 'host_get_all_by_filters') host_get_all.return_value = hosts + host_update = self.patch(db_api, 'host_update') hypervisors_list = self.patch( self.host_monitor_plugin.nova.hypervisors, 'list') hypervisors_list.return_value = [ - mock.MagicMock(id=1, state='up', status='disabled'), - mock.MagicMock(id=2, state='up', status='disabled')] + mock.MagicMock(id=1, state='up', status='disabled', hypervisor_hostname="hypvsr1"), + mock.MagicMock(id=2, state='up', status='disabled', hypervisor_hostname="hypvsr2")] - result = self.host_monitor_plugin._poll_resource_failures() + result = self.host_monitor_plugin.poll_resource_failures() self.assertEqual((hosts, []), result) + host_update.assert_has_calls([ + mock.call('1', {'last_error': "Hypervisor status is up and disabled"}), + mock.call('2', {'last_error': "Hypervisor status is up and disabled"}) + ], any_order=True) def test_poll_resource_failures_nothing(self): hosts = [ @@ -2397,10 +3289,10 @@ def test_poll_resource_failures_nothing(self): hypervisors_list = self.patch( self.host_monitor_plugin.nova.hypervisors, 'list') hypervisors_list.return_value = [ - mock.MagicMock(id=1, state='up', status='enabled'), - mock.MagicMock(id=2, state='up', status='enabled')] + mock.MagicMock(id=1, state='up', status='enabled', hypervisor_hostname="hypvsr1"), + mock.MagicMock(id=2, state='up', status='enabled', hypervisor_hostname="hypvsr2")] - result = self.host_monitor_plugin._poll_resource_failures() + result = self.host_monitor_plugin.poll_resource_failures() self.assertEqual(([], []), result) def test_poll_resource_failures_recover(self): @@ -2419,50 +3311,148 @@ def test_poll_resource_failures_recover(self): hypervisors_list = self.patch( self.host_monitor_plugin.nova.hypervisors, 'list') hypervisors_list.return_value = [ - mock.MagicMock(id=1, state='up', status='enabled'), - mock.MagicMock(id=2, state='up', status='enabled')] + mock.MagicMock(id=1, state='up', status='enabled', hypervisor_hostname="hypvsr1"), + mock.MagicMock(id=2, state='up', status='enabled', hypervisor_hostname="hypvsr2")] - result = self.host_monitor_plugin._poll_resource_failures() + result = self.host_monitor_plugin.poll_resource_failures() self.assertEqual(([], hosts), result) - def test_handle_failures(self): - failed_hosts = [ - {'id': '1', - 'hypervisor_hostname': 'hypvsr1'} - ] - host_update = self.patch(db_api, 'host_update') - heal = self.patch(self.host_monitor_plugin, 'heal') - - self.host_monitor_plugin._handle_failures(failed_hosts) - host_update.assert_called_once_with(failed_hosts[0]['id'], - {'reservable': False}) - heal.assert_called_once() - def test_heal(self): failed_hosts = [ {'id': '1', 'hypervisor_hostname': 'hypvsr1'} ] - reservation_flags = { - 'rsrv-1': {'missing_resources': True} - } + reservation_flags = {} hosts_get = self.patch(db_api, 'unreservable_host_get_all_by_queries') hosts_get.return_value = failed_hosts get_healing_interval = self.patch(self.host_monitor_plugin, 'get_healing_interval') get_healing_interval.return_value = 60 - healing_handler = mock.Mock() - healing_handler.return_value = reservation_flags - self.host_monitor_plugin.healing_handlers = [healing_handler] start_date = datetime.datetime(2020, 1, 1, 12, 00) + dummy_reservation = { + 'id': 'rsrv-1', + 'resource_type': plugin.RESOURCE_TYPE, + 'lease_id': 'lease-1', + 'status': 'pending', + 'hypervisor_properties': [], + 'resource_properties': [], + 'resource_id': 'resource-1', + 'computehost_allocations': [{ + 'id': 'alloc-1', 'compute_host_id': failed_hosts[0]['id'], + 'reservation_id': 'rsrv-1' + }] + } + get_reservations = self.patch(db_utils, 'get_reservations_by_host_ids') + get_reservations.return_value = [dummy_reservation] + with mock.patch.object(datetime, 'datetime', mock.Mock(wraps=datetime.datetime)) as patched: patched.utcnow.return_value = start_date result = self.host_monitor_plugin.heal() - healing_handler.assert_called_once_with( - failed_hosts, start_date, - start_date + datetime.timedelta(minutes=60) - ) self.assertEqual(reservation_flags, result) + + def test_poll_resource_failures_aggregate_cleanup(self): + def fake_get_reservations_by_host_id(host_id): + host_reservations = { + 1 : {'reservation_status': status.reservation.ERROR, 'reservation_id': 1}, + 2 : {'reservation_status': status.reservation.ACTIVE, 'reservation_id': 2}, + # if the host has no reservations, it should be moved to freepool + 3: None + } + return host_reservations[host_id] + + # Create a list of hosts + host_in_errored_res = { + 'id': 1, + 'hypervisor_hostname': 'host-1' + } + host_in_active_res = { + 'id': 2, + 'hypervisor_hostname': 'host-2' + } + host_in_freepool = { + 'id': 3, + 'hypervisor_hostname': 'host-3' + } + + # Set up mock responses for the database and Nova + hosts_get_all = self.patch(db_api, 'host_get_all_by_filters') + hosts_get_all.return_value = [] + hosts_list = self.patch(db_api, 'host_list') + hosts_list.return_value = [ + host_in_errored_res, + host_in_active_res, + host_in_freepool, + ] + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_host_id') + get_reservations.side_effect = fake_get_reservations_by_host_id + + # Create mock aggregate objects + aggregate1 = mock.MagicMock(id=1, hosts=[host_in_errored_res["hypervisor_hostname"]]) + aggregate1.configure_mock(name="aggregate-1") + aggregate2 = mock.MagicMock(id=2, hosts=[host_in_active_res["hypervisor_hostname"]]) + aggregate2.configure_mock(name="aggregate-2") + freepool = mock.MagicMock(id=3, hosts=[host_in_freepool["hypervisor_hostname"]]) + freepool.configure_mock(name="freepool") + freepool_get = self.patch(nova.ReservationPool, 'get_aggregate_from_name_or_id') + freepool_get.return_value = freepool + hosts_in_agg = self.patch(nova.ReservationPool, 'get_computehosts') + hosts_in_agg.return_value = None + + list_aggregates = self.patch( + self.host_monitor_plugin.nova.aggregates, 'list' + ) + list_aggregates.return_value = [aggregate1, aggregate2, freepool] + failed_hosts, recovered_hosts = self.host_monitor_plugin.poll_resource_failures() + + self.assertEqual(failed_hosts, []) + self.assertEqual(recovered_hosts, []) + freepool.add_host.assert_has_calls([ + mock.call(host_in_errored_res["hypervisor_hostname"]), + ]) + aggregate1.remove_host.assert_called_with(host_in_errored_res["hypervisor_hostname"]) + aggregate1.delete.assert_called_with() + + def test_poll_resource_failures_agg_cleanup_dry_run(self): + def fake_get_reservations_by_host_id(host_id): + return {'reservation_status': status.reservation.ERROR, 'reservation_id': 1} + + self.cfg.CONF.set_override('enable_polling_monitor_dry_run', 'true', group='physical:host') + + # Create a list of hosts + host_in_errored_res = {'id': 1, 'hypervisor_hostname': 'host-1'} + + # Set up mock responses for the database and Nova + hosts_get_all = self.patch(db_api, 'host_get_all_by_filters') + hosts_get_all.return_value = [] + hosts_list = self.patch(db_api, 'host_list') + hosts_list.return_value = [host_in_errored_res,] + get_reservations = self.patch(db_utils, 'get_most_recent_reservation_info_by_host_id') + get_reservations.side_effect = fake_get_reservations_by_host_id + + # Create mock aggregate objects + aggregate1 = mock.MagicMock(id=1, hosts=[host_in_errored_res["hypervisor_hostname"]]) + aggregate1.configure_mock(name="aggregate-1") + freepool = mock.MagicMock(id=3, hosts=[]) + freepool.configure_mock(name="freepool") + freepool_get = self.patch(nova.ReservationPool, 'get_aggregate_from_name_or_id') + freepool_get.return_value = freepool + hosts_in_agg = self.patch(nova.ReservationPool, 'get_computehosts') + hosts_in_agg.return_value = None + + list_aggregates = self.patch( + self.host_monitor_plugin.nova.aggregates, 'list' + ) + list_aggregates.return_value = [aggregate1, freepool] + failed_hosts, recovered_hosts = self.host_monitor_plugin.poll_resource_failures() + + self.assertEqual(failed_hosts, []) + self.assertEqual(recovered_hosts, []) + self.assertFalse(freepool.add_host.called) + self.assertFalse(aggregate1.remove_host.called) + self.assertFalse(aggregate1.delete.called) + + self.addCleanup(self.cfg.CONF.clear_override, 'enable_polling_monitor_dry_run', + group='physical:host') diff --git a/blazar/tests/plugins/test_base.py b/blazar/tests/plugins/test_base.py new file mode 100644 index 000000000..0b6278830 --- /dev/null +++ b/blazar/tests/plugins/test_base.py @@ -0,0 +1,74 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from blazar.plugins import base +from blazar import tests + + +class BasePluginDummy(base.BasePlugin): + def get(self, resource_id): + pass + + def reserve_resource(self, reservation_id, values): + pass + + def list_allocations(self, query, detail=False): + pass + + def query_allocations(self, resource_id_list, lease_id=None, + reservation_id=None): + pass + + def allocation_candidates(self, lease_values): + pass + + def update_reservation(self, reservation_id, values): + pass + + def on_end(self, resource_id, lease=None): + pass + + def on_start(self, resource_id, lease=None): + pass + + +class BasePluginTestCase(tests.TestCase): + def setUp(self): + super(BasePluginTestCase, self).setUp() + self.plugin = BasePluginDummy() + + def test__is_project_allowed(self): + # No project restrictions + project_id = "0ac67a48-e65c-11eb-ba80-0242ac130004" + resource = {} + self.assertTrue(self.plugin.is_project_allowed(project_id, resource)) + + # Single project restriction + resource = { + "authorized_projects": "0ac67a48-e65c-11eb-ba80-0242ac130004" + } + project_id = "0ac67a48-e65c-11eb-ba80-0242ac130004" + self.assertTrue(self.plugin.is_project_allowed(project_id, resource)) + project_id = "6bd9356e-e65c-11eb-ba80-0242ac130004" + self.assertFalse(self.plugin.is_project_allowed(project_id, resource)) + + resource = { + "authorized_projects": "0ac67a48-e65c-11eb-ba80-0242ac130004," + "6bd9356e-e65c-11eb-ba80-0242ac130004" + } + project_id = "0ac67a48-e65c-11eb-ba80-0242ac130004" + self.assertTrue(self.plugin.is_project_allowed(project_id, resource)) + project_id = "6bd9356e-e65c-11eb-ba80-0242ac130004" + self.assertTrue(self.plugin.is_project_allowed(project_id, resource)) + project_id = "923cf8d0-e65c-11eb-ba80-0242ac130004" + self.assertFalse(self.plugin.is_project_allowed(project_id, resource)) diff --git a/blazar/tests/test_context.py b/blazar/tests/test_context.py index edaf68ae1..c7af1cd19 100644 --- a/blazar/tests/test_context.py +++ b/blazar/tests/test_context.py @@ -13,18 +13,39 @@ # See the License for the specific language governing permissions and # limitations under the License. +from oslo_config import cfg +from oslo_config import fixture as conf_fixture from oslo_utils.fixture import uuidsentinel from blazar import context from blazar import tests +CONF = cfg.CONF + class TestBlazarContext(tests.TestCase): + def setUp(self): + super(TestBlazarContext, self).setUp() + + self.cfg = self.useFixture(conf_fixture.Config(CONF)) + self.cfg.config(os_admin_username='fake-admin') + self.cfg.config(os_admin_user_domain_name='fake-admin-domain') + self.cfg.config(os_admin_project_name='fake-admin-project') + self.cfg.config(os_admin_project_domain_name='fake-admin-domain') + def test_to_dict(self): ctx = context.BlazarContext( user_id=111, project_id=222, request_id='req-679033b7-1755-4929-bf85-eb3bfaef7e0b') + + # NOTE(priteau): for compatibility with oslo.context<4.0.0 which + # returns a tenant key instead of project_id + ctx_dict = ctx.to_dict() + if 'tenant' in ctx_dict: + ctx_dict['project_id'] = ctx_dict['tenant'] + del ctx_dict['tenant'] + expected = { 'auth_token': None, 'domain': None, @@ -34,7 +55,6 @@ def test_to_dict(self): 'project': 222, 'project_domain': None, 'project_id': 222, - 'project_name': None, 'read_only': False, 'request_id': 'req-679033b7-1755-4929-bf85-eb3bfaef7e0b', 'resource_uuid': None, @@ -42,17 +62,10 @@ def test_to_dict(self): 'service_catalog': [], 'show_deleted': False, 'system_scope': None, - 'tenant': 222, 'user': 111, 'user_domain': None, - 'user_id': 111, - 'user_identity': '111 222 - - -', - 'user_name': None} - self.assertEqual(expected, ctx.to_dict()) - - def test_elevated_empty(self): - ctx = context.BlazarContext.elevated() - self.assertTrue(ctx.is_admin) + 'user_identity': '111 222 - - -'} + self.assertEqual(expected, ctx_dict) def test_service_catalog_default(self): ctxt = context.BlazarContext(user_id=uuidsentinel.user_id, @@ -69,14 +82,30 @@ def test_service_catalog_default(self): service_catalog=None) self.assertEqual([], ctxt.service_catalog) - def test_blazar_context_elevated(self): - user_context = context.BlazarContext( - user_id=uuidsentinel.user_id, - project_id=uuidsentinel.project_id, is_admin=False) - self.assertFalse(user_context.is_admin) + def test_admin(self): + ctx = context.admin() + self.assertEqual(ctx.user_name, 'fake-admin') + self.assertEqual(ctx.user_domain_name, 'fake-admin-domain') + self.assertEqual(ctx.project_name, 'fake-admin-project') + self.assertEqual(ctx.project_domain_name, 'fake-admin-domain') + self.assertEqual(ctx.is_admin, True) - admin_context = user_context.elevated() - self.assertFalse(user_context.is_admin) - self.assertTrue(admin_context.is_admin) - self.assertNotIn('admin', user_context.roles) - self.assertIn('admin', admin_context.roles) + def test_admin_nested(self): + """Test that admin properties take priority over current context.""" + request_id = 'req-679033b7-1755-4929-bf85-eb3bfaef7e0b' + service_catalog = ['foo'] + ctx = context.BlazarContext( + user_name='fake-user', user_domain_name='fake-user-domain', + project_name='fake-project', + project_domain_name='fake-user-domain', + service_catalog=service_catalog, request_id=request_id) + with ctx: + admin_ctx = context.admin() + self.assertEqual(admin_ctx.user_name, 'fake-admin') + self.assertEqual(admin_ctx.user_domain_name, 'fake-admin-domain') + self.assertEqual(admin_ctx.project_name, 'fake-admin-project') + self.assertEqual(admin_ctx.project_domain_name, + 'fake-admin-domain') + self.assertEqual(admin_ctx.is_admin, True) + self.assertEqual(admin_ctx.request_id, request_id) + self.assertEqual(admin_ctx.service_catalog, service_catalog) diff --git a/blazar/tests/test_policy.py b/blazar/tests/test_policy.py index f7da9356a..5ace0922f 100644 --- a/blazar/tests/test_policy.py +++ b/blazar/tests/test_policy.py @@ -15,6 +15,8 @@ """Test of Policy Engine For Blazar.""" +import unittest.mock as mock + from oslo_config import cfg from blazar import context @@ -35,10 +37,10 @@ def setUp(self): roles=['member']) def test_standardpolicy(self): - target_good = {'user_id': self.context.user_id, - 'project_id': self.context.project_id} - target_wrong = {'user_id': self.context.user_id, - 'project_id': 'bad_project'} + target_good = {'user': self.context.user_id, + 'project': self.context.project_id} + target_wrong = {'user': self.context.user_id, + 'project': 'bad_project'} action = "blazar:leases:get" self.assertTrue(policy.enforce(self.context, action, target_good)) @@ -46,21 +48,12 @@ def test_standardpolicy(self): target_wrong, False)) def test_adminpolicy(self): - target = {'user_id': self.context.user_id, - 'project_id': self.context.project_id} + target = {'user': self.context.user_id, + 'project': self.context.project_id} action = "blazar:oshosts:get" self.assertRaises(exceptions.PolicyNotAuthorized, policy.enforce, self.context, action, target) - def test_elevatedpolicy(self): - target = {'user_id': self.context.user_id, - 'project_id': self.context.project_id} - action = "blazar:oshosts:get" - self.assertRaises(exceptions.PolicyNotAuthorized, policy.enforce, - self.context, action, target) - elevated_context = self.context.elevated() - self.assertTrue(policy.enforce(elevated_context, action, target)) - def test_authorize(self): @policy.authorize('leases', 'get', ctx=self.context) @@ -74,3 +67,24 @@ def adminonly_method_with_action(self): self.assertTrue(user_method_with_action(self)) self.assertRaises(exceptions.PolicyNotAuthorized, adminonly_method_with_action, self) + + @mock.patch( + 'blazar.db.api.lease_get', + mock.MagicMock(return_value={'id': '1', 'user_id': 'alt_fake', + 'project_id': 'alt_fake'})) + @policy.authorize('leases', 'get', ctx=self.context) + def alt_user_get_lease(self, **kwargs): + return True + + self.assertRaises(exceptions.PolicyNotAuthorized, alt_user_get_lease, + self, lease_id='1') + + @mock.patch( + 'blazar.db.api.lease_get', + mock.MagicMock(return_value={'id': '1', 'user_id': 'fake', + 'project_id': 'fake'})) + @policy.authorize('leases', 'get', ctx=self.context) + def user_get_lease(self, **kwargs): + return True + + self.assertTrue(user_get_lease(self, lease_id='1')) \ No newline at end of file diff --git a/blazar/tests/test_status.py b/blazar/tests/test_status.py index b2351f198..e86f98634 100644 --- a/blazar/tests/test_status.py +++ b/blazar/tests/test_status.py @@ -242,6 +242,40 @@ def dummy_start_lease(*args, **kwargs): [call(self.lease_id, {'status': status.LeaseStatus.STARTING}), call(self.lease_id, {'status': status.LeaseStatus.ERROR})]) + def test_lease_status_func_allow_non_fatal_exception(self): + """Test when non-fatal exception is raised during lease transition. + + When this happens, the exception should still get raised, but the + lease should be transitioned to its original status (not ERROR). + """ + lease = { + 'status': status.LeaseStatus.PENDING + } + lease_get = self.patch(self.db_api, 'lease_get') + lease_get.return_value = lease + lease_update = self.patch(self.db_api, 'lease_update') + self.patch(self.status.LeaseStatus, 'is_valid_transition' + ).return_value = True + + class NonFatalException(Exception): + pass + + @self.status.LeaseStatus.lease_status( + transition=status.LeaseStatus.STARTING, + result_in=(status.LeaseStatus.ACTIVE,), + non_fatal_exceptions=[NonFatalException]) + def dummy_start_lease(*args, **kwargs): + raise NonFatalException + + self.assertRaises(NonFatalException, + dummy_start_lease, + lease_id=self.lease_id) + + lease_get.assert_called_once_with(self.lease_id) + lease_update.assert_has_calls( + [call(self.lease_id, {'status': status.LeaseStatus.STARTING}), + call(self.lease_id, {'status': status.LeaseStatus.PENDING})]) + def test_lease_status_mismatch_result_in(self): lease = { 'status': status.LeaseStatus.PENDING diff --git a/blazar/tests/utils/openstack/test_base.py b/blazar/tests/utils/openstack/test_base.py index 524bfa2c1..e137fa9c9 100644 --- a/blazar/tests/utils/openstack/test_base.py +++ b/blazar/tests/utils/openstack/test_base.py @@ -34,7 +34,7 @@ def test_url_for_good_v3(self): pass def test_url_for_good_v2_public(self): - service_catalog = ( + service_catalog = tests.FakeServiceCatalog( [{"endpoints": [{"adminURL": self.url % 'admin', "region": "RegionOne", "internalURL": self.url % 'internal', @@ -46,7 +46,7 @@ def test_url_for_good_v2_public(self): self.assertEqual(url, self.url % 'public') def test_url_for_good_v2_admin(self): - service_catalog = ( + service_catalog = tests.FakeServiceCatalog( [{"endpoints": [{"adminURL": self.url % 'admin', "region": "RegionOne", "internalURL": self.url % 'internal', @@ -59,7 +59,7 @@ def test_url_for_good_v2_admin(self): self.assertEqual(url, self.url % 'admin') def test_url_for_no_service(self): - service_catalog = ( + service_catalog = tests.FakeServiceCatalog( [{"endpoints": [{"adminURL": self.url % 'admin', "region": "RegionOne", "internalURL": self.url % 'internal', @@ -71,7 +71,7 @@ def test_url_for_no_service(self): service_catalog, self.service_type) def test_url_for_no_endpoints(self): - service_catalog = ( + service_catalog = tests.FakeServiceCatalog( [{"type": "fake_service", "name": "foo"}]) @@ -79,7 +79,7 @@ def test_url_for_no_endpoints(self): service_catalog, self.service_type) def test_url_for_good_v2_region(self): - service_catalog = ( + service_catalog = tests.FakeServiceCatalog( [{"endpoints": [{"adminURL": self.url % 'admin', "region": "RegionOne", "internalURL": self.url % 'internal', @@ -96,7 +96,7 @@ def test_url_for_good_v2_region(self): self.assertEqual(url, self.url_region_two % 'public') def test_url_for_no_endpoint_in_region(self): - service_catalog = ( + service_catalog = tests.FakeServiceCatalog( [{"endpoints": [{"adminURL": self.url % 'admin', "region": "RegionOne", "internalURL": self.url % 'internal', diff --git a/blazar/tests/utils/openstack/test_keystone.py b/blazar/tests/utils/openstack/test_keystone.py index b0ff0e70a..be291ec54 100644 --- a/blazar/tests/utils/openstack/test_keystone.py +++ b/blazar/tests/utils/openstack/test_keystone.py @@ -13,116 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from keystoneclient import client as keystone_client - -from blazar import context -from blazar import exceptions from blazar import tests -from blazar.utils.openstack import base -from blazar.utils.openstack import keystone class TestCKClient(tests.TestCase): + """TODO: Update test class. - def setUp(self): - super(TestCKClient, self).setUp() - - self.keystone = keystone - self.context = context - self.k_client = keystone_client - self.base = base - - self.ctx = self.patch(self.context, 'current') - self.client = self.patch(self.k_client, 'Client') - self.patch(self.base, 'url_for').return_value = 'http://fake.com/' - - self.version = '3' - self.username = 'fake_user' - self.token = 'fake_token' - self.password = 'fake_pass' - self.tenant_name = 'fake_project' - self.auth_url = 'fake_url' - self.trust_id = 'fake_trust' - - def test_client_from_kwargs(self): - - self.ctx.side_effect = RuntimeError - - self.keystone.BlazarKeystoneClient(version=self.version, - username=self.username, - password=self.password, - tenant_name=self.tenant_name, - trust_id=self.trust_id, - auth_url=self.auth_url) - - self.client.assert_called_once_with(version=self.version, - trust_id=self.trust_id, - username=self.username, - password=self.password, - auth_url=self.auth_url) - - def test_client_from_kwargs_and_ctx(self): - - self.keystone.BlazarKeystoneClient(version=self.version, - username=self.username, - password=self.password, - tenant_name=self.tenant_name, - auth_url=self.auth_url) - - self.client.assert_called_once_with(version=self.version, - tenant_name=self.tenant_name, - endpoint='http://fake.com/', - username=self.username, - password=self.password, - auth_url=self.auth_url, - global_request_id=self. - context.current(). - global_request_id) - - def test_client_from_ctx(self): - - self.keystone.BlazarKeystoneClient() - - self.client.assert_called_once_with( - version='3', - username=self.ctx().user_name, - token=self.ctx().auth_token, - tenant_name=self.ctx().project_name, - auth_url='http://fake.com/', - endpoint='http://fake.com/', - global_request_id=self.context.current().global_request_id) - - def test_complement_auth_url_supported_api_version(self): - bkc = self.keystone.BlazarKeystoneClient() - expected_url = self.auth_url + '/v3' - returned_url = bkc.complement_auth_url(auth_url=self.auth_url, - version='v3') - self.assertEqual(expected_url, returned_url) - - def test_complement_auth_url_unsupported_api_version(self): - bkc = self.keystone.BlazarKeystoneClient() - kwargs = {'auth_url': self.auth_url, - 'version': 2.0} - self.assertRaises(exceptions.UnsupportedAPIVersion, - bkc.complement_auth_url, - **kwargs) - - def test_complement_auth_url_valid_url(self): - bkc = self.keystone.BlazarKeystoneClient() - auth_url = self.auth_url + '/v3' - returned_url = bkc.complement_auth_url(auth_url=auth_url, - version='v3') - self.assertEqual(auth_url, returned_url) - - def test_complement_auth_url_invalid_url(self): - bkc = self.keystone.BlazarKeystoneClient() - auth_url = self.auth_url + '/v2.0' - kwargs = {'auth_url': auth_url, - 'version': 'v3'} - self.assertRaises(exceptions.UnsupportedAPIVersion, - bkc.complement_auth_url, - **kwargs) - - def test_getattr(self): - # TODO(n.s.): Will be done as soon as pypi package will be updated - pass + This test originally tested functionality implemented in the + third-party keystoneclient library, which is redundant. It should test + primarily the branching b/w user and non-user authentication params, as + that is the main function this wrapper serves. + """ diff --git a/blazar/tests/utils/openstack/test_nova.py b/blazar/tests/utils/openstack/test_nova.py index a36641410..5b57e6469 100644 --- a/blazar/tests/utils/openstack/test_nova.py +++ b/blazar/tests/utils/openstack/test_nova.py @@ -21,6 +21,7 @@ from novaclient import exceptions as nova_exceptions from novaclient.v2 import availability_zones from novaclient.v2 import hypervisors +from novaclient.v2 import servers from oslo_config import cfg from oslo_config import fixture @@ -40,63 +41,17 @@ def setUp(self): self.nova = nova self.context = context - self.n_client = nova_client self.base = base self.ctx = self.patch(self.context, 'current') - self.client = self.patch(self.n_client, 'Client') + self.client = self.patch(nova_client, 'Client') self.auth = self.patch(token_endpoint, 'Token') self.session = self.patch(session, 'Session') self.url = 'http://fake.com/' self.patch(self.base, 'url_for').return_value = self.url self.version = '2' - - def test_client_from_kwargs(self): - self.ctx.side_effect = RuntimeError - endpoint = 'fake_endpoint' - username = 'blazar_admin' - password = 'blazar_password' - user_domain = 'User_Domain' - project_name = 'admin' - project_domain = 'Project_Domain' - auth_url = "%s://%s:%s" % (CONF.os_auth_protocol, - CONF.os_auth_host, - CONF.os_auth_port) - if CONF.os_auth_prefix: - auth_url += "/%s" % CONF.os_auth_prefix - - kwargs = {'version': self.version, - 'endpoint_override': endpoint, - 'username': username, - 'password': password, - 'user_domain_name': user_domain, - 'project_name': project_name, - 'project_domain_name': project_domain} - - self.nova.BlazarNovaClient(**kwargs) - - self.client.assert_called_once_with(version=self.version, - username=username, - password=password, - user_domain_name=user_domain, - project_name=project_name, - project_domain_name=project_domain, - auth_url=auth_url, - endpoint_override=endpoint) - - def test_client_from_ctx(self): - kwargs = {'version': self.version} - - self.nova.BlazarNovaClient(**kwargs) - - self.auth.assert_called_once_with(self.url, - self.ctx().auth_token) - self.session.assert_called_once_with(auth=self.auth.return_value) - self.client.assert_called_once_with(version=self.version, - endpoint_override=self.url, - session=self.session.return_value, - global_request_id=mock.ANY) + self.endpoint_type = 'internalURL' def test_getattr(self): # TODO(n.s.): Will be done as soon as pypi package will be updated @@ -145,6 +100,9 @@ def setUp(self): self.patch(self.nova.aggregates, 'set_metadata') self.patch(self.nova.aggregates, 'remove_host') + self.patch(self.nova, 'BlazarNovaClient') + + self.servers = self.patch(nova, "ServerManager").return_value self.patch(base, 'url_for').return_value = 'http://foo.bar' self.pool = nova.ReservationPool() @@ -152,13 +110,6 @@ def setUp(self): self.p_name = self.patch(self.pool, '_generate_aggregate_name') self.p_name.return_value = self.pool_name - def test_configuration(self): - self.assertEqual("fake-user", self.pool.username) - self.assertEqual("fake-passwd", self.pool.password) - self.assertEqual("fake-user-domain", self.pool.user_domain_name) - self.assertEqual("fake-pj-name", self.pool.project_name) - self.assertEqual("fake-pj-domain", self.pool.project_domain_name) - def _patch_get_aggregate_from_name_or_id(self): def get_fake_aggregate(*args): if self.freepool_name in args or self.fake_freepool.id in args: @@ -201,7 +152,7 @@ def test_create(self): az_name = self.blazar_az_prefix + self.pool_name - agg = self.pool.create(az=az_name) + agg = self.pool.create(az=az_name, project_id=self.project_id) self.assertEqual(agg, self.fake_aggregate) @@ -216,7 +167,7 @@ def test_create_no_az(self): self.patch(self.nova.aggregates, 'create').return_value = ( self.fake_aggregate) - self.pool.create() + self.pool.create(project_id=self.project_id) self.nova.aggregates.create.assert_called_once_with(self.pool_name, None) @@ -284,12 +235,15 @@ def test_get(self): def test_add_computehost(self): self._patch_get_aggregate_from_name_or_id() + terminate_preemptibles = self.patch( + self.pool, 'terminate_preemptibles') self.pool.add_computehost('pool', 'host3') check0 = self.nova.aggregates.add_host check0.assert_any_call(self.fake_aggregate.id, 'host3') check1 = self.nova.aggregates.remove_host check1.assert_any_call(self.fake_freepool.id, 'host3') + terminate_preemptibles.assert_called_with('host3') def test_add_computehost_with_host_id(self): # NOTE(sbauza): Freepool.hosts only contains names of hosts, not UUIDs @@ -341,6 +295,8 @@ def test_add_computehost_to_freepool(self): def test_add_computehost_revert(self): self._patch_get_aggregate_from_name_or_id() self.fake_freepool.hosts = ['host1', 'host2'] + terminate_preemptibles = self.patch( + self.pool, 'terminate_preemptibles') self.assertRaises(manager_exceptions.HostNotInFreePool, self.pool.add_computehost, 'pool', ['host1', 'host2', 'host3']) @@ -355,6 +311,8 @@ def test_add_computehost_revert(self): mock.call(self.fake_freepool.id, 'host2'), mock.call(self.fake_aggregate.id, 'host1'), mock.call(self.fake_aggregate.id, 'host2')]) + terminate_preemptibles.assert_has_calls([mock.call('host1'), + mock.call('host2')]) def test_remove_computehost_from_freepool(self): self._patch_get_aggregate_from_name_or_id() @@ -458,7 +416,7 @@ class FakeHost(object): local_gb = 10 servers = ['server1', 'server2'] - service = {'host': 'fake_name'} + service = {'host': 'fake_service'} @classmethod def get(cls, host): @@ -475,7 +433,8 @@ def get(cls, host): def search(cls, host, servers=False): if host == 'multiple': return [cls.FakeHost, cls.FakeHost] - if host == cls.FakeHost.service['host']: + # nova matches using LIKE hypervisor_hostname + if host in cls.FakeHost.hypervisor_hostname: return [cls.FakeHost] else: raise nova_exceptions.NotFound(404) @@ -494,12 +453,23 @@ def expected(cls): 'local_gb': cls.FakeHost.local_gb} +class FakeNovaServers(object): + + @classmethod + def list(cls, search_opts): + host = FakeNovaHypervisors.FakeHost + node = search_opts.get("node") + if node == host.hypervisor_hostname: + return host.servers + return [] + + class FakeAvailabilityZones(object): class FakeAZ1(object): zoneName = 'fake_az1' hosts = { - "fake_name": { + "fake_service": { "nova-compute": {} }, } @@ -507,7 +477,7 @@ class FakeAZ1(object): class FakeAZ2(object): zoneName = 'fake_az2' hosts = { - "fake_name": { + "fake_service": { "nova-conductor": {}, "nova-scheduler": {} }, @@ -536,6 +506,9 @@ def setUp(self): availability_zones.AvailabilityZoneManager, 'list') self.availability_zones.side_effect = FakeAvailabilityZones.list + self.servers_list = self.patch(servers.ServerManager, 'list') + self.servers_list.side_effect = FakeNovaServers.list + def test_get_host_details_with_host_id(self): host = self.inventory.get_host_details('1') expected = FakeNovaHypervisors.expected() @@ -578,24 +551,48 @@ def test_get_host_details_without_az(self): self.assertEqual(expected, host) def test_get_servers_per_host(self): - servers = self.inventory.get_servers_per_host('fake_name') + servers = self.inventory.get_servers_per_host('fake_name.openstack.org') self.assertEqual(FakeNovaHypervisors.FakeHost.servers, servers) + # hypervisors_search with servers and filtered on hypervisor_hostname + # does not work, assert we use servers_list instead. + self.hypervisors_search.assert_called_once_with('fake_name.openstack.org') + self.servers_list.assert_called_once_with( + search_opts={"node": "fake_name.openstack.org", "all_tenants": 1}) def test_get_servers_per_host_with_host_id(self): self.assertRaises(manager_exceptions.HostNotFound, self.inventory.get_servers_per_host, '1') + self.hypervisors_search.assert_called_once_with('1') + self.servers_list.assert_not_called() def test_get_servers_per_host_with_host_not_found(self): self.assertRaises(manager_exceptions.HostNotFound, self.inventory.get_servers_per_host, 'wrong_name') + self.hypervisors_search.assert_called_once_with('wrong_name') + self.servers_list.assert_not_called() def test_get_servers_per_host_having_multiple_results(self): self.assertRaises(manager_exceptions.MultipleHostsFound, self.inventory.get_servers_per_host, 'multiple') + self.hypervisors_search.assert_called_once_with('multiple') + self.servers_list.assert_not_called() def test_get_servers_per_host_with_host_having_no_servers(self): - host_with_zero_servers = FakeNovaHypervisors.FakeHost - # NOTE(sbauza): We need to simulate a host having zero servers - del host_with_zero_servers.servers - servers = self.inventory.get_servers_per_host('fake_name') + self.servers_list.side_effect = None + self.servers_list.return_value = [] + servers = self.inventory.get_servers_per_host('fake_name.openstack.org') self.assertEqual(None, servers) + self.hypervisors_search.assert_called_once_with('fake_name.openstack.org') + self.servers_list.assert_called_once_with( + search_opts={ + "node": "fake_name.openstack.org", + "all_tenants": 1}) + + def test_get_servers_per_host_filters_by_hypervisor_hostname(self): + # Given the service host name ('fake_name'), servers are still looked + # up by the resolved hypervisor_hostname (node), never the service host. + servers = self.inventory.get_servers_per_host('fake_name') + self.assertEqual(FakeNovaHypervisors.FakeHost.servers, servers) + self.hypervisors_search.assert_called_once_with('fake_name') + self.servers_list.assert_called_once_with( + search_opts={"node": "fake_name.openstack.org", "all_tenants": 1}) diff --git a/blazar/tests/utils/openstack/test_placement.py b/blazar/tests/utils/openstack/test_placement.py index d32fc3979..145e4bd1b 100644 --- a/blazar/tests/utils/openstack/test_placement.py +++ b/blazar/tests/utils/openstack/test_placement.py @@ -45,7 +45,7 @@ def test_client_auth_url(self): def _add_default_kwargs(self, kwargs): kwargs['endpoint_filter'] = {'service_type': 'placement', - 'interface': 'public', + 'interface': 'internal', 'region_name': 'region_foo'} kwargs['headers'] = {'accept': 'application/json'} kwargs['microversion'] = PLACEMENT_MICROVERSION @@ -150,6 +150,31 @@ def test_get_resource_provider_fail(self, kss_req): exceptions.ResourceProviderRetrievalFailed, self.client.get_resource_provider, rp_name) + @mock.patch('keystoneauth1.session.Session.request') + def test_get_traits(self, kss_req): + rp_uuid = uuidutils.generate_uuid() + + mock_json_data = { + "resource_provider_generation": 1, + "traits": [ + "CUSTOM_HW_FPGA_CLASS1", + "CUSTOM_HW_FPGA_CLASS3" + ] + } + + kss_req.return_value = fake_requests.FakeResponse( + 200, content=jsonutils.dump_as_bytes(mock_json_data)) + + result = self.client.get_traits(rp_uuid) + + expected_url = '/resource_providers/%s/traits' % rp_uuid + self._assert_keystone_called_once(kss_req, expected_url, 'GET') + expected = [ + "CUSTOM_HW_FPGA_CLASS1", + "CUSTOM_HW_FPGA_CLASS3" + ] + self.assertEqual(expected, result) + @mock.patch('keystoneauth1.session.Session.request') def test_create_resource_provider(self, kss_req): rp_name = 'Blazar' diff --git a/blazar/tests/utils/test_trusts.py b/blazar/tests/utils/test_trusts.py index a6084619b..049d8c25e 100644 --- a/blazar/tests/utils/test_trusts.py +++ b/blazar/tests/utils/test_trusts.py @@ -19,6 +19,7 @@ from oslo_config import fixture as conf_fixture from blazar import context +from blazar import policy from blazar import tests from blazar.utils.openstack import base from blazar.utils.openstack import keystone @@ -60,28 +61,52 @@ def test_create_ctx_from_trust(self): self.cfg.config(os_admin_username='admin') ctx = self.trusts.create_ctx_from_trust('1') fake_ctx_dict = { - 'auth_token': self.client().auth_token, + 'auth_token': self.client().session.get_token(), 'domain': None, 'global_request_id': self.context.current().global_request_id, - 'is_admin': False, 'is_admin_project': True, - 'project': self.client().tenant_id, + 'project': self.client().session.get_project_id(), 'project_domain': None, - 'project_id': self.client().tenant_id, - 'project_name': 'admin', 'read_only': False, 'request_id': ctx.request_id, 'resource_uuid': None, - 'roles': [], 'service_catalog': ctx.service_catalog, 'show_deleted': False, 'system_scope': None, - 'tenant': self.client().tenant_id, 'user': None, - 'user_domain': None, - 'user_id': None} + 'user_domain': None} self.assertDictContainsSubset(fake_ctx_dict, ctx.to_dict()) + def test_create_ctx_from_trust_preserves_admin(self): + """Regression: trust context must preserve admin roles. + + When an admin creates a lease, the manager wraps execution in + create_ctx_from_trust. The resulting context must carry roles + from the trust's auth_ref so that _is_admin() returns True + in the host plugin, allowing admins to reserve unreservable hosts. + """ + self.cfg.config(os_admin_project_name='admin') + self.cfg.config(os_admin_username='admin') + + # Set up the trust's auth_ref to return admin roles + mock_auth_ref = mock.MagicMock() + mock_auth_ref.role_names = ['admin', 'member'] + mock_auth_ref.service_catalog = [] + self.client().session.auth.get_auth_ref.return_value = mock_auth_ref + + trust_ctx = self.trusts.create_ctx_from_trust('1') + + # Verify roles passed through + self.assertIn('admin', trust_ctx.roles) + self.assertIn('member', trust_ctx.roles) + + # Verify policy.enforce recognizes this context as admin + policy.init() + self.assertTrue( + policy.enforce(trust_ctx, 'admin', {}, do_raise=False) + ) + + def test_use_trust_auth_dict(self): def to_wrap(self, arg_to_update): return arg_to_update diff --git a/blazar/utils/openstack/base.py b/blazar/utils/openstack/base.py index 4a76460ff..26c833421 100644 --- a/blazar/utils/openstack/base.py +++ b/blazar/utils/openstack/base.py @@ -13,10 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +from keystoneauth1.access import create as create_access_info +from keystoneauth1.identity import access +from keystoneauth1.identity import v3 +from keystoneauth1 import session +from keystoneclient import client as keystone_client import netaddr +from oslo_config import cfg +from blazar import context from blazar.manager import exceptions +CONF = cfg.CONF + + +CONF = cfg.CONF + def get_os_auth_host(conf): """Description @@ -29,6 +41,91 @@ def get_os_auth_host(conf): return os_auth_host +def client_kwargs(**_kwargs): + kwargs = _kwargs.copy() + + ctx = kwargs.pop('ctx', None) + username = kwargs.pop('username', + CONF.os_admin_username) + password = kwargs.pop('password', + CONF.os_admin_password) + project_name = kwargs.pop('project_name', + CONF.os_admin_project_name) + user_domain_name = kwargs.pop('user_domain_name', + CONF.os_admin_user_domain_name) + project_domain_name = kwargs.pop('project_domain_name', + CONF.os_admin_project_domain_name) + trust_id = kwargs.pop('trust_id', None) + auth_url = kwargs.pop('auth_url', None) + region_name = kwargs.pop('region_name', CONF.os_region_name) + if ctx is None: + try: + ctx = context.current() + except RuntimeError: + pass + if ctx is not None: + kwargs.setdefault('global_request_id', ctx.global_request_id) + + if auth_url is None: + auth_url = "%s://%s:%s/%s/%s" % (CONF.os_auth_protocol, + get_os_auth_host(CONF), + CONF.os_auth_port, + CONF.os_auth_prefix, + CONF.os_auth_version) + + auth_kwargs = dict( + auth_url=auth_url, + username=username, + password=password, + user_domain_name=user_domain_name, + project_domain_name=project_domain_name + ) + + if trust_id is not None: + auth_kwargs.update(trust_id=trust_id) + else: + auth_kwargs.update(project_name=project_name) + + auth = v3.Password(**auth_kwargs) + sess = session.Session(auth=auth) + + kwargs.setdefault('session', sess) + kwargs.setdefault('region_name', region_name) + return kwargs + + +def client_user_kwargs(**_kwargs): + kwargs = _kwargs.copy() + + auth_url = kwargs.pop('auth_url', None) + region_name = kwargs.pop('region_name', CONF.os_region_name) + + if auth_url is None: + auth_url = "%s://%s:%s/%s/%s" % (CONF.os_auth_protocol, + get_os_auth_host(CONF), + CONF.os_auth_port, + CONF.os_auth_prefix, + CONF.os_auth_version) + + # Pass the auth token present on the context directly on to the + # next service; this effectively proxies the user's token they used + # to authenticate to Blazar, and prevents having to re-authenticate + # (which has issues for certain auth types, such as application creds) + ctx = context.current() + admin_ks_client = keystone_client.Client( + version='3', + **client_kwargs(**_kwargs) + ) + data = admin_ks_client.tokens.get_token_data(ctx.auth_token) + access_info = create_access_info(body=data, auth_token=ctx.auth_token) + auth = access.AccessInfoPlugin(access_info, auth_url=auth_url) + sess = session.Session(auth=auth) + + kwargs.setdefault('session', sess) + kwargs.setdefault('region_name', region_name) + return kwargs + + def url_for(service_catalog, service_type, admin=False, endpoint_interface=None, os_region_name=None): @@ -39,9 +136,15 @@ def url_for(service_catalog, service_type, admin=False, service_type - OpenStack service type specification """ if not endpoint_interface: - endpoint_interface = 'public' - if admin: - endpoint_interface = 'admin' + if service_type == 'identity': + endpoint_interface = CONF.endpoint_type + elif service_type == 'compute': + endpoint_interface = CONF.nova.endpoint_type + else: + endpoint_interface = 'public' + + if not isinstance(service_catalog, list): + service_catalog = service_catalog.normalize_catalog() service = None for srv in service_catalog: diff --git a/blazar/utils/openstack/exceptions.py b/blazar/utils/openstack/exceptions.py index 46eb54f65..e87c3c3c3 100644 --- a/blazar/utils/openstack/exceptions.py +++ b/blazar/utils/openstack/exceptions.py @@ -66,3 +66,25 @@ class FloatingIPSubnetNotFound(exceptions.NotFound): class NeutronUsesFloatingIP(exceptions.InvalidInput): msg_fmt = _("The floating IP %(floatingip)s is used in allocation_pools " "or gateway_ip in subnet %(subnet)s .") + + +class TraitRetrievalFailed(exceptions.BlazarException): + msg_fmt = _("Failed to get trait %(trait)s") + + +class TraitCreationFailed(exceptions.BlazarException): + msg_fmt = _("Failed to create trait '%(trait)s'") + + +class TraitDeletionFailed(exceptions.BlazarException): + msg_fmt = _("Failed to delete trait '%(trait)s'") + + +class TraitAssociationFailed(exceptions.BlazarException): + msg_fmt = _("Failed to associate traits '%(traits)s' with resource " + "provider %(uuid)s") + + +class TraitDissociationFailed(exceptions.BlazarException): + msg_fmt = _("Failed to dissociate traits '%(traits)s' with resource " + "provider %(uuid)s") diff --git a/blazar/utils/openstack/heat.py b/blazar/utils/openstack/heat.py new file mode 100644 index 000000000..decc9f9de --- /dev/null +++ b/blazar/utils/openstack/heat.py @@ -0,0 +1,62 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from heatclient import client as heat_client +from keystoneauth1 import session +from keystoneauth1 import token_endpoint +from oslo_config import cfg +from oslo_log import log as logging + +from blazar import context +from blazar.utils.openstack import base + + +heat_opts = [ + cfg.StrOpt( + 'heat_api_version', + default='1', + deprecated_group='DEFAULT', + help='Heat API version'), + cfg.StrOpt( + 'orchestration_service', + default='orchestration', + deprecated_group='DEFAULT', + help='Heat name in keystone') +] + +CONF = cfg.CONF +CONF.register_opts(heat_opts, group='heat') +CONF.import_opt('identity_service', 'blazar.utils.openstack.keystone') +LOG = logging.getLogger(__name__) + + +class BlazarHeatClient(object): + + def __init__(self, ctx=None): + + if ctx is None: + ctx = context.current() + + endpoint_override = base.url_for( + ctx.service_catalog, + CONF.heat.orchestration_service, + os_region_name=CONF.os_region_name) + + auth = token_endpoint.Token(endpoint_override, ctx.auth_token) + sess = session.Session(auth=auth) + + self.heat = heat_client.Client( + CONF.heat.heat_api_version, session=sess) + + def __getattr_(self, name): + return getattr(self.heat, name) diff --git a/blazar/utils/openstack/ironic.py b/blazar/utils/openstack/ironic.py new file mode 100644 index 000000000..c340400b7 --- /dev/null +++ b/blazar/utils/openstack/ironic.py @@ -0,0 +1,47 @@ +# Copyright (c) 2020 University of Chicago +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ironicclient import client as ironic_client +from oslo_config import cfg + +from blazar.utils.openstack import base + +ironic_opts = [ + cfg.StrOpt( + 'ironic_api_version', + default='1', + help='Ironic API version'), + cfg.StrOpt( + 'ironic_api_microversion', + default='1.31', + help='Ironic API microversion') +] + +CONF = cfg.CONF +CONF.register_opts(ironic_opts, group='ironic') + + +class BlazarIronicClient(object): + """Client class for Ironic service.""" + + def __init__(self, **kwargs): + client_kwargs = base.client_kwargs(**kwargs) + client_kwargs.setdefault('os_ironic_api_version', + CONF.ironic.ironic_api_microversion) + self.ironic = ironic_client.Client( + CONF.ironic.ironic_api_version, **client_kwargs) + + def __getattr__(self, attr): + return getattr(self.ironic, attr) diff --git a/blazar/utils/openstack/keystone.py b/blazar/utils/openstack/keystone.py index 5ada2ac7a..a8a8a797f 100644 --- a/blazar/utils/openstack/keystone.py +++ b/blazar/utils/openstack/keystone.py @@ -13,15 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re - from keystoneclient import client as keystone_client from keystoneclient import exceptions as keystone_exception from oslo_config import cfg -from blazar import context -from blazar import exceptions -from blazar.manager import exceptions as manager_exceptions from blazar.utils.openstack import base @@ -42,6 +37,12 @@ ] keystone_opts = [ + cfg.StrOpt('endpoint_type', + default='internal', + choices=['public', 'admin', 'internal'], + help='Type of the keystone endpoint to use. This endpoint will ' + 'be looked up in the keystone catalog and should be one ' + 'of public, internal or admin.'), cfg.StrOpt('keystone_client_version', default='3', help='Keystoneclient version'), @@ -53,121 +54,16 @@ class BlazarKeystoneClient(object): - def __init__(self, **kwargs): - """Description - - Return Keystone client for defined in 'identity_service' conf. - NOTE: We will use tenant_name until we start using keystone V3 - client for all our needs. - - :param version: service client version which we will use - :type version: str - - :param username: username - :type username: str - - :param password: password - :type password: str - - :param tenant_name: tenant_name - :type tenant_name: str - - :param auth_url: auth_url - :type auth_url: str - - :param ctx: blazar context object - :type ctx: dict - - :param auth_url: keystone auth url - :type auth_url: string - - :param endpoint: keystone management (endpoint) url - :type endpoint: string - - :param trust_id: keystone trust ID - :type trust_id: string - - :param token: user token to use for authentication - :type token: string - """ - - ctx = kwargs.pop('ctx', None) - if ctx is None: - try: - ctx = context.current() - except RuntimeError: - pass - - kwargs.setdefault('version', cfg.CONF.keystone_client_version) - if ctx is not None: - kwargs.setdefault('username', ctx.user_name) - kwargs.setdefault('tenant_name', ctx.project_name) - kwargs.setdefault('global_request_id', ctx.global_request_id) - if not kwargs.get('auth_url'): - kwargs['auth_url'] = base.url_for( - ctx.service_catalog, CONF.identity_service, - os_region_name=CONF.os_region_name) - if not kwargs.get('trust_id'): - try: - kwargs.setdefault('endpoint', base.url_for( - ctx.service_catalog, CONF.identity_service, - endpoint_interface='admin', - os_region_name=CONF.os_region_name)) - except AttributeError: - raise manager_exceptions.NoManagementUrl() - if not kwargs.get('password'): - kwargs.setdefault('token', ctx.auth_token) - - # NOTE(dbelova): we need this checking to support current - # keystoneclient: token can only be scoped now to either - # a trust or project, not both. - if kwargs.get('trust_id') and kwargs.get('tenant_name'): - kwargs.pop('tenant_name') - - try: - # NOTE(n.s.): we shall remove this try: except: clause when - # https://review.openstack.org/#/c/66494/ will be merged - self.keystone = keystone_client.Client(**kwargs) - self.keystone.session.auth = self.keystone - auth_url = self.complement_auth_url(kwargs.get('auth_url', None), - kwargs.get('version', None)) - self.keystone.authenticate(auth_url=auth_url) - except AttributeError: - raise manager_exceptions.WrongClientVersion() - - self.exceptions = keystone_exception - - def complement_auth_url(self, auth_url, version): - """Return auth_url with api version. - - This method checks whether auth_url contains api version info. - If api version info is not contained in auth_url, this method - complements version info to auth_url. This method only support - complementing auth_url to v3 api url since we use keystone v3 - api to treat trusts. - """ - - # Create api version from major number of keystoneclient version. - # keystoneclient version can take forms of "3", "v3" or "3.0" and - # so this method convert them to form of "v3" for keystone api - # version. - api_version = version - if isinstance(api_version, str): - api_version = api_version.lstrip('v') - - api_version = int(float(api_version)) - - if api_version != 3: - raise exceptions.UnsupportedAPIVersion(version=api_version) - - if re.search(r'/v2.0/{,1}$', auth_url) is not None: - raise exceptions.UnsupportedAPIVersion(version='v2.0') - elif re.search(r'/v3/{,1}$', auth_url) is None: - complemented_url = auth_url.rstrip('/') + '/v' + str(api_version) + def __init__(self, as_user=False, **kwargs): + """Return Keystone client for defined in 'identity_service' conf.""" + if as_user: + client_kwargs = base.client_user_kwargs(**kwargs) else: - return auth_url + client_kwargs = base.client_kwargs(**kwargs) - return complemented_url + client_kwargs.setdefault('version', cfg.CONF.keystone_client_version) + self.keystone = keystone_client.Client(**client_kwargs) + self.exceptions = keystone_exception def __getattr__(self, name): func = getattr(self.keystone, name) diff --git a/blazar/utils/openstack/manila.py b/blazar/utils/openstack/manila.py new file mode 100644 index 000000000..7fac02e45 --- /dev/null +++ b/blazar/utils/openstack/manila.py @@ -0,0 +1,51 @@ +# Copyright (c) 2020 University of Chicago +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from blazar.utils.openstack import base +from oslo_log import log as logging +from manilaclient import client as manila_client + + +manila_opts = [ + cfg.StrOpt( + 'manila_api_version', + default='2', + help='Manila API version'), + cfg.StrOpt( + 'manila_api_microversion', + default='2.69', + help='Manila API microversion') +] + +CONF = cfg.CONF +CONF.register_opts(manila_opts, group='manila') + +LOG = logging.getLogger(__name__) + + +class BlazarManilaClient(object): + """Client class for Manila service.""" + + def __init__(self, **kwargs): + client_kwargs = base.client_kwargs(**kwargs) + client_kwargs.setdefault('os_manila_api_version', + CONF.manila.manila_api_microversion) + self.manila = manila_client.Client( + CONF.manila.manila_api_version, **client_kwargs) + + def __getattr__(self, attr): + return getattr(self.manila, attr) diff --git a/blazar/utils/openstack/neutron.py b/blazar/utils/openstack/neutron.py index 2e662b6ad..f2db6fe11 100644 --- a/blazar/utils/openstack/neutron.py +++ b/blazar/utils/openstack/neutron.py @@ -13,20 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from keystoneauth1.identity import v3 -from keystoneauth1 import session import netaddr from neutronclient.common import exceptions as neutron_exceptions from neutronclient.v2_0 import client as neutron_client -from oslo_config import cfg from oslo_log import log as logging -from blazar import context from blazar.utils.openstack import base from blazar.utils.openstack import exceptions +from blazar.manager import exceptions as manager_ex -CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -34,46 +30,17 @@ class BlazarNeutronClient(object): """Client class for Neutron service.""" def __init__(self, **kwargs): - ctx = kwargs.pop('ctx', None) - username = kwargs.pop('username', - CONF.os_admin_username) - password = kwargs.pop('password', - CONF.os_admin_password) - project_name = kwargs.pop('project_name', - CONF.os_admin_project_name) - user_domain_name = kwargs.pop('user_domain_name', - CONF.os_admin_user_domain_name) - project_domain_name = kwargs.pop('project_domain_name', - CONF.os_admin_project_domain_name) - auth_url = kwargs.pop('auth_url', None) - region_name = kwargs.pop('region_name', CONF.os_region_name) - if ctx is None: - try: - ctx = context.current() - except RuntimeError: - pass - if ctx is not None: - kwargs.setdefault('global_request_id', ctx.global_request_id) - - if auth_url is None: - auth_url = "%s://%s:%s" % (CONF.os_auth_protocol, - base.get_os_auth_host(CONF), - CONF.os_auth_port) - if CONF.os_auth_prefix: - auth_url += "/%s" % CONF.os_auth_prefix - if CONF.os_auth_version: - auth_url += "/%s" % CONF.os_auth_version - - auth = v3.Password(auth_url=auth_url, - username=username, - password=password, - project_name=project_name, - user_domain_name=user_domain_name, - project_domain_name=project_domain_name) - sess = session.Session(auth=auth) - kwargs.setdefault('session', sess) - kwargs.setdefault('region_name', region_name) - self.neutron = neutron_client.Client(**kwargs) + client_kwargs = base.client_kwargs(**kwargs) + self.neutron = neutron_client.Client(**client_kwargs) + + def __getattr__(self, attr): + return getattr(self.neutron, attr) + + +class NeutronClientWrapper(object): + @property + def neutron(self): + return BlazarNeutronClient() class FloatingIPPool(BlazarNeutronClient): @@ -156,3 +123,16 @@ def delete_reserved_floatingip(self, address): self.neutron.update_floatingip(fip['id'], body) self.neutron.delete_floatingip(fip['id']) + + def show_floatingip(self, address): + # using list_floatingips with query as show_floatingip() + # does not work and raises neutronclient.common.exceptions.NotFound + query = { + 'floating_ip_address': address, + 'floating_network_id': self.network_id + } + fips = self.neutron.list_floatingips(**query)['floatingips'] + if not fips: + # The floating ip address already deleted by the user. + raise manager_ex.FloatingIPNotFound(floatingip=address) + return next(iter(fips)) diff --git a/blazar/utils/openstack/nova.py b/blazar/utils/openstack/nova.py index ba7a6a2be..95118c56b 100644 --- a/blazar/utils/openstack/nova.py +++ b/blazar/utils/openstack/nova.py @@ -13,22 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. import uuid as uuidgen +import concurrent.futures -from keystoneauth1 import session -from keystoneauth1 import token_endpoint from novaclient import client as nova_client from novaclient import exceptions as nova_exception from novaclient.v2 import servers from oslo_config import cfg from oslo_log import log as logging -from blazar import context from blazar.manager import exceptions as manager_exceptions from blazar.plugins import oshosts from blazar.utils.openstack import base nova_opts = [ + cfg.StrOpt('endpoint_type', + default='internal', + choices=['public', 'admin', 'internal'], + help='Type of the nova endpoint to use. This endpoint will be ' + 'looked up in the keystone catalog and should be one of ' + 'public, internal or admin.'), cfg.StrOpt('nova_client_version', default='2', deprecated_group='DEFAULT', @@ -56,7 +60,9 @@ help='Aggregate metadata key for knowing owner project_id'), cfg.BoolOpt('az_aware', default=True, - help='A flag to store original availability zone') + help='A flag to store original availability zone'), + cfg.StrOpt('endpoint_override', + help='Nova endpoint URL to use') ] @@ -67,96 +73,11 @@ class BlazarNovaClient(object): - def __init__(self, **kwargs): - """Description - - BlazarNovaClient can be used in two ways: from context or kwargs. - - :param version: service client version which we will use - :type version: str - - :param ctx: request context - :type ctx: context object - - :param auth_token: keystone auth token - :type auth_token: str - - :param endpoint_override: endpoint url which we will use - :type endpoint_override: str - - :param username: username to use with nova client - :type username: str - - :param password: password to use with nova client - :type password: str - - :param user_domain_name: domain name of the user - :type user_domain_name: str - - :param project_name: project name to use with nova client - :type project_name: str - - :param project_domain_name: domain name of the project - :type project_domain_name: str - - :param auth_url: keystone url to authenticate against - :type auth_url: str - """ - - ctx = kwargs.pop('ctx', None) - auth_token = kwargs.pop('auth_token', None) - endpoint_override = kwargs.pop('endpoint_override', None) - version = kwargs.pop('version', CONF.nova.nova_client_version) - username = kwargs.pop('username', None) - password = kwargs.pop('password', None) - user_domain_name = kwargs.pop('user_domain_name', None) - project_name = kwargs.pop('project_name', None) - project_domain_name = kwargs.pop('project_domain_name', None) - auth_url = kwargs.pop('auth_url', None) - - if ctx is None: - try: - ctx = context.current() - except RuntimeError: - pass - if ctx is not None: - auth_token = auth_token or ctx.auth_token - endpoint_override = endpoint_override or \ - base.url_for(ctx.service_catalog, - CONF.nova.compute_service, - os_region_name=CONF.os_region_name) - auth_url = base.url_for(ctx.service_catalog, CONF.identity_service, - os_region_name=CONF.os_region_name) - kwargs.setdefault('global_request_id', ctx.global_request_id) - - if auth_url is None: - auth_url = "%s://%s:%s" % (CONF.os_auth_protocol, - base.get_os_auth_host(CONF), - CONF.os_auth_port) - if CONF.os_auth_prefix: - auth_url += "/%s" % CONF.os_auth_prefix - - if username: - kwargs.setdefault('username', username) - kwargs.setdefault('password', password) - kwargs.setdefault('project_name', project_name) - kwargs.setdefault('auth_url', auth_url) - - if "v2.0" not in auth_url: - kwargs.setdefault('project_domain_name', project_domain_name) - kwargs.setdefault('user_domain_name', user_domain_name) - else: - auth = token_endpoint.Token(endpoint_override, - auth_token) - sess = session.Session(auth=auth) - kwargs.setdefault('session', sess) - - kwargs.setdefault('endpoint_override', endpoint_override) - kwargs.setdefault('version', version) - self.nova = nova_client.Client(**kwargs) - + def __init__(self, version=None, **kwargs): + client_kwargs = base.client_kwargs(**kwargs) + self.nova = nova_client.Client( + version or CONF.nova.nova_client_version, **client_kwargs) self.nova.servers = ServerManager(self.nova) - self.exceptions = nova_exception def __getattr__(self, name): @@ -175,33 +96,22 @@ def create_image(self, server, image_name=None, metadata=None): class NovaClientWrapper(object): - def __init__(self, username=None, password=None, user_domain_name=None, - project_name=None, project_domain_name=None): - self.username = username - self.password = password - self.user_domain_name = user_domain_name - self.project_name = project_name - self.project_domain_name = project_domain_name + _version = None + + def __init__(self, version=None): + self._version = version @property def nova(self): - nova = BlazarNovaClient(username=self.username, - password=self.password, - user_domain_name=self.user_domain_name, - project_name=self.project_name, - project_domain_name=self.project_domain_name) + nova = BlazarNovaClient( + version=self._version, + endpoint_override=CONF.nova.endpoint_override) return nova class ReservationPool(NovaClientWrapper): def __init__(self): - super(ReservationPool, self).__init__( - username=CONF.os_admin_username, - password=CONF.os_admin_password, - user_domain_name=CONF.os_admin_user_domain_name, - project_name=CONF.os_admin_project_name, - project_domain_name=CONF.os_admin_project_domain_name) - + super(ReservationPool, self).__init__() self.config = CONF.nova self.freepool_name = self.config.aggregate_freepool_name @@ -238,7 +148,7 @@ def get_aggregate_from_name_or_id(self, aggregate_obj): def _generate_aggregate_name(): return str(uuidgen.uuid4()) - def create(self, name=None, az=None, metadata=None): + def create(self, name=None, project_id=None, az=None, metadata=None): """Create a Pool (an Aggregate) with or without Availability Zone. By default expose to user the aggregate with an Availability Zone. @@ -247,19 +157,17 @@ def create(self, name=None, az=None, metadata=None): """ name = name or self._generate_aggregate_name() - - LOG.debug('Creating pool aggregate: %(name)s with Availability Zone ' - '%(az)s', {'name': name, 'az': az}) + if not project_id: + raise manager_exceptions.ProjectIdNotFound() + + LOG.debug('Creating pool aggregate: %(name)s for %(project_id)s with ' + 'Availability Zone %(az)s', { + 'name': name, + 'project_id': project_id, + 'az': az + }) agg = self.nova.aggregates.create(name, az) - try: - ctx = context.current() - project_id = ctx.project_id - except RuntimeError: - e = manager_exceptions.ProjectIdNotFound() - LOG.error(str(e)) - raise e - if metadata: metadata[self.config.blazar_owner] = project_id else: @@ -346,8 +254,8 @@ def add_computehost(self, pool, hosts, stay_in=False): except manager_exceptions.AggregateNotFound: raise manager_exceptions.NoFreePool() - try: - for host in hosts: + def process_host(host): + try: if freepool_agg.id != agg.id and not stay_in: if host not in freepool_agg.hosts: raise manager_exceptions.HostNotInFreePool( @@ -361,6 +269,15 @@ def add_computehost(self, pool, hosts, stay_in=False): except nova_exception.NotFound: raise manager_exceptions.HostNotFound(host=host) + # When moving a host out of the freepool, we need to + # terminate preemptible instances before adding hosts to + # the reservation aggregate, which makes them available for + # scheduling. + # + # NOTE(priteau): Preemptibles should not be used with + # instance reservation yet. + self.terminate_preemptibles(host) + LOG.info("adding host '%(host)s' to aggregate %(id)s", {'host': host, 'id': agg.id}) try: @@ -371,6 +288,15 @@ def add_computehost(self, pool, hosts, stay_in=False): except nova_exception.Conflict as e: raise manager_exceptions.AggregateAlreadyHasHost( pool=pool, host=host, nova_exception=str(e)) + except Exception as e: + LOG.exception('Error processing host %s: %s', host, str(e)) + raise e + + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = executor.map(process_host, hosts) + try: + # exception will be raised when the value is retrieved from the iterator + results = list(futures) except Exception as e: if added_hosts: LOG.warn('Removing hosts added to aggregate %s: %s', @@ -407,11 +333,11 @@ def remove_computehost(self, pool, hosts): hosts_failing_to_remove = [] hosts_failing_to_add = [] hosts_not_in_freepool = [] - for host in hosts: + def process_host(host): if freepool_agg.id == agg.id: if host not in freepool_agg.hosts: hosts_not_in_freepool.append(host) - continue + return try: self.nova.aggregates.remove_host(agg.id, host) except nova_exception.ClientException: @@ -423,7 +349,8 @@ def remove_computehost(self, pool, hosts): self.nova.aggregates.add_host(freepool_agg.id, host) except nova_exception.ClientException: hosts_failing_to_add.append(host) - + with concurrent.futures.ThreadPoolExecutor() as executor: + executor.map(process_host, hosts) if hosts_failing_to_remove: raise manager_exceptions.CantRemoveHost( host=hosts_failing_to_remove, pool=agg) @@ -451,6 +378,20 @@ def remove_project(self, pool, project_id): metadata = {project_id: None} return self.nova.aggregates.set_metadata(agg.id, metadata) + def terminate_preemptibles(self, host): + """Terminate preemptible instances running on host""" + for server in self.nova.servers.list( + search_opts={"host": host, "all_tenants": 1}): + try: + LOG.info('Terminating preemptible instance %s (%s)', + server.name, server.id) + self.nova.servers.delete(server=server) + except nova_exception.NotFound: + LOG.info('Could not find server %s, may have been deleted ' + 'concurrently.', server) + except Exception as e: + LOG.exception('Failed to delete %s: %s.', server, str(e)) + class NovaInventory(NovaClientWrapper): @@ -507,22 +448,20 @@ def get_host_details(self, host): raise manager_exceptions.InvalidHost(host=host) def get_servers_per_host(self, host): - """List all servers of a nova-compute host + """List all servers running on a given Nova hypervisor - :param host: Name (not UUID) of nova-compute host - :return: Dict of servers or None + :param host: hypervisor_hostname (or name pattern) of the host + :return: list of server or None """ try: - hypervisors_list = self.nova.hypervisors.search(host, servers=True) + hypervisors_list = self.nova.hypervisors.search(host) except nova_exception.NotFound: raise manager_exceptions.HostNotFound(host=host) if len(hypervisors_list) > 1: raise manager_exceptions.MultipleHostsFound(host=host) - else: - try: - return hypervisors_list[0].servers - except AttributeError: - # NOTE(sbauza): nova.hypervisors.search(servers=True) returns - # a list of hosts without 'servers' attribute if no servers - # are running on that host - return None + + hypervisor = hypervisors_list[0] + servers = self.nova.servers.list( + search_opts={"node": hypervisor.hypervisor_hostname, + "all_tenants": 1}) + return servers or None diff --git a/blazar/utils/openstack/placement.py b/blazar/utils/openstack/placement.py index 5eccf8f25..d5c80f1f7 100644 --- a/blazar/utils/openstack/placement.py +++ b/blazar/utils/openstack/placement.py @@ -11,20 +11,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -import retrying - from keystoneauth1 import adapter from keystoneauth1.identity import v3 from keystoneauth1 import session - from oslo_config import cfg -from oslo_log import log as logging from blazar import context from blazar.utils.openstack import base from blazar.utils.openstack import exceptions +from oslo_log import log as logging +import retrying + + +placement_opts = [ + cfg.StrOpt('endpoint_type', + default='internal', + choices=['public', 'admin', 'internal'], + help='Type of the placement endpoint to use. This endpoint ' + 'will be looked up in the keystone catalog and should be ' + 'one of public, internal or admin.'), +] CONF = cfg.CONF +CONF.register_opts(placement_opts, group='placement') LOG = logging.getLogger(__name__) PLACEMENT_MICROVERSION = 1.29 @@ -78,7 +87,7 @@ def _create_client(self, **kwargs): # service of our response body media type preferences. headers = {'accept': 'application/json'} kwargs.setdefault('service_type', 'placement') - kwargs.setdefault('interface', 'public') + kwargs.setdefault('interface', CONF.placement.endpoint_type) kwargs.setdefault('additional_headers', headers) kwargs.setdefault('region_name', region_name) client = adapter.Adapter(sess, **kwargs) @@ -104,6 +113,15 @@ def delete(self, url, microversion=PLACEMENT_MICROVERSION): return client.delete(url, raise_exc=False, microversion=microversion) + def _get_reservation_provider_name(self, host_name): + """Get the name of a reservation provider from the host name. + + :param host_name: Name of the host + :return: The name of the reservation provider + """ + + return 'blazar_' + host_name + def get_resource_provider(self, rp_name): """Calls the placement API for a resource provider record. @@ -112,6 +130,7 @@ def get_resource_provider(self, rp_name): or None if the resource provider doesn't exist. :raise: ResourceProviderRetrievalFailed on error. """ + url = "/resource_providers?name=%s" % rp_name resp = self.get(url) if resp: @@ -131,6 +150,19 @@ def get_resource_provider(self, rp_name): LOG.error(msg, args) raise exceptions.ResourceProviderRetrievalFailed(name=rp_name) + def get_reservation_provider(self, host_name): + """Calls the placement API for a reservation provider record. + + :param host_name: Name of the host + :return: A dict of resource provider information + or None if the resource provider doesn't exist. + :raise: ResourceProviderRetrievalFailed on error. + """ + + return self.get_resource_provider( + self._get_reservation_provider_name(host_name) + ) + def create_resource_provider(self, rp_name, rp_uuid=None, parent_uuid=None): """Calls the placement API to create a new resource provider record. @@ -142,6 +174,7 @@ def create_resource_provider(self, rp_name, rp_uuid=None, the newly-created resource provider. :raise: ResourceProviderCreationFailed error. """ + url = "/resource_providers" payload = {'name': rp_name} if rp_uuid is not None: @@ -186,6 +219,7 @@ def delete_resource_provider(self, rp_uuid): :param rp_uuid: UUID of the resource provider to delete :raise: ResourceProviderDeletionFailed error """ + url = '/resource_providers/%s' % rp_uuid resp = self.delete(url) @@ -210,7 +244,7 @@ def create_reservation_provider(self, host_name): raise exceptions.ResourceProviderNotFound( resource_provider=host_name) host_uuid = host_rp['uuid'] - rp_name = "blazar_" + host_name + rp_name = self._get_reservation_provider_name(host_name) reservation_rp = self.create_resource_provider( rp_name, parent_uuid=host_uuid) @@ -218,7 +252,7 @@ def create_reservation_provider(self, host_name): def delete_reservation_provider(self, host_name): """Delete the reservation provider, the child of the given host""" - rp_name = "blazar_" + host_name + rp_name = self._get_reservation_provider_name(host_name) rp = self.get_resource_provider(rp_name) if rp is None: # If the reservation provider doesn't exist, @@ -234,6 +268,7 @@ def create_resource_class(self, rc_name): shall be something like "CUSTOM_RESERVATION_{uuid}". :raises: ResourceClassCreationFailed error. """ + url = '/resource_classes' payload = {'name': rc_name} resp = self.post(url, payload) @@ -257,6 +292,7 @@ def delete_resource_class(self, rc_name): shall be something like "CUSTOM_RESERVATION_{uuid}" :raises: ResourceClassDeletionFailed error. """ + url = '/resource_classes/%s' % rc_name resp = self.delete(url) if resp: @@ -298,12 +334,24 @@ def get_inventory(self, rp_uuid): :param rp_uuid: UUID of the resource provider to get """ + url = '/resource_providers/%s/inventories' % rp_uuid resp = self.get(url) if resp: return resp.json() raise exceptions.ResourceProviderNotFound(resource_provider=rp_uuid) + def get_traits(self, rp_uuid): + """Calls the placement API to get resource trait information. + + :param rp_uuid: UUID of the resource provider to get + """ + url = '/resource_providers/%s/traits' % rp_uuid + resp = self.get(url) + if resp: + return resp.json().get("traits", []) + raise exceptions.ResourceProviderNotFound(resource_provider=rp_uuid) + @retrying.retry(stop_max_attempt_number=5, retry_on_exception=lambda e: isinstance( e, exceptions.InventoryConflict)) @@ -317,6 +365,7 @@ def update_inventory(self, rp_uuid, rc_name, num, additional): True, else just overwrite the total value :raises: ResourceProviderNotFound or InventoryUpdateFailed error. """ + curr = self.get_inventory(rp_uuid) inventories = curr['inventories'] generation = curr['resource_provider_generation'] @@ -371,6 +420,7 @@ def delete_inventory(self, rp_uuid, rc_name): :param rc_name: The resource class name to delete from inventory :raises: InventoryUpdateFailed error """ + url = '/resource_providers/%s/inventories/%s' % (rp_uuid, rc_name) resp = self.delete(url) @@ -388,8 +438,9 @@ def update_reservation_inventory(self, host_name, reserv_uuid, num, :param num: The number of the instances to reserve on the host :return: The updated inventory record """ + # Get reservation provider uuid - rp_name = "blazar_" + host_name + rp_name = self._get_reservation_provider_name(host_name) rp = self.get_resource_provider(rp_name) if rp is None: # If the reservation provider is not created yet, @@ -411,8 +462,9 @@ def delete_reservation_inventory(self, host_name, reserv_uuid): :raises: ResourceProviderNotFound if the reservation provider is not found """ + # Get reservation provider uuid - rp_name = "blazar_" + host_name + rp_name = self._get_reservation_provider_name(host_name) rp = self.get_resource_provider(rp_name) if rp is None: raise exceptions.ResourceProviderNotFound( @@ -429,3 +481,259 @@ def delete_reservation_inventory(self, host_name, reserv_uuid): LOG.info("Resource class %s doesn't exist or there is no " "inventory for that resource class on resource provider " "%s. Skipped the deletion", rc_name, rp_name) + + def _get_custom_reservation_trait_name(self, reserv_uuid, project_id): + # A valid trait must be no longer than 255 characters, + # start with the prefix "CUSTOM_" + # and use following characters: "A"-"Z", "0"-"9" and "_" + reserv_uuid = reserv_uuid.upper().replace("-", "_") + project_id = project_id.upper().replace("-", "_") + return "CUSTOM_RESERVATION_" + reserv_uuid + "_PROJECT_" + project_id + + def _trait_exists(self, trait_name): + """Calls the placement API check if a trait exists. + + :param trait: Name of the trait + :return: True if the trait exists + or False if the trait doesn't exist. + :raise: TraitRetrievalFailed on error. + """ + + url = '/traits/%s' % trait_name + resp = self.get(url) + if resp.status_code == 204: + return True + elif resp.status_code == 404: + return False + + msg = ("Failed to get trait %(name)s. " + "Got %(status_code)d: %(err_text)s.") + args = { + 'name': trait_name, + 'status_code': resp.status_code, + 'err_text': resp.text, + } + LOG.error(msg, args) + raise exceptions.TraitRetrievalFailed(trait=trait_name) + + def create_trait(self, trait_name): + """Calls the placement API to create a trait. + + :param trait_name: The name of the trait + :raises: TraitCreationFailed error. + """ + + url = '/traits/%s' % trait_name + resp = self.put(url, {}) + if resp: + LOG.info("Created trait %s", trait_name) + return + msg = ("Failed to create trait with placement API for " + "%(trait_name)s. Got %(status_code)d: %(err_text)s.") + args = { + 'trait_name': trait_name, + 'status_code': resp.status_code, + 'err_text': resp.text, + } + LOG.error(msg, args) + raise exceptions.TraitCreationFailed(trait=trait_name) + + def delete_trait(self, trait_name): + """Calls the placement API to delete a trait. + + :param trait_name: The name of the trait + :raises: TraitDeletionFailed error. + """ + + url = '/traits/%s' % trait_name + resp = self.delete(url) + if resp: + LOG.info("Deleted trait %s", trait_name) + return + msg = ("Failed to delete trait with placement API for " + "%(trait_name)s. Got %(status_code)d: %(err_text)s.") + args = { + 'trait_name': trait_name, + 'status_code': resp.status_code, + 'err_text': resp.text, + } + LOG.error(msg, args) + raise exceptions.TraitDeletionFailed(trait=trait_name) + + def reservation_trait_exists(self, reserv_uuid, project_id): + """Check if the reservation trait exists. + + :param reservation_uuid: The reservation uuid + :raises: TraitRetrievalFailed error. + """ + + return self._trait_exists(self._get_custom_reservation_trait_name( + reserv_uuid, project_id)) + + def create_reservation_trait(self, reserv_uuid, project_id): + """Create the reservation trait. + + :param reservation_uuid: The reservation uuid + :raises: TraitCreationFailed error. + """ + + self.create_trait(self._get_custom_reservation_trait_name( + reserv_uuid, project_id)) + + def delete_reservation_trait(self, reserv_uuid, project_id): + """Delete the reservation trait. + + :param reservation_uuid: The reservation uuid + :raises: TraitDeletionFailed error. + """ + + self.delete_trait(self._get_custom_reservation_trait_name( + reserv_uuid, project_id)) + + def associate_traits_with_resource_provider(self, rp_uuid, traits): + """Associate traits with the resource provider. + + :param rp_uuid: The uuid of the resource provider + :param traits: The list of traits + :raises: TraitAssociationFailed error. + """ + + url = '/resource_providers/%s/traits' % rp_uuid + resp = self.get(url) + current_traits = [] + resource_provider_generation = 0 + if resp: + json_resp = resp.json() + if json_resp['traits']: + current_traits = json_resp['traits'] + if json_resp['resource_provider_generation']: + resource_provider_generation = \ + json_resp['resource_provider_generation'] + + updated_traits = list(set(current_traits) | set(traits)) + payload = { + 'traits': updated_traits, + 'resource_provider_generation': resource_provider_generation, + } + resp = self.put(url, payload) + if resp: + LOG.info("Associated traits %s with resource provider %s", + ",".join(traits), rp_uuid) + return + msg = ("Failed to associate traits %(traits)s with resource " + "provider %(rp_uuid)s. Got %(status_code)d: %(err_text)s.") + args = { + 'traits': ','.join(traits), + 'rp_uuid': rp_uuid, + 'status_code': resp.status_code, + 'err_text': resp.text, + } + LOG.error(msg, args) + raise exceptions.TraitAssociationFailed( + traits=','.join(traits), uuid=rp_uuid) + + def dissociate_traits_with_resource_provider(self, rp_uuid, traits): + """Dissociate traits with the resource provider. + + :param rp_uuid: The uuid of the resource provider + :param traits: The list of traits + :raises: TraitdissociationFailed error. + """ + + url = '/resource_providers/%s/traits' % rp_uuid + resp = self.get(url) + current_traits = [] + resource_provider_generation = 0 + if resp: + json_resp = resp.json() + if json_resp['traits']: + current_traits = json_resp['traits'] + if json_resp['resource_provider_generation']: + resource_provider_generation = \ + json_resp['resource_provider_generation'] + + updated_traits = list(set(current_traits) - set(traits)) + + payload = { + 'traits': updated_traits, + 'resource_provider_generation': resource_provider_generation, + } + resp = self.put(url, payload) + if resp: + LOG.info("Dissociated traits %s with resource provider %s", + ",".join(traits), rp_uuid) + return + msg = ("Failed to dissociate traits %(traits)s with resource " + "provider %(rp_uuid)s. Got %(status_code)d: %(err_text)s.") + args = { + 'traits': ','.join(traits), + 'rp_uuid': rp_uuid, + 'status_code': resp.status_code, + 'err_text': resp.text, + } + LOG.error(msg, args) + raise exceptions.TraitDissociationFailed( + traits=','.join(traits), uuid=rp_uuid) + + def associate_reservation_trait_with_resource_provider( + self, rp_uuid, reserv_uuid, project_id): + """Associate reservation trait with the resource provider. + + :param rp_uuid: The uuid of the resource provider + :param reserv_uuid: The reservation uuid + :raises: TraitAssociationFailed error. + """ + + self.associate_traits_with_resource_provider( + rp_uuid, + [self._get_custom_reservation_trait_name(reserv_uuid, project_id)]) + + def dissociate_reservation_trait_with_resource_provider( + self, rp_uuid, reserv_uuid, project_id): + """Dissociate reservation trait with the resource provider. + + :param rp_uuid: The uuid of the resource provider + :param reserv_uuid: The reservation uuid + :raises: TraitAssociationFailed error. + """ + + self.dissociate_traits_with_resource_provider( + rp_uuid, + [self._get_custom_reservation_trait_name(reserv_uuid, project_id)]) + + def list_resource_providers(self): + """Get all resource providers.""" + resp = self.get('/resource_providers') + resource_providers = [] + if resp: + json_resp = resp.json() + if json_resp['resource_providers']: + resource_providers = json_resp['resource_providers'] + return resource_providers + + def get_trait_resource_providers(self, trait_name): + """Get all resource providers that associate with the trait + + :param trait_name: The name of the trait + :return: A list of resource providers + """ + + trait_rps = [] + # get all resource providers + resource_providers = self.list_resource_providers() + + # filter resource providers with the trait + url = '/resource_providers/%s/traits' + for rp in resource_providers: + resp = self.get(url % rp['uuid']) + if resp: + json_resp = resp.json() + if json_resp['traits'] and trait_name in json_resp['traits']: + trait_rps.append(rp) + + return trait_rps + + def get_reservation_trait_resource_providers( + self, reserv_uuid, project_id): + return self.get_trait_resource_providers( + self._get_custom_reservation_trait_name(reserv_uuid, project_id)) diff --git a/blazar/utils/openstack/zun.py b/blazar/utils/openstack/zun.py new file mode 100644 index 000000000..375240f09 --- /dev/null +++ b/blazar/utils/openstack/zun.py @@ -0,0 +1,90 @@ +# Copyright (c) 2020 University of Chicago +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from blazar.manager import exceptions as manager_exceptions +from blazar.utils.openstack import base +from oslo_log import log as logging +from zunclient import client as zun_client +from zunclient import exceptions as zun_exception + + +zun_opts = [ + cfg.StrOpt( + 'zun_api_version', + default='1', + help='Zun API version'), + cfg.StrOpt( + 'zun_api_microversion', + default='1.22', + help='Zun API microversion'), + cfg.StrOpt( + 'endpoint_override', + help='Zun endpoint URL to use') +] + +CONF = cfg.CONF +CONF.register_opts(zun_opts, group='zun') + +LOG = logging.getLogger(__name__) + + +class BlazarZunClient(object): + """Client class for Zun service.""" + + def __init__(self, **kwargs): + client_kwargs = base.client_kwargs(**kwargs) + client_kwargs.setdefault('os_zun_api_version', + CONF.zun.zun_api_microversion) + self.zun = zun_client.Client( + CONF.zun.zun_api_version, **client_kwargs) + + def __getattr__(self, attr): + return getattr(self.zun, attr) + + +class ZunClientWrapper(object): + @property + def zun(self): + zun = BlazarZunClient(endpoint_override=CONF.zun.endpoint_override) + return zun + + +class ZunInventory(BlazarZunClient): + def get_host_details(self, host): + """Get Zun capabilities of a single host + + :param host: UUID or name of zun compute node + :return: Dict of capabilities or raise HostNotFound + """ + try: + host = self.zun.hosts.get(host) + except (zun_exception.NotFound, zun_exception.BadRequest): + host_ids = [] + for h in self.zun.hosts.list(): + if h.hostname == host: + host_ids.append(h.uuid) + if len(host_ids) == 0: + raise manager_exceptions.HostNotFound(host=host) + elif len(host_ids) > 1: + raise manager_exceptions.MultipleHostsFound(host=host) + else: + host = self.zun.hosts.get(host_ids[0]) + + return {'id': host.uuid, + 'name': host.hostname, + 'containers': self.zun.containers.list(host=host.hostname) + } diff --git a/blazar/utils/plugins.py b/blazar/utils/plugins.py index 0afd7b8af..36bb29654 100644 --- a/blazar/utils/plugins.py +++ b/blazar/utils/plugins.py @@ -13,10 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy from oslo_serialization import jsonutils from blazar.manager import exceptions as manager_ex +from blazar.utils.openstack import keystone +import copy +import logging +import shlex +import subprocess + + +LOG = logging.getLogger(__name__) def convert_requirements(requirements): @@ -100,3 +107,41 @@ def list_subtract(list_a, list_b): result1 = list_subtract(list1, list2) result2 = list_subtract(list2, list1) return result1, result2 + + +def send_lease_extension_reminder(lease, region_name): + project_id = lease['project_id'] + user_id = lease['user_id'] + keystoneclient = keystone.BlazarKeystoneClient() + project = keystoneclient.projects.get(project_id) + user = keystoneclient.users.get(user_id) + params_tmp = ('--to "{recipient}" ' + '--username "{username}" ' + '--project-name "{project_name}" ' + '--project-id "{project_id}" ' + '--lease-name "{lease_name}" ' + '--lease-id "{lease_id}" ' + '--end-datetime "{end_datetime}" ' + '--site "{site}"') + try: + if not user.email: + LOG.error('Email address not found for user {username}'.format( + username=user.name)) + return + except AttributeError: + LOG.error(f"User does not have email attribute - username: {user.name}") + return + params = params_tmp.format(recipient=user.email, + username=user.name, + project_name=project.name, + project_id=project_id, + lease_name=lease['name'], + lease_id=lease['id'], + end_datetime=lease['end_date'], + site=region_name) + try: + subprocess.check_call(shlex.split( + '/usr/local/bin/blazar_before_end_action_email {params}'. + format(params=params))) + except Exception as e: + LOG.exception(str(e)) diff --git a/blazar/utils/service.py b/blazar/utils/service.py index 1690de7ab..ef94d03e1 100644 --- a/blazar/utils/service.py +++ b/blazar/utils/service.py @@ -74,7 +74,7 @@ def __getattr__(self, name): method = getattr(self.__endpoint, name) def run_method(__ctx, **kwargs): - with context.BlazarContext(**__ctx): + with context.BlazarContext.from_dict(__ctx): return method(**kwargs) return run_method diff --git a/blazar/utils/trusts.py b/blazar/utils/trusts.py index 74a704831..57ae7733c 100644 --- a/blazar/utils/trusts.py +++ b/blazar/utils/trusts.py @@ -13,32 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -import functools - from oslo_config import cfg from blazar import context -from blazar.utils.openstack import base from blazar.utils.openstack import keystone +import functools + CONF = cfg.CONF def create_trust(): """Creates trust via Keystone API v3 to use in plugins.""" - client = keystone.BlazarKeystoneClient() - - trustee_id = keystone.BlazarKeystoneClient( - username=CONF.os_admin_username, - password=CONF.os_admin_password, - tenant_name=CONF.os_admin_project_name).user_id + trustee_id = keystone.BlazarKeystoneClient().session.get_user_id() ctx = context.current() - trust = client.trusts.create(trustor_user=ctx.user_id, - trustee_user=trustee_id, - impersonation=False, - role_names=ctx.roles, - project=ctx.project_id) + user_client = keystone.BlazarKeystoneClient(as_user=True) + trust = user_client.trusts.create(trustor_user=ctx.user_id, + trustee_user=trustee_id, + impersonation=True, + role_names=ctx.roles, + project=ctx.project_id) return trust @@ -52,32 +47,19 @@ def delete_trust(lease): def create_ctx_from_trust(trust_id): """Return context built from given trust.""" ctx = context.current() + client = keystone.BlazarKeystoneClient(trust_id=trust_id) + session = client.session - ctx = context.BlazarContext( - user_name=CONF.os_admin_username, - project_name=CONF.os_admin_project_name, - request_id=ctx.request_id, - global_request_id=ctx.global_request_id - ) - auth_url = "%s://%s:%s" % (CONF.os_auth_protocol, - base.get_os_auth_host(CONF), - CONF.os_auth_port) - if CONF.os_auth_prefix: - auth_url += "/%s" % CONF.os_auth_prefix - client = keystone.BlazarKeystoneClient( - password=CONF.os_admin_password, - trust_id=trust_id, - auth_url=auth_url, - ctx=ctx, - ) + auth_ref = session.auth.get_auth_ref(session) # use 'with ctx' statement in the place you need context from trust return context.BlazarContext( user_name=ctx.user_name, - project_name=ctx.project_name, - auth_token=client.auth_token, - service_catalog=client.service_catalog.catalog['catalog'], - project_id=client.tenant_id, + user_domain_name=ctx.user_domain_name, + roles=auth_ref.role_names, + auth_token=session.get_token(), + project_id=session.get_project_id(), + service_catalog=(ctx.service_catalog or auth_ref.service_catalog), request_id=ctx.request_id, global_request_id=ctx.global_request_id ) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 977e2aba8..2c40d868b 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -31,7 +31,7 @@ function configure_blazar { touch $BLAZAR_CONF_FILE iniset $BLAZAR_CONF_FILE DEFAULT os_auth_version v3 - iniset $BLAZAR_CONF_FILE DEFAULT os_auth_host $(ipv6_unquote $KEYSTONE_AUTH_HOST) + iniset $BLAZAR_CONF_FILE DEFAULT os_auth_host $(ipv6_unquote $KEYSTONE_SERVICE_HOST) iniset $BLAZAR_CONF_FILE DEFAULT os_auth_port 80 iniset $BLAZAR_CONF_FILE DEFAULT os_auth_prefix identity iniset $BLAZAR_CONF_FILE DEFAULT os_admin_password $SERVICE_PASSWORD @@ -39,16 +39,22 @@ function configure_blazar { iniset $BLAZAR_CONF_FILE DEFAULT os_admin_project_name $SERVICE_TENANT_NAME iniset $BLAZAR_CONF_FILE DEFAULT identity_service $BLAZAR_IDENTITY_SERVICE_NAME iniset $BLAZAR_CONF_FILE DEFAULT os_region_name $REGION_NAME + iniset $BLAZAR_CONF_FILE DEFAULT endpoint_type public # Keystone authtoken _blazar_setup_keystone $BLAZAR_CONF_FILE keystone_authtoken + iniset $BLAZAR_CONF_FILE neutron endpoint_type public + iniset $BLAZAR_CONF_FILE nova aggregate_freepool_name $BLAZAR_FREEPOOL_NAME + iniset $BLAZAR_CONF_FILE nova endpoint_type public + + iniset $BLAZAR_CONF_FILE placement endpoint_type public iniset $BLAZAR_CONF_FILE DEFAULT host $(ipv6_unquote $SERVICE_HOST) iniset $BLAZAR_CONF_FILE DEFAULT debug $BLAZAR_DEBUG - iniset $BLAZAR_CONF_FILE manager plugins physical.host.plugin,virtual.instance.plugin,virtual.floatingip.plugin + iniset $BLAZAR_CONF_FILE manager plugins physical.host.plugin,virtual.instance.plugin,virtual.floatingip.plugin,flavor.instance.plugin iniset $BLAZAR_CONF_FILE api api_v2_controllers oshosts,leases @@ -62,21 +68,12 @@ function configure_blazar { ACTUAL_FILTERS=$(iniget $NOVA_CONF filter_scheduler enabled_filters) if [[ -z "$ACTUAL_FILTERS" ]]; then - iniadd $NOVA_CONF filter_scheduler enabled_filters "RetryFilter, AvailabilityZoneFilter, RamFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, AggregateInstanceExtraSpecsFilter, AggregateMultiTenancyIsolation, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, BlazarFilter" + iniadd $NOVA_CONF filter_scheduler enabled_filters "AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter,BlazarFilter" else - if [[ $ACTUAL_FILTERS != *AggregateInstanceExtraSpecsFilter* ]]; then - ACTUAL_FILTERS="$ACTUAL_FILTERS,AggregateInstanceExtraSpecsFilter" - fi - if [[ $ACTUAL_FILTERS != *AggregateMultiTenancyIsolation* ]]; then - ACTUAL_FILTERS="$ACTUAL_FILTERS,AggregateMultiTenancyIsolation" - fi - if [[ $ACTUAL_FILTERS != *ServerGroupAntiAffinityFilter* ]]; then - ACTUAL_FILTERS="$ACTUAL_FILTERS,ServerGroupAntiAffinityFilter" - fi if [[ $ACTUAL_FILTERS != *BlazarFilter* ]]; then - ACTUAL_FILTERS="$ACTUAL_FILTERS,BlazarFilter" + ACTUAL_FILTERS="$ACTUAL_FILTERS,BlazarFilter" fi - iniset $NOVA_CONF filter_scheduler enabled_filters $ACTUAL_FILTERS + iniset $NOVA_CONF filter_scheduler enabled_filters $ACTUAL_FILTERS fi ACTUAL_AVAILABLE_FILTERS=$(iniget $NOVA_CONF filter_scheduler available_filters) @@ -134,14 +131,6 @@ function create_blazar_accounts { get_or_create_endpoint $BLAZAR_SERVICE \ "$REGION_NAME" \ "$blazar_api_url/v1" - - KEYSTONEV3_SERVICE=$(get_or_create_service "keystonev3" \ - "identityv3" "Keystone Identity Service V3") - get_or_create_endpoint $KEYSTONEV3_SERVICE \ - "$REGION_NAME" \ - "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:$KEYSTONE_SERVICE_PORT/v3" \ - "$KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:$KEYSTONE_AUTH_PORT/v3" \ - "$KEYSTONE_SERVICE_PROTOCOL://$KEYSTONE_SERVICE_HOST:$KEYSTONE_SERVICE_PORT/v3" } diff --git a/doc/api_samples/hosts/host-property-list-detail.json b/doc/api_samples/hosts/host-property-list-detail.json new file mode 100644 index 000000000..ac386c8a4 --- /dev/null +++ b/doc/api_samples/hosts/host-property-list-detail.json @@ -0,0 +1,12 @@ +{ + "resource_properties": [ + { + "property": "gpu", + "private": false, + "values": [ + "True", + "False" + ] + } + ] +} diff --git a/doc/api_samples/hosts/host-property-list.json b/doc/api_samples/hosts/host-property-list.json new file mode 100644 index 000000000..713c41dbd --- /dev/null +++ b/doc/api_samples/hosts/host-property-list.json @@ -0,0 +1,7 @@ +{ + "resource_properties": [ + { + "property": "gpu" + } + ] +} diff --git a/doc/api_samples/hosts/host-property-update-res.json b/doc/api_samples/hosts/host-property-update-res.json new file mode 100644 index 000000000..1464df62d --- /dev/null +++ b/doc/api_samples/hosts/host-property-update-res.json @@ -0,0 +1,10 @@ +{ + "resource_property": { + "created_at": "2021-12-15T19:38:16.000000", + "updated_at": "2021-12-21T21:37:19.000000", + "id": "19e48cd0-042d-4044-a69a-d44d672849b5", + "resource_type": "physical:host", + "property_name": "gpu", + "private": true + } +} diff --git a/doc/api_samples/hosts/host-property-update.json b/doc/api_samples/hosts/host-property-update.json new file mode 100644 index 000000000..e72dc5449 --- /dev/null +++ b/doc/api_samples/hosts/host-property-update.json @@ -0,0 +1,3 @@ +{ + "private": true +} diff --git a/doc/source/admin/usage-enforcement.rst b/doc/source/admin/usage-enforcement.rst index 2ae96f086..a8cbc9434 100644 --- a/doc/source/admin/usage-enforcement.rst +++ b/doc/source/admin/usage-enforcement.rst @@ -6,16 +6,17 @@ Synopsis ======== Usage enforcement and lease constraints can be implemented by operators via -custom usage enforcement filters. +custom usage enforcement filters or an external service. Description =========== Usage enforcement filters are called on ``lease_create``, ``lease_update`` and ``on_end`` operations. The filters check whether or not lease values or -allocation criteria pass admin defined thresholds. There is currently one -filter provided out-of-the-box. The ``MaxLeaseDurationFilter`` restricts the -duration of leases. +allocation criteria pass admin defined thresholds. There are currently two +filters provided out-of-the-box. ``MaxLeaseDurationFilter`` restricts the +duration of leases. ``ExternalServiceFilter`` calls a third-party service for +implementing policies using a URL configured in ``blazar.conf``. Options ======= @@ -36,10 +37,15 @@ as follows: .. +Do note that filter config options follow filter names - the prefix is always +the snake case of the filter name (``MaxLeaseDurationFilter`` becomes +``max_lease_duration``; in this case it is special that there is nothing +beyond the prefix but there is also ``max_lease_duration_exempt_project_ids``). + MaxLeaseDurationFilter ---------------------- -This filter simply examines the lease ``start_time`` and ``end_time`` +This filter simply examines the lease ``start_date`` and ``end_date`` attributes and rejects the lease if its duration exceeds a threshold. It supports two configuration options: @@ -48,3 +54,106 @@ supports two configuration options: See the :doc:`../configuration/blazar-conf` page for a description of these options. + +ExternalServiceFilter +--------------------- + +This filter delegates the decision for each API to an external HTTP service. +The service must use token-based authentication, accepting (or ignoring) +the static token sent by Blazar in the ``X-Auth-Token`` header. +The following endpoints should be implemented: + +* ``POST /check-create`` +* ``POST /check-update`` +* ``POST /on-end`` + +The exact URLs can be overridden and not all have to be used (although +we imagine a proper implementation requires at least both checks unless +lease updates are disabled in the first place). + +The external service should return ``204 No Content`` if the parameters meet +defined criteria and ``403 Forbidden`` if not. The service may send a JSON +body response with the ``403 Forbidden`` reply, including the rejection +reasoning in the field named ``message`` as in: + +.. sourcecode:: json + + { + "message": "You shall not pass!" + } + +An example of data the external service will receive in a request body (do note +all dates and times are encoded as strings following the ISO8601 standard that +is expected in JSON to represent dates and times): + +* Request example: + +.. sourcecode:: json + + { + "context": { + "user_id": "c631173e-dec0-4bb7-a0c3-f7711153c06c", + "project_id": "a0b86a98-b0d3-43cb-948e-00689182efd4", + "auth_url": "https://api.example.com:5000/v3", + "region_name": "RegionOne" + }, + "current_lease": { + "start_date": "2020-05-13T00:00:00.012345+02:00", + "end_time": "2020-05-14T23:59:00.012345+02:00", + "reservations": [ + { + "resource_type": "physical:host", + "min": 1, + "max": 2, + "hypervisor_properties": "[]", + "resource_properties": "[\"==\", \"$availability_zone\", \"az1\"]", + "allocations": [ + { + "id": "1", + "hypervisor_hostname": "32af5a7a-e7a3-4883-a643-828e3f63bf54", + "extra": { + "availability_zone": "az1" + } + } + ] + } + ] + }, + "lease": { + "start_date": "2020-05-13T00:00:00.012345+02:00", + "end_time": "2020-05-14T23:59:00.012345+02:00", + "reservations": [ + { + "resource_type": "physical:host", + "min": 2, + "max": 3, + "hypervisor_properties": "[]", + "resource_properties": "[\"==\", \"$availability_zone\", \"az1\"]", + "allocations": [ + { + "id": "1", + "hypervisor_hostname": "32af5a7a-e7a3-4883-a643-828e3f63bf54", + "extra": { + "availability_zone": "az1" + } + }, + { + "id": "2", + "hypervisor_hostname": "af69aabd-8386-4053-a6dd-1a983787bd7f", + "extra": { + "availability_zone": "az1" + } + } + ] + } + ] + } + } + +The ``current_lease`` field is present only in ``check-update`` requests and +describes the existing lease. In both checks the ``lease`` field describes +the new lease. In ``on-end``, the ``lease`` field describes the lease that +has just ended. + +There is no guarantee on the delivery of the ``on-end`` event and it should be +considered an optimisation rather than a reliable mechanism. diff --git a/doc/source/cli/flavor-based-instance-reservation.rst b/doc/source/cli/flavor-based-instance-reservation.rst new file mode 100644 index 000000000..32ba7834c --- /dev/null +++ b/doc/source/cli/flavor-based-instance-reservation.rst @@ -0,0 +1,200 @@ +================================= +Flavor-based Instance Reservation +================================= + +Prerequisites +------------- + +The following packages should be installed: + +* blazar +* blazar-nova +* python-blazarclient + +1. Add hosts into the freepool +------------------------------ + +1. Add hosts into the Blazar freepool using the host-create command: + +.. sourcecode:: console + + # Using the blazar CLI + blazar host-create compute-1 + + # Using the openstack CLI + openstack reservation host create compute-1 + +.. + +2. Check hosts in the freepool: + +.. sourcecode:: console + + # Using the blazar CLI + blazar host-list + + # Using the openstack CLI + openstack reservation host list + +.. + +Result: + +.. sourcecode:: console + + +----+---------------------+-------+-----------+----------+ + | id | hypervisor_hostname | vcpus | memory_mb | local_gb | + +----+---------------------+-------+-----------+----------+ + | 1 | compute-1 | 2 | 3951 | 38 | + +----+---------------------+-------+-----------+----------+ + +.. + +2. Create a lease +----------------- + +1. Create a lease (instance reservation) using lease-create command: + +.. sourcecode:: console + + # Using the blazar CLI + blazar lease-create --reservation resource_type=flavor:instance,flavor_id=3,amount=1 --start-date "2024-08-23 12:00" --end-date "2024-09-08 13:00" lease-1 + + # Using the openstack CLI + openstack reservation lease create --reservation resource_type=flavor:instance,flavor_id=3,amount=1 --start-date "2024-08-23 14:00" --end-date "2024-09-08 15:00" lease-1 + +.. + +Result: + +.. sourcecode:: console + + +--------------+-------------------------------------------------------------------------------------------------------------------------------------+ + | Field | Value | + +--------------+-------------------------------------------------------------------------------------------------------------------------------------+ + | created_at | 2024-08-23 12:40:53 | + | degraded | False | + | end_date | 2024-09-08T15:00:00.000000 | + | events | { | + | | "id": "29e939d9-a158-4376-86d7-5093c3a379cb", | + | | "lease_id": "d7779e56-2b78-4465-8f19-9cc916d97b11", | + | | "event_type": "end_lease", | + | | "time": "2024-09-08T15:00:00.000000", | + | | "status": "UNDONE", | + | | "created_at": "2024-08-23 12:40:53", | + | | "updated_at": null | + | | } | + | | { | + | | "id": "6c985a6e-416a-496a-9714-ae7b2dea0753", | + | | "lease_id": "d7779e56-2b78-4465-8f19-9cc916d97b11", | + | | "event_type": "before_end_lease", | + | | "time": "2024-09-08T14:00:00.000000", | + | | "status": "UNDONE", | + | | "created_at": "2024-08-23 12:40:53", | + | | "updated_at": null | + | | } | + | | { | + | | "id": "e3308415-78b1-441d-8881-74e9569e0635", | + | | "lease_id": "d7779e56-2b78-4465-8f19-9cc916d97b11", | + | | "event_type": "start_lease", | + | | "time": "2024-08-23T14:00:00.000000", | + | | "status": "UNDONE", | + | | "created_at": "2024-08-23 12:40:53", | + | | "updated_at": null | + | | } | + | id | d7779e56-2b78-4465-8f19-9cc916d97b11 | + | name | lease-1 | + | project_id | 09dd299a860b4933a2ebc271377e77a6 | + | reservations | { | + | | "id": "c1fb5090-e046-42a5-8287-2aa380a8d31a", | + | | "lease_id": "d7779e56-2b78-4465-8f19-9cc916d97b11", | + | | "resource_id": "41448036-4bd0-4b03-bd72-829ad636024b", | + | | "resource_type": "flavor:instance", | + | | "status": "pending", | + | | "missing_resources": false, | + | | "resources_changed": false, | + | | "created_at": "2024-08-23 12:40:53", | + | | "updated_at": "2024-08-23 12:40:53", | + | | "vcpus": 2, | + | | "memory_mb": 4096, | + | | "disk_gb": 40, | + | | "amount": 1, | + | | "affinity": null, | + | | "resource_properties": "{\"id\": \"3\", \"name\": \"m1.medium\", \"ram\": 4096, \"disk\": 40, \"swap\": \"\", \"OS-FLV-EXT- | + | | DATA:ephemeral\": 0, \"OS-FLV-DISABLED:disabled\": false, \"vcpus\": 2, \"os-flavor-access:is_public\": true, \"rxtx_factor\": 1.0, | + | | \"extra_specs\": {\"hw_rng:allowed\": \"True\"}}", | + | | "flavor_id": "c1fb5090-e046-42a5-8287-2aa380a8d31a", | + | | "aggregate_id": 6, | + | | "server_group_id": null | + | | } | + | start_date | 2024-08-23T14:00:00.000000 | + | status | PENDING | + | trust_id | 8023d766983a493c898991430493e81a | + | updated_at | 2024-08-23 12:40:53 | + | user_id | 3ecacbbc4acc467faa664ee26eea115d | + +--------------+-------------------------------------------------------------------------------------------------------------------------------------+ + +.. + +2. Check leases: + +.. sourcecode:: console + + # Using the blazar CLI + blazar lease-list + + # Using the openstack CLI + openstack reservation lease list + +.. + +Result: + +.. sourcecode:: console + + +--------------------------------------+---------+----------------------------+----------------------------+ + | id | name | start_date | end_date | + +--------------------------------------+---------+----------------------------+----------------------------+ + | d7779e56-2b78-4465-8f19-9cc916d97b11 | lease-1 | 2024-08-23T14:00:00.000000 | 2024-09-08T15:00:00.000000 | + +--------------------------------------+---------+----------------------------+----------------------------+ + +.. + +3. Use the leased resources +--------------------------- + +While the reservation you created is active you can see and use the flavor of your reservation. + +.. sourcecode:: console + + openstack flavor list + +.. + +Result: + +.. sourcecode:: console + + +--------------------------------------+--------------------------------------------------+-------+------+-----------+-------+-----------+ + | ID | Name | RAM | Disk | Ephemeral | VCPUs | Is Public | + +--------------------------------------+--------------------------------------------------+-------+------+-----------+-------+-----------+ + | 1 | m1.tiny | 512 | 1 | 0 | 1 | True | + | 2 | m1.small | 2048 | 20 | 0 | 1 | True | + | 3 | m1.medium | 4096 | 40 | 0 | 2 | True | + | 4 | m1.large | 8192 | 80 | 0 | 4 | True | + | 42 | m1.nano | 192 | 1 | 0 | 1 | True | + | 5 | m1.xlarge | 16384 | 160 | 0 | 8 | True | + | 84 | m1.micro | 256 | 1 | 0 | 1 | True | + | c1 | cirros256 | 256 | 1 | 0 | 1 | True | + | c1fb5090-e046-42a5-8287-2aa380a8d31a | reservation:c1fb5090-e046-42a5-8287-2aa380a8d31a | 4096 | 40 | 0 | 2 | False | + +--------------------------------------+--------------------------------------------------+-------+------+-----------+-------+-----------+ + +.. + +1. Create a server: Please specify the flavor of the reservation. + +.. sourcecode:: console + + openstack server create --flavor c1fb5090-e046-42a5-8287-2aa380a8d31a --image --network + +.. diff --git a/doc/source/cli/host-reservation.rst b/doc/source/cli/host-reservation.rst index 43c5c05fa..8a8740635 100644 --- a/doc/source/cli/host-reservation.rst +++ b/doc/source/cli/host-reservation.rst @@ -51,6 +51,49 @@ Result: .. +3. (Optional) Add extra capabilities to host to add other properties. These can + be used to filter hosts when creating a reservation. + +.. sourcecode:: console + + # Using the blazar CLI + blazar host-update --extra gpu=True compute-1 + + # Using the openstack CLI + openstack reservation host set --extra gpu=True compute-1 + +.. + +Result: + +.. sourcecode:: console + + Updated host: compute-1 + +.. + +Multiple ``--extra`` parameters can be included. They can also be specified in +``host-create``. Properties can be made private or public. By default, they +are public. + +.. sourcecode:: console + + # Using the blazar CLI + blazar host-capability-update gpu --private + + # Using the openstack CLI + openstack reservation host capability update gpu --private + +.. + +Result: + +.. sourcecode:: console + + Updated host extra capability: gpu + +.. + 2. Create a lease ----------------- @@ -128,6 +171,103 @@ Result: .. +3. Alternatively, create leases with resource properties. + First list properties. + +.. sourcecode:: console + + # Using the blazar CLI + blazar host-capability-list + + # Using the openstack CLI + openstack reservation host capability list + +.. + +Result: + +.. sourcecode:: console + + +----------+ + | property | + +----------+ + | gpu | + +----------+ + +.. + +List possible values for a property + +.. sourcecode:: console + + # Using the blazar CLI + blazar host-capability-show gpu + + # Using the openstack CLI + openstack reservation host capability show gpu + +.. + +Result: + +.. sourcecode:: console + + +-------------------+-------+ + | Field | Value | + +-------------------+-------+ + | capability_values | True | + | | False | + | private | False | + | property | gpu | + +-------------------+-------+ + +.. + +Create a lease. + +.. sourcecode:: console + + # Using the blazar CLI + blazar lease-create --physical-reservation min=1,max=1,resource_properties='["=", "$gpu", "True"]' --start-date "2020-06-08 12:00" --end-date "2020-06-09 12:00" lease-1 + + # Using the openstack CLI + openstack reservation lease create --reservation resource_type=physical:host,min=1,max=1,resource_properties='[">=", "$vcpus", "2"]' --start-date "2020-06-08 12:00" --end-date "2020-06-09 12:00" lease-1 + +.. + +Result: + +.. sourcecode:: console + + +---------------+---------------------------------------------------------------------------------------------------------------------------------------------+ + | Field | Value | + +---------------+---------------------------------------------------------------------------------------------------------------------------------------------+ + | action | | + | created_at | 2020-06-08 02:43:40 | + | end_date | 2020-06-09T12:00:00.000000 | + | events | {"status": "UNDONE", "lease_id": "6638c31e-f6c8-4982-9b98-d2ca0a8cb646", "event_type": "before_end_lease", "created_at": "2020-06-08 | + | | 02:43:40", "updated_at": null, "time": "2020-06-08T12:00:00.000000", "id": "420caf25-dba5-4ac3-b377-50503ea5c886"} | + | | {"status": "UNDONE", "lease_id": "6638c31e-f6c8-4982-9b98-d2ca0a8cb646", "event_type": "start_lease", "created_at": "2020-06-08 02:43:40", | + | | "updated_at": null, "time": "2020-06-08T12:00:00.000000", "id": "b9696139-55a1-472d-baff-5fade2c15243"} | + | | {"status": "UNDONE", "lease_id": "6638c31e-f6c8-4982-9b98-d2ca0a8cb646", "event_type": "end_lease", "created_at": "2020-06-08 02:43:40", | + | | "updated_at": null, "time": "2020-06-09T12:00:00.000000", "id": "ff9e6f52-db50-475a-81f1-e6897fdc769d"} | + | id | 6638c31e-f6c8-4982-9b98-d2ca0a8cb646 | + | name | lease-1 | + | project_id | 4527fa2138564bd4933887526d01bc95 | + | reservations | {"status": "pending", "lease_id": "6638c31e-f6c8-4982-9b98-d2ca0a8cb646", "resource_id": "8", "max": 1, "created_at": "2020-06-08 | + | | 02:43:40", "min": 1, "updated_at": null, "hypervisor_properties": "", "resource_properties": "[\"=\", \"$gpu\", \"True\"]", "id": | + | | "4d3dd68f-0e3f-4f6b-bef7-617525c74ccb", "resource_type": "physical:host"} | + | start_date | 2020-06-08T12:00:00.000000 | + | status | | + | status_reason | | + | trust_id | ba4c321878d84d839488216de0a9e945 | + | updated_at | | + | user_id | | + +---------------+---------------------------------------------------------------------------------------------------------------------------------------------+ + +.. + + 3. Use the leased resources --------------------------- diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst index 8306facb6..85e176b62 100644 --- a/doc/source/cli/index.rst +++ b/doc/source/cli/index.rst @@ -6,6 +6,7 @@ Command-Line Interface Reference host-reservation instance-reservation + flavor-based-instance-reservation floatingip-reservation Two command-line interfaces exist: one as a standalone ``blazar`` client and diff --git a/doc/source/cli/instance-reservation.rst b/doc/source/cli/instance-reservation.rst index 45f6ccc01..c85982dd2 100644 --- a/doc/source/cli/instance-reservation.rst +++ b/doc/source/cli/instance-reservation.rst @@ -11,10 +11,6 @@ The following packages should be installed: * blazar-nova * python-blazarclient -The following four scheduler filters should be configured in nova.conf: - -* BlazarFilter - 1. Add hosts into the freepool ------------------------------ @@ -170,3 +166,20 @@ Result: openstack server create --flavor db83d6fd-c69c-4259-92cf-012db2e55a58 --image --network .. + + +Affinity +-------- + +A lease can be created with the optional ``--affinity`` parameter. This +provides the following behavior: + +* ``affinity=True``: Instances will be deployed on the same host for this + reservation, by adding them to a server group with an affinity policy. + +* ``affinity=False``: Instances will be deployed on different hosts for this + reservation, by adding them to a server group with an anti-affinity policy. + +* ``affinity=None`` (default): Instances can be deployed on any host, + regardless of other instances in this reservation. No server group is + created. diff --git a/doc/source/configuration/nova-conf.rst b/doc/source/configuration/nova-conf.rst index c915291f3..079b222e3 100644 --- a/doc/source/configuration/nova-conf.rst +++ b/doc/source/configuration/nova-conf.rst @@ -9,6 +9,6 @@ Please add the following lines to the nova.conf configuration file: [filter_scheduler] available_filters = nova.scheduler.filters.all_filters available_filters = blazarnova.scheduler.filters.blazar_filter.BlazarFilter - enabled_filters=RetryFilter,AvailabilityZoneFilter,RamFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,AggregateInstanceExtraSpecsFilter,AggregateMultiTenancyIsolation,ServerGroupAntiAffinityFilter,BlazarFilter + enabled_filters = AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter,BlazarFilter .. diff --git a/doc/source/contributor/contribution.rst b/doc/source/contributor/contribution.rst index 03bbaea42..66be7eab8 100644 --- a/doc/source/contributor/contribution.rst +++ b/doc/source/contributor/contribution.rst @@ -36,8 +36,8 @@ How to keep in touch with community subject. * All questions may be asked on our IRC channel #openstack-blazar on `OFTC `_. -* We also have weekly meetings on #openstack-meeting-alt. Please check - `meeting details `_. +* We also have bi-weekly meetings on #openstack-blazar. Please check `meeting + details `_. Your first commit to Blazar ---------------------------- diff --git a/doc/source/index.rst b/doc/source/index.rst index 09bd4294f..6067ffef3 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,9 +1,9 @@ Blazar, the OpenStack Reservation Service ========================================= -Blazar is an OpenStack service to provide resource reservations in the -OpenStack cloud for different resource types - both virtual (instances, -volumes, stacks) and physical (hosts). +Blazar is the *Resource Reservation Service* for OpenStack. + +If you are new to Blazar, please start with the :ref:`Introduction`. Installation Guide ------------------ diff --git a/doc/source/install/install-without-devstack.rst b/doc/source/install/install-without-devstack.rst index 2fbc89edc..35c37b7ac 100644 --- a/doc/source/install/install-without-devstack.rst +++ b/doc/source/install/install-without-devstack.rst @@ -90,7 +90,7 @@ Next you need to configure Nova. Please add the following lines to nova.conf fil [filter_scheduler] available_filters = nova.scheduler.filters.all_filters available_filters = blazarnova.scheduler.filters.blazar_filter.BlazarFilter - enabled_filters=RetryFilter,AvailabilityZoneFilter,RamFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,AggregateInstanceExtraSpecsFilter,AggregateMultiTenancyIsolation,ServerGroupAntiAffinityFilter,BlazarFilter + enabled_filters = AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,SameHostFilter,DifferentHostFilter,BlazarFilter .. diff --git a/doc/source/user/architecture.rst b/doc/source/user/architecture.rst index 7816f224a..435752bcc 100644 --- a/doc/source/user/architecture.rst +++ b/doc/source/user/architecture.rst @@ -2,74 +2,141 @@ Blazar architecture ==================== -Blazar design can be described by following diagram: +Blazar's architecture is described by the following diagram: .. image:: ../images/blazar-architecture.png :width: 700 px :scale: 99 % :align: left -**blazar-client** - provides the opportunity to communicate with Blazar via -*REST API* (blazar-api service). +**blazar-client** - provides a Python implementation of communication with +Blazar API (``blazar-api`` service). It works as a library, a standalone +``blazar`` CLI client, as well as a plugin of the ``openstack`` client. -**blazar-api** - waits for the REST calls from the outside world to redirect -them to the manager. blazar-api communicates with blazar-manager via RPC. -Runs as a separated process. +**blazar-api** - waits for REST API calls from the outside world to redirect +them to the manager. ``blazar-api`` communicates with ``blazar-manager`` +via RPC. **blazar-manager** - implements all logic and operations with leases, reservations and events. Communicates with Blazar DB and stores there data -structure of connected leases, reservations (both physical and virtual) and -events. blazar-manager service is responsible for running events created for -lease and process all actions that should be done this moment. Manager uses -resource-plugins to work with concrete resources (instances, volumes, compute -hosts). +on leases, reservations and events. +``blazar-manager`` service is responsible for handling events created for +leases and running all relevant actions. ``blazar-manager`` uses +resource-plugins to work with other services' resources. -**resource-plugin** - responsible for exact actions to do with reserved -resources (VMs, volumes, etc.) When working knows only about resource ID and -token to use. All resource plugins work in the same process as blazar-manager. +**resource-plugin** - responsible for exact actions to apply to other services' +resources. All resource plugins reside in the same process as +``blazar-manager``. + +Nova resource reservation ++++++++++++++++++++++++++ + +Blazar's integration with Nova is based on the following components working +in tandem: + +* Nova + + * Nova host aggregates are used to control which hosts have their access + managed by Blazar. + The canonical name for the Blazar-owned host aggregate is ``freepool``. + + * Nova flavors are used for instance reservations. + +* Blazar-Nova + + * The ``blazar-nova`` package offers the ``BlazarFilter`` filter for + nova-scheduler. ``BlazarFilter`` ensures that Nova does not schedule + regular (non-reservation-belonging) instances on Blazar-owned hosts + (unless preemption is enabled) and that host lease boundaries are respected. + +* Placement + + * Nested resource providers are used to control which hosts are managed by + Blazar. + + * Reservation classes are created and nested resource providers' inventories + are updated for instance reservations. + +It is worth noting that Blazar integrations are one-way, i.e. the other services +never call Blazar. Even the ``BlazarFilter`` operates in such a way that all +required data is present in Nova and Placement. + +.. note:: + + Only the compute host reservation is compatible with preemptible instances + at the moment. + +Compute host reservation +------------------------ + +Compute host reservation is analogous to a dedicated virtualisation host +offering. The user asks for a certain number of hosts with specified +characteristics, such as: + +* the region, +* the availability zone, +* the host capabilities extra specs, +* the number of CPU cores, +* the amount of available RAM, +* the amount of available disk space. + +Matching hosts are reserved for the sole use in the user's project. +Other projects will not share the same hosts during the lease period. Virtual instance reservation ---------------------------- -**Note** virtual instance reservation feature is not available in current -release. Expected to be available in the future (`bug tracker`_). +Virtual instance reservation offers a more granular compute resource reservation +that does not book entire hypervisors but instead allows using Blazar-owned +hosts via special time-limited flavors set up by Blazar. The user asks for +a certain number of instances with specific flavor characteristics, +such as: -.. _bug tracker: https://blueprints.launchpad.net/blazar/+spec/new-instance-reservation +* the number of provided CPU cores, +* the amount of provided RAM, +* the amount of provided disk space, +* affinity rule. -Virtual instance reservation mostly looks like usual instance booting for user -- he/she only passes special hints to Nova containing information about future -lease - lease start and end dates, its name, etc. Special Nova API extensions -parse these parameter and use them to call Blazar, passing to it ID of just -created instance. If there is a need to reserve all instances in cloud (like in -developer labs to automate process of resource reclaiming), default reservation -extension might be used. By default it starts lease at the moment of request -and gives it one month of lifetime. +When the lease is active, a dedicated flavor is presented to the leasee. +No other project may use this flavor. Blazar ensures, albeit in a best-effort +manner, that supporting compute resources are reserved, e.g., it will not allow +for oversubscribing with multiple reservation types (hosts supporting the +virtual instance reservation cannot also be targeted for compute host +reservation at the same time). -During the time lease has not started yet, instance will be shelved. +Flavor-based instance reservation +--------------------------------- -Compute host reservation +Flavor-based instance reservation is similar in design to virtual instance +reservation. However, instead of accepting specific amounts of resources as +arguments, the reservation is based on an existing Nova flavor which must be +accessible to the user. + +.. note:: + + The implementation of flavor-based instance reservation is still incomplete. + Only a limited number of flavor extra specs is supported. Virtual instance + reservations and flavor-based instance reservations cannot yet be created on + the same resources. Affinity rules and resource traits are not yet + supported. + +Neutron resource reservation +++++++++++++++++++++++++++++ + +Apart from compute resources of Nova, Blazar allows to reserve certain Neutron +resources. At the moment, these are only floating IPs. + +The Neutron integration thus far does not require changes to the Neutron +environment. The Blazar interaction looks to Neutron like any other service +client interaction. + +Floating IPs reservation ------------------------ -Now process of compute hosts reserving contains two steps: - -* admin marks hosts from common pool as possible to be reserved. That is - implemented by moving these hosts to special aggregate; -* user asks for reserving of host with specified characteristics like: - - * the region - * the availability zone - * the host capabilities extra specs (scoped and non-scoped format is - accepted) - * the number of CPU cores - * the amount of free RAM - * the amount of free disk space - * the number of hosts - -Technically speaking, resource ID here will be not host ID, because there might -be many of them wanted. Resource here will be new aggregate containing reserved -hosts. The time lease starts, user may use reserved compute capacity to run -his/her instances on it passing special scheduler hint to Nova. When host is -reserved, it's not used for usual instance running, it might be used only when -lease starts and only by passing reservation ID to Nova. That is implemented -using special Nova Scheduler filter, that passes reservation ID to Blazar and -checks if user really can use reserved compute capacity. +Blazar admin may register floating IPs with Blazar which can then later be +leased to end users. End users request floating IPs from a chosen network +and they will be created in user's project once the lease starts. +The allowed floating IPs *must not* exist in subnet's allocation pools. +Blazar will validate this *only once* when admin registers the floating IP +with Blazar. The integration *will break* if admin then adds the same floating +IP to the allocation pool as Blazar will try to own it and fail. diff --git a/doc/source/user/introduction.rst b/doc/source/user/introduction.rst index dbcc56475..bbaca153f 100644 --- a/doc/source/user/introduction.rst +++ b/doc/source/user/introduction.rst @@ -1,107 +1,55 @@ +.. _Introduction: + ============ Introduction ============ -Blazar is a resource reservation service for OpenStack. -Idea of creating Blazar originated with two different use cases: +Blazar is the *Resource Reservation Service* for OpenStack. + +The idea of creating Blazar originated with two different use cases: * Compute host reservation (when user with admin privileges can reserve hardware resources that are dedicated to the sole use of a project) -* Virtual machine (instance) reservation (when user may ask reservation service - to provide him working VM not necessarily now, but also in the future) +* Virtual machine (instance) reservation (when user may ask Blazar + to provide them working VM not necessarily now, but also in the future) -Now these ideas have been transformed to more general view: with Blazar, user -can request the resources of cloud environment to be provided ("leased") to his +These ideas have been transformed to a more general view: with Blazar, user +can request the resources of cloud environment to be provided ("leased") to their project for specific amount of time, immediately or in the future. -Both virtual (Instances, Volumes, Networks) and hardware (full hosts with -specific characteristics of RAM, CPU, etc) resources can be allocated via -"lease". +Currently, Blazar supports reservations of: + +* Nova resources: + + * Compute hosts / hypervisors + * Instances / servers + +* Neutron resources: + + * Floating IPs -In terms of benefits added, Resource Reservation Service will: +In terms of benefits added, Blazar: -* improve visibility of cloud resources consumption (current and planned for +* improves visibility of cloud resources consumption (current and planned for future); -* enable cloud resource planning based on current and future demand from end +* enables cloud resource planning based on current and future demand from end users; -* automate the processes of resource allocation and reclaiming; -* provide energy efficiency for physical hosts (both compute and storage ones); -* potentially provide leases as billable items for which customers can be - charged a flat fee or a premium price depending on the amount of reserved cloud - resources and their usage. +* automates the processes of resource allocation and reclaiming. Glossary of terms ----------------- -**Reservation** is an allocation of certain cloud resource (Nova instance, Cinder -volume, compute host, etc.) to a particular project. Speaking about virtual -reservations, we may have not only simple, solid ones (like already mentioned -instances and volumes), but also complex ones - like Heat stacks and Savanna -clusters. Reservation is characterized by status, resource type, identifier -and lease it belongs to. +**Reservation** is an allocation of cloud resources to a particular project. +Main properties of a reservation are its status, resource type, identifier and +the lease it belongs to. **Lease** is a negotiation agreement between the provider (Blazar, using OpenStack resources) and the consumer (user) where the former agrees to make some kind of -resources (both virtual and physical) available to the latter, based on a set of -lease terms presented by the consumer. Here lease may be described as a contract -between user and reservation service about cloud resources to be provided right -now or later. Technically speaking, lease is a group of reservations granted to -a particular project upon request. Lease is characterized by start time, end -time, set of individual reservations and associated events. - -**Event** is simply something that may happen to a lease. In most simple case, event -might describe lease start and lease end. Also it might be a notification to user -(e.g. about soon lease expiration) and some extra actions. - -Rationale ---------- - -Blazar is created to: - -* manage cloud resources not only right now, but also in the future; -* have dedicated resources for a certain amount of time; -* prepare for the peak loads and perform capacity planning; -* optimize energy consumption. - -Lease types (concepts) ----------------------- - -* **Immediate reservation**. Resources are provisioned immediately (like VM - boot or moving host to reserved user aggregate) or not at all. If request can - be fulfilled, lease is created and **success** status is returned. Lease - should be marked as **active** or **to_be_started**. Otherwise (if - request resource cannot be provisioned right now) failure status for this - request should be returned. -* **Reservation with retries**. Mostly looks like previous variant, but in case - of failure, user may want to have several (configurable number) retries to - process lease creation action. In this case request will be processed till - that will be possible to create lease but not more than set in configuration - file number of times. -* **Best-effort reservation**. Also might have place if lease creation request - cannot be fulfilled immediately. Best-effort mechanism starts something like - scavenger hunt trying to find resources for reservations. For compute hosts - reservation that makes much sense, because in case there are instances - belonging to other project on eligible hosts, and without them there will be - possible to reserve these hosts, Blazar may start instances migration. - This operation can be timely and fairly complex and so different strategies - may be applied depending on heuristic factors such as the number, type and - state of the instances to be migrated. Also Blazar should assert that there - are at least enough potential candidates for the migration prior to starting - the actual migration. If Blazar decides to start migration, it returns - **success** state and marks lease as **in_progress**, otherwise - - **failure**. If this 'hunting' ends successfully before configurable - timeout has passed, lease should be marked as **active**, otherwise its - status is set to **timedout**. -* **Delayed resource acquiring** or **scheduled reservation**. In this - reservation type, lease is created successfully if Blazar thinks there will - be enough resources to process provisioning later (otherwise this request - returns **failure** status). Lease is marked as **inactive** till all - resources will be actually provisioned. That works pretty nice and - predictable speaking about compute hosts reservation (because hosts as - resources are got not from common cloud pool, but from admin defined pool). - So it is possible for Blazar to predict these physical resources usage and use - that information during lease creation. If we speak about virtual reservations, - here situation is more complicated, because all resources are got from common - cloud resources pool, and Blazar cannot guarantee there will be enough - resources to provision them. In this failure case lease state will be marked - as **error** with appropriate explanation. \ No newline at end of file +cloud resources available to the latter, based on a set of lease terms presented +by the consumer. Technically speaking, lease is a group of reservations granted to +a particular project upon request. Main properties of a lease are its start time, end +time, set of reservations and set of events. + +**Event** is something that may happen to a lease. In the simplest case, an event +might describe lease start and lease end. It might also be a notification to user +(e.g. about an upcoming lease expiration). diff --git a/etc/blazar/blazar-config-generator.conf b/etc/blazar/blazar-config-generator.conf index 24536793b..4615fbee8 100644 --- a/etc/blazar/blazar-config-generator.conf +++ b/etc/blazar/blazar-config-generator.conf @@ -8,5 +8,6 @@ namespace = oslo.log namespace = oslo.messaging namespace = oslo.middleware namespace = oslo.policy +namespace = oslo.service.periodic_task namespace = oslo.service.service namespace = keystonemiddleware.auth_token diff --git a/lower-constraints.txt b/lower-constraints.txt index 27634cd8c..1999c8fbd 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -75,11 +75,14 @@ pylint==1.9.1 pyparsing==2.2.0 python-dateutil==2.7.0 python-editor==1.0.3 +python-ironicclient==2.4.0 python-keystoneclient==3.8.0 +python-manilaclient==1.29.0 python-mimeparse==1.6.0 python-neutronclient==6.0.0 python-novaclient==9.1.0 python-subunit==1.2.0 +python-zunclient==3.5.1 pytz==2018.3 PyYAML==5.1 reno==3.1.0 @@ -100,7 +103,7 @@ statsd==3.2.2 stevedore==1.20.0 Tempita==0.5.2 tenacity==4.9.0 -testrepository==0.0.18 +stestr==2.0.0 testresources==2.0.1 testscenarios==0.4 testtools==2.2.0 diff --git a/releasenotes/notes/bug-1674524-914e4bb2ac1704e1.yaml b/releasenotes/notes/bug-1674524-914e4bb2ac1704e1.yaml new file mode 100644 index 000000000..ff6c1bfb4 --- /dev/null +++ b/releasenotes/notes/bug-1674524-914e4bb2ac1704e1.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + It was not previously possible to remove host extra capabilities. + Now, when updating a host's capabilities, specifying a value of + ``None`` will remove the capability (``null`` in JSON.) diff --git a/releasenotes/notes/bug-1786031-836c6d6acae08403.yaml b/releasenotes/notes/bug-1786031-836c6d6acae08403.yaml new file mode 100644 index 000000000..100680129 --- /dev/null +++ b/releasenotes/notes/bug-1786031-836c6d6acae08403.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Updating a lease with invalid parameters (for example, specifying an + invalid date range) now has no effect on the lease, where previously this + would cause the lease to be set to ERROR status. For more details, see `bug + 1786031 `_. diff --git a/releasenotes/notes/bug-1881162-ebe012fcc7176594.yaml b/releasenotes/notes/bug-1881162-ebe012fcc7176594.yaml new file mode 100644 index 000000000..8ad56c91b --- /dev/null +++ b/releasenotes/notes/bug-1881162-ebe012fcc7176594.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Allows users of multiple Keystone domains to create leases; previously only users + and projects in the default domain could use Blazar. diff --git a/releasenotes/notes/bug-1957761-8b126a392c0c79ee.yaml b/releasenotes/notes/bug-1957761-8b126a392c0c79ee.yaml new file mode 100644 index 000000000..256cba283 --- /dev/null +++ b/releasenotes/notes/bug-1957761-8b126a392c0c79ee.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes failure to update reservations when the ``resource_type`` parameter + is not provided in the API request. For more details, see `bug 1957761 + `_. diff --git a/releasenotes/notes/bug-1958307-63bf308ca6a97068.yaml b/releasenotes/notes/bug-1958307-63bf308ca6a97068.yaml new file mode 100644 index 000000000..aaea7ae0f --- /dev/null +++ b/releasenotes/notes/bug-1958307-63bf308ca6a97068.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes result of the List Allocations API for reservations with multiple + physical hosts. For more details, see `bug 1958307 + `_. diff --git a/releasenotes/notes/drop-python-3-6-and-3-7-2340dc9b36dc7ac5.yaml b/releasenotes/notes/drop-python-3-6-and-3-7-2340dc9b36dc7ac5.yaml new file mode 100644 index 000000000..3d5e4e3d3 --- /dev/null +++ b/releasenotes/notes/drop-python-3-6-and-3-7-2340dc9b36dc7ac5.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Python 3.6 & 3.7 support has been dropped. The minimum version of Python now + supported is Python 3.8. \ No newline at end of file diff --git a/releasenotes/notes/external-enforcement-filter-6e8fee75580fc169.yaml b/releasenotes/notes/external-enforcement-filter-6e8fee75580fc169.yaml new file mode 100644 index 000000000..0425a8f4b --- /dev/null +++ b/releasenotes/notes/external-enforcement-filter-6e8fee75580fc169.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add a usage enforcement filter delegating decisions to an external HTTP + service. This new filter is called ``ExternalServiceFilter``. diff --git a/releasenotes/notes/fix-host-randomization-bcab5276ef6199e6.yaml b/releasenotes/notes/fix-host-randomization-bcab5276ef6199e6.yaml new file mode 100644 index 000000000..e6ed412fc --- /dev/null +++ b/releasenotes/notes/fix-host-randomization-bcab5276ef6199e6.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes functionality of host randomization feature. + `LP#2099927 `__ diff --git a/releasenotes/notes/flavor-based-instance-reservation-5a375363e8011775.yaml b/releasenotes/notes/flavor-based-instance-reservation-5a375363e8011775.yaml new file mode 100644 index 000000000..b4ce855da --- /dev/null +++ b/releasenotes/notes/flavor-based-instance-reservation-5a375363e8011775.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Add a new resource type ``flavor:instance`` to reserve instances based on + existing flavors. This is implemented by a new plugin named + ``flavor.instance.plugin``. Note that this feature is available as a + preview but is not yet complete. See `documentation + `__ for + more details. diff --git a/releasenotes/notes/internal-endpoint-interface-748511b8b25455c6.yaml b/releasenotes/notes/internal-endpoint-interface-748511b8b25455c6.yaml new file mode 100644 index 000000000..08d6acdaf --- /dev/null +++ b/releasenotes/notes/internal-endpoint-interface-748511b8b25455c6.yaml @@ -0,0 +1,14 @@ +--- +fixes: + - | + Requests to other OpenStack services are now made using the internal + interface instead of the public interface. This fixes issues when public + endpoints are not reachable by Blazar. For more details, see `bug 1960335 + `_. + + This can be configured with the following options: + + * ``[DEFAULT]/endpoint_type`` to configure the Keystone endpoint type + * ``[neutron]/endpoint_type`` to configure the Neutron endpoint type + * ``[nova]/endpoint_type`` to configure the Nova endpoint type + * ``[placement]/endpoint_type`` to configure the Placement endpoint type diff --git a/releasenotes/notes/network-segment-reservation-39eab8648d73b257.yaml b/releasenotes/notes/network-segment-reservation-39eab8648d73b257.yaml new file mode 100644 index 000000000..cc2d1e817 --- /dev/null +++ b/releasenotes/notes/network-segment-reservation-39eab8648d73b257.yaml @@ -0,0 +1,14 @@ +--- +prelude: > + Adds support for reserving VLAN or VXLAN network segments. +features: + - | + Site operators can now add network segments to Blazar's reservable + inventory. Users can create a lease that includes one or more reservations + for a network segment. While the user's lease is active, Neutron networks + for any reserved segments will be associated with the user's project. +other: + - | + When tearing down a network reservation, the Neutron network is deleted. + As part of this, any instances connected to the network will have their ports + deleted, and will lose connectivity if not attached to any other networks. diff --git a/releasenotes/notes/option-for-randomising-host-selection-101e95baab67dd88.yaml b/releasenotes/notes/option-for-randomising-host-selection-101e95baab67dd88.yaml new file mode 100644 index 000000000..c4bcca7ab --- /dev/null +++ b/releasenotes/notes/option-for-randomising-host-selection-101e95baab67dd88.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds optional randomness to physical host allocation. This is disabled by + default and can be enabled with the configuration option + ``[physical:host]/randomize_host_selection``. diff --git a/releasenotes/notes/resource-property-discovery-42df197a1a49bd76.yaml b/releasenotes/notes/resource-property-discovery-42df197a1a49bd76.yaml new file mode 100644 index 000000000..90e57a141 --- /dev/null +++ b/releasenotes/notes/resource-property-discovery-42df197a1a49bd76.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds a host resource property discovery API, which allows users to + enumerate what properties are available, and current property values. + Properties can be made private by operators, which filters them from the + public list. diff --git a/releasenotes/notes/stop-using-trusts-to-get-context-acf1e8f0c93698f0.yaml b/releasenotes/notes/stop-using-trusts-to-get-context-acf1e8f0c93698f0.yaml new file mode 100644 index 000000000..c64d4a70a --- /dev/null +++ b/releasenotes/notes/stop-using-trusts-to-get-context-acf1e8f0c93698f0.yaml @@ -0,0 +1,11 @@ +--- +fixes: + - | + Fixes an issue wherein operations modifying existing leases (update or + delete) and events such as ``start_lease`` or ``end_lease`` would fail + after the user who created the lease was disabled or removed from the + project. For more details, see `bug 1712381 + `_. +deprecations: + - | + Context information has been removed from all notification messages. diff --git a/releasenotes/notes/update-affinity-behaviour-a03231e461bd2b21.yaml b/releasenotes/notes/update-affinity-behaviour-a03231e461bd2b21.yaml new file mode 100644 index 000000000..e87ae7528 --- /dev/null +++ b/releasenotes/notes/update-affinity-behaviour-a03231e461bd2b21.yaml @@ -0,0 +1,14 @@ +--- +upgrade: + - | + Instance reservations will no longer create a server group when the + ``affinity`` parameter is not set. This means that instances can be + deployed on any host. To achieve the old behaviour, you must now explicity + set ``--affinity=false``. + `LP#2071832 `__ +fixes: + - | + Instance reservations will no longer create a server group when the + ``affinity`` parameter is not set. This means that instances can be + deployed on any host. + `LP#2071832 `__ diff --git a/releasenotes/notes/usage-enforcement-f997ce618f542104.yaml b/releasenotes/notes/usage-enforcement-f997ce618f542104.yaml index 897154040..597e4e8ab 100644 --- a/releasenotes/notes/usage-enforcement-f997ce618f542104.yaml +++ b/releasenotes/notes/usage-enforcement-f997ce618f542104.yaml @@ -1,6 +1,8 @@ --- features: - | - A filter-based usage enforcement framework is introduced in this release. - Enforcement filters allow operators to define lease constraints. The first - filter introduced in this release restricts maximum lease duration. + Usage enforcement filters are introduced in this release. The enforcement + filters allow operators to define constraints lease constrains in + blazar.conf. The first two filters introduced in these release are a + maximum lease length filter and a filter for calling an external service + where usage policies can be defined. diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index 8e453edc8..fa7a94b28 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -58,7 +58,7 @@ master_doc = 'index' # General information about the project. -copyright = '2017, Blazar developers' +copyright = '2013-2022, Blazar developers' # Release notes are version independent. # The short X.Y version. diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index ce08dfe6f..1ee612d0a 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -8,6 +8,8 @@ Contents :maxdepth: 2 unreleased + zed + yoga xena wallaby victoria diff --git a/releasenotes/source/yoga.rst b/releasenotes/source/yoga.rst new file mode 100644 index 000000000..7cd5e908a --- /dev/null +++ b/releasenotes/source/yoga.rst @@ -0,0 +1,6 @@ +========================= +Yoga Series Release Notes +========================= + +.. release-notes:: + :branch: stable/yoga diff --git a/releasenotes/source/zed.rst b/releasenotes/source/zed.rst new file mode 100644 index 000000000..9608c05e4 --- /dev/null +++ b/releasenotes/source/zed.rst @@ -0,0 +1,6 @@ +======================== +Zed Series Release Notes +======================== + +.. release-notes:: + :branch: stable/zed diff --git a/requirements.txt b/requirements.txt index 4cf37d635..9673f21d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +# Requirements lower bounds listed here are our best effort to keep them up to +# date but we do not test them so no guarantee of having them all correct. If +# you find any incorrect lower bounds, let us know or propose a fix. + # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. @@ -9,6 +13,7 @@ Flask>=1.0.2 # BSD iso8601>=0.1.11 # MIT keystoneauth1>=3.13.0 # Apache-2.0 keystonemiddleware>=4.17.0 # Apache-2.0 +kubernetes>=10.0.1 # Apache-2.0 microversion-parse>=0.2.1 # Apache-2.0 oslo.concurrency>=3.26.0 # Apache-2.0 oslo.config>=6.8.0 # Apache-2.0 @@ -18,16 +23,22 @@ oslo.i18n>=3.15.3 # Apache-2.0 oslo.log>=3.36.0 # Apache-2.0 oslo.messaging>=5.29.0 # Apache-2.0 oslo.middleware>=3.31.0 # Apache-2.0 -oslo.policy>=3.6.0 # Apache-2.0 +oslo.policy>=1.30.0 # Apache-2.0 oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 oslo.service>=1.34.0 # Apache-2.0 -oslo.upgradecheck>=1.3.0 # Apache-2.0 -oslo.utils>=4.5.0 # Apache-2.0 +oslo.upgradecheck>=0.1.0 # Apache-2.0 +oslo.utils>=3.33.0 # Apache-2.0 +python-ironicclient>=1.11.0 # Apache-2.0 +python-manilaclient>=1.29.0 # Apache-2.0 python-neutronclient>=6.0.0 # Apache-2.0 python-novaclient>=9.1.0 # Apache-2.0 +python-zunclient>=3.5.1 # Apache-2.0 netaddr>=0.7.18 # BSD +python-heatclient>=1.16.2 # Apache-2.0 python-keystoneclient>=3.8.0 # Apache-2.0 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD +sqlalchemy-migrate>=0.11.0 # Apache-2.0 +redis>=2.10.5 requests>=2.18.4 # Apache-2.0 retrying>=1.3.3,!=1.3.0 # Apache-2.0 Routes>=2.3.1 # MIT diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..e58635bb4 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,22 @@ +line-length = 79 + +[lint] +select = [ + "E", # pycodestyle (error) + "F", # pyflakes + "G", # flake8-logging-format + "LOG", # flake8-logging +] + + +ignore = [ + "E402", # many files monkeypatch eventlet, so imports not at top of file + "E501", # line too long + "E741", # Ambiguous variable name + "F401", # imported but unused + "F841", # Local variable is assigned to but never used + "G001", # logging statement uses str.format + "G002", # Logging statement uses `%` + "G004", # logging statement uses f-string + "G010", # Logging statement uses `warn` instead of `warning` +] diff --git a/setup.cfg b/setup.cfg index a46a00654..812af739c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,15 +3,14 @@ name = blazar summary = Reservation Service for OpenStack clouds description_file = README.rst license = Apache Software License -python_requires = >=3.6 +python_requires = >=3.8 classifiers = Programming Language :: Python Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 Environment :: OpenStack Development Status :: 3 - Alpha Framework :: Setuptools Plugin @@ -34,15 +33,28 @@ console_scripts = blazar-rpc-zmq-receiver=blazar.cmd.rpc_zmq_receiver:main blazar-manager=blazar.cmd.manager:main blazar-status=blazar.cmd.status:main + blazar-usage=blazar.cmd.usage:main blazar.resource.plugins = + device.plugin=blazar.plugins.devices.device_plugin:DevicePlugin dummy.vm.plugin=blazar.plugins.dummy_vm_plugin:DummyVMPlugin physical.host.plugin=blazar.plugins.oshosts.host_plugin:PhysicalHostPlugin + network.plugin=blazar.plugins.networks.network_plugin:NetworkPlugin virtual.instance.plugin=blazar.plugins.instances.instance_plugin:VirtualInstancePlugin virtual.floatingip.plugin=blazar.plugins.floatingips.floatingip_plugin:FloatingIpPlugin + flavor.instance.plugin=blazar.plugins.flavor.flavor_plugin:FlavorPlugin + +blazar.device.driver.plugins = + k8s.plugin=blazar.plugins.devices.k8s_plugin:K8sPlugin + zun.plugin=blazar.plugins.devices.zun_plugin:ZunPlugin + +blazar.network.usage.type.plugins = + storage.plugin=blazar.plugins.networks.storage_plugin:StoragePlugin blazar.api.v1.extensions = leases=blazar.api.v1.leases.v1_0:get_rest + device.plugin=blazar.api.v1.devices.v1_0:get_rest + network.plugin=blazar.api.v1.networks.v1_0:get_rest physical.host.plugin=blazar.api.v1.oshosts.v1_0:get_rest virtual.floatingip.plugin=blazar.api.v1.floatingips.v1_0:get_rest diff --git a/test-requirements.txt b/test-requirements.txt index 08c90830d..516249df1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,8 +5,9 @@ hacking>=3.0.1,<3.1.0 # Apache-2.0 ddt>=1.0.1 # MIT fixtures>=3.0.0 # Apache-2.0/BSD -testrepository>=0.0.18 # Apache-2.0/BSD +stestr>=2.0.0 # Apache-2.0 testscenarios>=0.4 # Apache-2.0/BSD testtools>=2.2.0 # MIT coverage!=4.4,>=4.0 # Apache-2.0 oslotest>=3.2.0 # Apache-2.0 +WebTest>=2.0.29 # MIT, for testing with pecan diff --git a/tox.ini b/tox.ini index c03c378da..b2fd97ab9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,22 +1,22 @@ [tox] -envlist = py36,py38,pep8 +envlist = py310,py39,py38,pep8 minversion = 3.18.0 -skipsdist = True ignore_basepython_conflict = True [testenv] -basepython = python3 +basepython = python310 usedevelop = True allowlist_externals = rm deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.1} -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt + setuptools<82.0.0 setenv = VIRTUAL_ENV={envdir} DISCOVER_DIRECTORY=blazar/tests PYTHONHASHSEED=0 commands = - lockutils-wrapper python setup.py testr --slowest --testr-args="{posargs}" + stestr run --slowest {posargs} sitepackages = False @@ -25,22 +25,25 @@ setenv = {[testenv]setenv} PYTHON=coverage run --source blazar --parallel-mode commands = - coverage erase - lockutils-wrapper python setup.py testr --testr-args="{posargs}" - coverage combine - coverage html -d cover - coverage xml -o cover/coverage.xml - coverage report + coverage erase + stestr run {posargs} + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + coverage report [testenv:pep8] +skip_install = True commands = flake8 {posargs} [testenv:venv] commands = {posargs} [testenv:docs] -deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} +deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.1} + -r{toxinidir}/requirements.txt -r{toxinidir}/doc/requirements.txt + setuptools<82.0.0 commands = rm -rf doc/html doc/build rm -rf doc/source/apidoc doc/source/api @@ -79,7 +82,8 @@ paths = ./blazar/hacking [testenv:pylint] deps = -r{toxinidir}/requirements.txt - pylint==1.9.1 + pylint + setuptools<82.0.0 commands = pylint blazar [testenv:releasenotes] @@ -98,12 +102,6 @@ commands = commands = oslopolicy-sample-generator --config-file etc/blazar/blazar-policy-generator.conf -[testenv:lower-constraints] -deps = - -c{toxinidir}/lower-constraints.txt - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - # This environment can be used to quickly validate that all needed system # packages required to successfully execute test targets are installed [testenv:bindep]