diff --git a/src/coldfront_plugin_cloud/attributes.py b/src/coldfront_plugin_cloud/attributes.py index fca38d8..c421f63 100644 --- a/src/coldfront_plugin_cloud/attributes.py +++ b/src/coldfront_plugin_cloud/attributes.py @@ -1,3 +1,23 @@ +from dataclasses import dataclass + + +@dataclass +class CloudResourceAttribute: + """Class for configuring Cloud Resource Attributes""" + name: str + type: str = 'Text' + + +@dataclass +class CloudAllocationAttribute: + """Class for configuring Cloud Allocation Attributes""" + name: str + type: str = 'Int' + has_usage: bool = False + is_private: bool = False + is_changeable: bool = True + + RESOURCE_AUTH_URL = 'Identity Endpoint URL' RESOURCE_ROLE = 'Role for User in Project' @@ -10,58 +30,40 @@ RESOURCE_EULA_URL = "EULA URL" -RESOURCE_ATTRIBUTES = { - RESOURCE_AUTH_URL: { - 'type': 'Text', - }, - RESOURCE_FEDERATION_PROTOCOL: { - 'type': 'Text', - }, - RESOURCE_IDP: { - 'type': 'Text', - }, - RESOURCE_PROJECT_DOMAIN: { - 'type': 'Text', - }, - RESOURCE_ROLE: { - 'type': 'Text', - }, - RESOURCE_USER_DOMAIN: { - 'type': 'Text', - }, - RESOURCE_EULA_URL: { - 'type': 'Text', - }, - RESOURCE_DEFAULT_PUBLIC_NETWORK: { - 'type': 'Text', - }, - RESOURCE_DEFAULT_NETWORK_CIDR: { - 'type': 'Text', - }, -} +RESOURCE_ATTRIBUTES = [ + CloudResourceAttribute(name=RESOURCE_AUTH_URL), + CloudResourceAttribute(name=RESOURCE_FEDERATION_PROTOCOL), + CloudResourceAttribute(name=RESOURCE_IDP), + CloudResourceAttribute(name=RESOURCE_PROJECT_DOMAIN), + CloudResourceAttribute(name=RESOURCE_ROLE), + CloudResourceAttribute(name=RESOURCE_USER_DOMAIN), + CloudResourceAttribute(name=RESOURCE_EULA_URL), + CloudResourceAttribute(name=RESOURCE_DEFAULT_PUBLIC_NETWORK), + CloudResourceAttribute(name=RESOURCE_DEFAULT_NETWORK_CIDR), +] # TODO: Migration to rename the OpenStack specific prefix out of these attrs ALLOCATION_PROJECT_ID = 'Allocated Project ID' ALLOCATION_PROJECT_NAME = 'Allocated Project Name' ALLOCATION_INSTITUTION_SPECIFIC_CODE = 'Institution-Specific Code' -ALLOCATION_ATTRIBUTES = { - ALLOCATION_PROJECT_ID: { - 'type': 'Text', - 'is_private': False, - 'is_changeable': False, - }, - ALLOCATION_PROJECT_NAME: { - 'type': 'Text', - 'is_private': False, - 'is_changeable': False, - }, - ALLOCATION_INSTITUTION_SPECIFIC_CODE: { - 'type': 'Text', - 'is_private': False, - 'is_changeable': True, - }, -} +ALLOCATION_ATTRIBUTES = [ + CloudAllocationAttribute( + name=ALLOCATION_PROJECT_ID, + type='Text', + is_changeable=False + ), + CloudAllocationAttribute( + name=ALLOCATION_PROJECT_NAME, + type='Text', + is_changeable=False + ), + CloudAllocationAttribute( + name=ALLOCATION_INSTITUTION_SPECIFIC_CODE, + type='Text', + is_changeable=False + ), +] ########################################################### # OpenStack Quota Attributes @@ -88,75 +90,19 @@ QUOTA_PVC = 'OpenShift Persistent Volume Claims Quota' -ALLOCATION_QUOTA_ATTRIBUTES = { - QUOTA_INSTANCES: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_RAM: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_VCPU: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_VOLUMES: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_VOLUMES_GB: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_FLOATING_IPS: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_OBJECT_GB: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_GPU: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_LIMITS_CPU: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_LIMITS_MEMORY: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_LIMITS_EPHEMERAL_STORAGE_GB: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_REQUESTS_STORAGE: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_REQUESTS_GPU: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, - QUOTA_PVC: { - 'type': 'Int', - 'is_private': False, - 'is_changeable': True, - }, -} +ALLOCATION_QUOTA_ATTRIBUTES = [ + CloudAllocationAttribute(name=QUOTA_INSTANCES), + CloudAllocationAttribute(name=QUOTA_RAM), + CloudAllocationAttribute(name=QUOTA_VCPU), + CloudAllocationAttribute(name=QUOTA_VOLUMES), + CloudAllocationAttribute(name=QUOTA_VOLUMES_GB), + CloudAllocationAttribute(name=QUOTA_FLOATING_IPS), + CloudAllocationAttribute(name=QUOTA_OBJECT_GB), + CloudAllocationAttribute(name=QUOTA_GPU), + CloudAllocationAttribute(name=QUOTA_LIMITS_CPU), + CloudAllocationAttribute(name=QUOTA_LIMITS_MEMORY), + CloudAllocationAttribute(name=QUOTA_LIMITS_EPHEMERAL_STORAGE_GB), + CloudAllocationAttribute(name=QUOTA_REQUESTS_STORAGE), + CloudAllocationAttribute(name=QUOTA_REQUESTS_GPU), + CloudAllocationAttribute(name=QUOTA_PVC), +] diff --git a/src/coldfront_plugin_cloud/management/commands/list_cloud_allocations.py b/src/coldfront_plugin_cloud/management/commands/list_cloud_allocations.py index c1125b3..678a12a 100644 --- a/src/coldfront_plugin_cloud/management/commands/list_cloud_allocations.py +++ b/src/coldfront_plugin_cloud/management/commands/list_cloud_allocations.py @@ -38,7 +38,7 @@ def add_arguments(self, parser): def get_cloud_attrs(self, cloud_type): attrs = [ - i for i in attributes.ALLOCATION_QUOTA_ATTRIBUTES if cloud_type in i + i for i in attributes.ALLOCATION_QUOTA_ATTRIBUTES if cloud_type in i.name ] return attrs @@ -78,7 +78,7 @@ def get_allocations(self, cloud_type, project_id=None): alloc_attrs = [] for attr in cloud_attrs: try: - alloc_attrs.append(float(allocation.get_attribute(attr))) + alloc_attrs.append(float(allocation.get_attribute(attr.name))) except TypeError: logger.debug(f'!!! TYPE ERROR FOR ATTR {attr} (ALLOCATION: {alloc_id})') alloc_attrs.append(0) @@ -93,7 +93,7 @@ def get_allocations(self, cloud_type, project_id=None): def render_csv(self, allocations, cloud_type): headers = ['pi_email', 'cloud_type', 'project_id', 'project_title', 'alloc_id'] - headers = headers + [i.replace(' ', '_') for i in self.get_cloud_attrs(cloud_type)] + headers = headers + [i.name.replace(' ', '_') for i in self.get_cloud_attrs(cloud_type)] f = csv.writer(sys.stdout) allocations.insert(0, headers) f.writerows(allocations) diff --git a/src/coldfront_plugin_cloud/management/commands/register_cloud_attributes.py b/src/coldfront_plugin_cloud/management/commands/register_cloud_attributes.py index 99d4e8c..2ea7cf1 100644 --- a/src/coldfront_plugin_cloud/management/commands/register_cloud_attributes.py +++ b/src/coldfront_plugin_cloud/management/commands/register_cloud_attributes.py @@ -74,29 +74,28 @@ def migrate_resource_attributes(self): f' Cannot perform automatic migration.') def register_allocation_attributes(self): - alloc_attrs = {} - alloc_attrs.update(attributes.ALLOCATION_ATTRIBUTES) - alloc_attrs.update(attributes.ALLOCATION_QUOTA_ATTRIBUTES) + alloc_attrs = ( + attributes.ALLOCATION_ATTRIBUTES + + attributes.ALLOCATION_QUOTA_ATTRIBUTES + ) for attr in alloc_attrs: - cfg = alloc_attrs[attr] allocation_models.AllocationAttributeType.objects.get_or_create( - name=attr, + name=attr.name, attribute_type=allocation_models.AttributeType.objects.get( - name=cfg.get('type', 'Text') + name=attr.type, ), - has_usage=cfg.get('has_usage', False), - is_private=cfg.get('is_private', False), - is_changeable=cfg.get('is_changeable', 'Quota' in attr) + has_usage=attr.has_usage, + is_private=attr.is_private, + is_changeable=attr.is_changeable, ) def register_resource_attributes(self): for attr in attributes.RESOURCE_ATTRIBUTES: - cfg = attributes.RESOURCE_ATTRIBUTES[attr] resource_models.ResourceAttributeType.objects.get_or_create( - name=attr, + name=attr.name, attribute_type=resource_models.AttributeType.objects.get( - name=cfg.get('type', 'Text')) + name=attr.type), ) def register_resource_type(self): diff --git a/src/coldfront_plugin_cloud/management/commands/validate_allocations.py b/src/coldfront_plugin_cloud/management/commands/validate_allocations.py index a51b27b..7f89aa0 100644 --- a/src/coldfront_plugin_cloud/management/commands/validate_allocations.py +++ b/src/coldfront_plugin_cloud/management/commands/validate_allocations.py @@ -101,28 +101,28 @@ def handle(self, *args, **options): failed_validation = Command.sync_users(project_id, allocation, allocator, options["apply"]) for attr in attributes.ALLOCATION_QUOTA_ATTRIBUTES: - if 'OpenStack' in attr: - key = openstack.QUOTA_KEY_MAPPING_ALL_KEYS.get(attr, None) + if 'OpenStack' in attr.name: + key = openstack.QUOTA_KEY_MAPPING_ALL_KEYS.get(attr.name, None) if not key: # Note(knikolla): Some attributes are only maintained # for bookkeeping purposes and do not have a # corresponding quota set on the service. continue - expected_value = allocation.get_attribute(attr) + expected_value = allocation.get_attribute(attr.name) current_value = quota.get(key, None) if expected_value is None and current_value: - msg = (f'Attribute "{attr}" expected on allocation {allocation_str} but not set.' + msg = (f'Attribute "{attr.name}" expected on allocation {allocation_str} but not set.' f' Current quota is {current_value}.') if options['apply']: utils.set_attribute_on_allocation( - allocation, attr, current_value + allocation, attr.name, current_value ) msg = f'{msg} Attribute set to match current quota.' logger.warning(msg) elif not current_value == expected_value: failed_validation = True - msg = (f'Value for quota for {attr} = {current_value} does not match expected' + msg = (f'Value for quota for {attr.name} = {current_value} does not match expected' f' value of {expected_value} on allocation {allocation_str}') logger.warning(msg) @@ -173,13 +173,13 @@ def handle(self, *args, **options): failed_validation = Command.sync_users(project_id, allocation, allocator, options["apply"]) for attr in attributes.ALLOCATION_QUOTA_ATTRIBUTES: - if "OpenShift" in attr: - key_with_lambda = openshift.QUOTA_KEY_MAPPING.get(attr, None) + if "OpenShift" in attr.name: + key_with_lambda = openshift.QUOTA_KEY_MAPPING.get(attr.name, None) # This gives me just the plain key key = list(key_with_lambda(1).keys())[0] - expected_value = allocation.get_attribute(attr) + expected_value = allocation.get_attribute(attr.name) current_value = quota.get(key, None) PATTERN = r"([0-9]+)(m|Ki|Mi|Gi|Ti|Pi|Ei|K|M|G|T|P|E)?" @@ -205,7 +205,7 @@ def handle(self, *args, **options): if result is None: raise CommandError( - f"Unable to parse current_value = '{current_value}' for {attr}" + f"Unable to parse current_value = '{current_value}' for {attr.name}" ) value = int(result.groups()[0]) @@ -220,25 +220,25 @@ def handle(self, *args, **options): # Convert some attributes to units that coldfront uses - if "RAM" in attr: + if "RAM" in attr.name: current_value = round(current_value / suffix["Mi"]) - elif "Storage" in attr: + elif "Storage" in attr.name: current_value = round(current_value / suffix["Gi"]) if expected_value is None and current_value: msg = ( - f'Attribute "{attr}" expected on allocation {allocation_str} but not set.' + f'Attribute "{attr.name}" expected on allocation {allocation_str} but not set.' f" Current quota is {current_value}." ) if options["apply"]: utils.set_attribute_on_allocation( - allocation, attr, current_value + allocation, attr.name, current_value ) msg = f"{msg} Attribute set to match current quota." logger.warning(msg) elif not (current_value == expected_value): msg = ( - f"Value for quota for {attr} = {current_value} does not match expected" + f"Value for quota for {attr.name} = {current_value} does not match expected" f" value of {expected_value} on allocation {allocation_str}" ) logger.warning(msg) diff --git a/src/coldfront_plugin_cloud/tests/functional/openstack/test_allocation.py b/src/coldfront_plugin_cloud/tests/functional/openstack/test_allocation.py index d95867b..0606541 100644 --- a/src/coldfront_plugin_cloud/tests/functional/openstack/test_allocation.py +++ b/src/coldfront_plugin_cloud/tests/functional/openstack/test_allocation.py @@ -109,8 +109,8 @@ def test_new_allocation(self): # Check correct attributes for attr in attributes.ALLOCATION_QUOTA_ATTRIBUTES: - if 'OpenStack' in attr: - self.assertIsNotNone(allocation.get_attribute(attr)) + if 'OpenStack' in attr.name: + self.assertIsNotNone(allocation.get_attribute(attr.name)) def test_new_allocation_with_quantity(self): user = self.new_user() diff --git a/src/coldfront_plugin_cloud/tests/unit/test_attribute_migration.py b/src/coldfront_plugin_cloud/tests/unit/test_attribute_migration.py index 37d106e..9e88699 100644 --- a/src/coldfront_plugin_cloud/tests/unit/test_attribute_migration.py +++ b/src/coldfront_plugin_cloud/tests/unit/test_attribute_migration.py @@ -20,16 +20,18 @@ def setUp(self) -> None: call_command('load_test_data') sys.stdout = backup - @mock.patch( - 'coldfront_plugin_cloud.management.commands.register_cloud_attributes.RESOURCE_ATTRIBUTE_MIGRATIONS', + @mock.patch.object( + register_cloud_attributes, + 'RESOURCE_ATTRIBUTE_MIGRATIONS', [ ('Before Migration', 'After First Migration'), ('After First Migration', 'After Migration'), ('Not Present', 'Still not present'), ] ) - @mock.patch( - 'coldfront_plugin_cloud.management.commands.register_cloud_attributes.ALLOCATION_ATTRIBUTE_MIGRATIONS', + @mock.patch.object( + register_cloud_attributes, + 'ALLOCATION_ATTRIBUTE_MIGRATIONS', [ ('Before Migration', { 'name': 'After First Migration' @@ -71,7 +73,9 @@ def test_rename_attribute(self): is_private=False ) - with self.assertRaises(resource_models.ResourceAttributeType.DoesNotExist): + with self.assertRaises( + resource_models.ResourceAttributeType.DoesNotExist + ): resource_models.ResourceAttributeType.objects.get( name='Before Migration' ) @@ -82,7 +86,9 @@ def test_rename_attribute(self): name='Still not present' ) - with self.assertRaises(allocation_models.AllocationAttributeType.DoesNotExist): + with self.assertRaises( + allocation_models.AllocationAttributeType.DoesNotExist + ): allocation_models.AllocationAttributeType.objects.get( name='Before Migration' ) @@ -112,36 +118,52 @@ def test_rename_attribute(self): ) call_command('register_cloud_attributes') - with self.assertRaises(allocation_models.AllocationAttributeType.DoesNotExist): + with self.assertRaises( + allocation_models.AllocationAttributeType.DoesNotExist + ): allocation_models.AllocationAttributeType.objects.get( name='No Migration' ) def test_rename_identity_url(self): - with mock.patch( - 'coldfront_plugin_cloud.management.commands.register_cloud_attributes.RESOURCE_ATTRIBUTE_MIGRATIONS', - [] + with mock.patch.object( + register_cloud_attributes, + 'RESOURCE_ATTRIBUTE_MIGRATIONS', + [], ): - resource_attrs = {} - resource_attrs.update(attributes.RESOURCE_ATTRIBUTES) - new_auth_url_attr = 'OpenStack Auth URL' - auth_url_cfg = attributes.RESOURCE_ATTRIBUTES[attributes.RESOURCE_AUTH_URL] - resource_attrs[new_auth_url_attr] = auth_url_cfg - del resource_attrs[attributes.RESOURCE_AUTH_URL] - with mock.patch('coldfront_plugin_cloud.attributes.RESOURCE_AUTH_URL', new_auth_url_attr): - with mock.patch.dict('coldfront_plugin_cloud.attributes.RESOURCE_ATTRIBUTES', resource_attrs, clear=True): + orig_auth_url_name = attributes.RESOURCE_AUTH_URL + new_auth_url_name = 'OpenStack Auth URL' + assert orig_auth_url_name != new_auth_url_name + auth_url_val = 'https://example.com' + new_auth_url_attr = attributes.CloudResourceAttribute( + name=new_auth_url_name, + ) + new_resource_attrs = [] + new_resource_attrs.extend(attributes.RESOURCE_ATTRIBUTES) + new_resource_attrs[0] = new_auth_url_attr + + with mock.patch.object( + attributes, + 'RESOURCE_AUTH_URL', + new_auth_url_name, + ): + with mock.patch.object( + attributes, + 'RESOURCE_ATTRIBUTES', + new_resource_attrs, + ): call_command('register_cloud_attributes') - resource = self.new_resource('Example', 'https://example.com') + resource = self.new_resource('Example', auth_url_val) self.assertEqual( - resource.get_attribute('OpenStack Auth URL'), - 'https://example.com' + resource.get_attribute(new_auth_url_name), + auth_url_val, ) call_command('register_cloud_attributes') self.assertEqual( - resource.get_attribute('Identity Endpoint URL'), + resource.get_attribute(orig_auth_url_name), 'https://example.com' )