From 44f8dbbefd555aab8474f5a855107a3d3493a1c5 Mon Sep 17 00:00:00 2001 From: Jerome Pansanel Date: Thu, 14 Dec 2017 11:36:21 +0100 Subject: [PATCH 1/3] New cleanup feature to support CEPH storage backend --- cloudkeeper_os/constants.py | 1 + cloudkeeper_os/imagemanager.py | 56 +++++++++++++++++++++++++++------- cloudkeeper_os/server.py | 9 ++++-- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/cloudkeeper_os/constants.py b/cloudkeeper_os/constants.py index c42ba41..1cb7bbe 100644 --- a/cloudkeeper_os/constants.py +++ b/cloudkeeper_os/constants.py @@ -20,3 +20,4 @@ APPLIANCE_INT_VALUES = ['ram', 'core', 'expiration_date'] IMAGE_ID_TAG = 'APPLIANCE_ID' IMAGE_LIST_ID_TAG = 'APPLIANCE_IMAGE_LIST_ID' +IMAGE_STATUS_TAG = 'APPLIANCE_STATUS' diff --git a/cloudkeeper_os/imagemanager.py b/cloudkeeper_os/imagemanager.py index ebad361..8708fe3 100644 --- a/cloudkeeper_os/imagemanager.py +++ b/cloudkeeper_os/imagemanager.py @@ -34,6 +34,7 @@ IMAGE_ID_TAG = constants.IMAGE_ID_TAG IMAGE_LIST_ID_TAG = constants.IMAGE_LIST_ID_TAG APPLIANCE_INT_VALUES = constants.APPLIANCE_INT_VALUES +IMAGE_STATUS_TAG = constants.IMAGE_STATUS_TAG class ApplianceManager(object): @@ -86,6 +87,7 @@ def add_appliance(self, appliance): appliance.ClearField('image') properties = utils.extract_appliance_properties(appliance) + properties[IMAGE_STATUS_TAG] = 'Active' LOG.debug("Creating image '%s' (format: '%s', " "properties %s)" % (appliance.title, @@ -112,17 +114,17 @@ def update_appliance(self, appliance): """Update an appliance stored in glance """ LOG.info("Updating image: '%s'" % appliance.identifier) - LOG.debug("Deleting old release of the appliance") - old_image_id = self.remove_appliance(appliance) - LOG.debug("The glance image '%s' has been deleted" % old_image_id) + LOG.debug("Marking appliance for removal") + old_image_id = self.mark_appliance_for_removal(appliance) + LOG.debug("The glance image '%s' has been marked" % old_image_id) LOG.debug("Creating new release of the appliance") image_id = self.add_appliance(appliance) LOG.debug("The glance image '%s' has been created" % image_id) return image_id - def remove_appliance(self, appliance): - """Remove an appliance in glance + def mark_appliance_for_removal(self, appliance): + """Mark an appliance in glance for removal """ project_name = self.mapping.get_project_from_vo(appliance.vo) if not project_name: @@ -139,13 +141,44 @@ def remove_appliance(self, appliance): glance_image = utils.find_image(glance, appliance.identifier, appliance.image_list_identifier) if not glance_image: - LOG.info("Cannot delete image: image not found") + LOG.info("Cannot mark image for removal: image not found") return None - LOG.info("Deleting image: '%s'" % glance_image.id) - glance.images.delete(glance_image.id) + LOG.info("Marking image for removal: '%s'" % glance_image.id) + properties = {} + properties[IMAGE_STATUS_TAG] = 'EOL' + glance.images.update(glance_image.id, visibility='private', **properties) + return glance_image.id + def cleanup_appliances(self): + """Try to remove all appliances marked for removal + """ + for project_name in self.mapping.get_projects(): + glance = openstack_client.get_glance_client(project_name) + if not glance: + LOG.error("Not authorized to manage images from the " + "project: %s" % project_name) + continue + try: + img_generator = glance.images.list() + image_list = list(img_generator) + except Exception as err: + LOG.error("Not authorized to retrieve the image list from " + "the project: %s" % project_name) + LOG.exception(err) + continue + + for image in image_list: + if IMAGE_LIST_ID_TAG in image: + if IMAGE_STATUS_TAG in image and image[IMAGE_STATUS_TAG] == 'EOL': + try: + LOG.debug("Trying to delete image '%s'" % image['id']) + glance.images.delete(image['id']) + LOG.debug("Succesfully deleted image '%s'" % image['id']) + except Exception as err: + LOG.debug("Cannot cleanup image '%s'" % image['id']) + LOG.debug(err) class ImageListManager(object): """A class for managing image lists @@ -178,9 +211,10 @@ def update_image_list_identifiers(self): for image in image_list: if IMAGE_LIST_ID_TAG in image: - if image[IMAGE_LIST_ID_TAG] not in appliances: - appliances[image[IMAGE_LIST_ID_TAG]] = [] - appliances[image[IMAGE_LIST_ID_TAG]].append(image) + if IMAGE_STATUS_TAG in image and image[IMAGE_STATUS_TAG] != 'EOL': + if image[IMAGE_LIST_ID_TAG] not in appliances: + appliances[image[IMAGE_LIST_ID_TAG]] = [] + appliances[image[IMAGE_LIST_ID_TAG]].append(image) self.appliances = appliances diff --git a/cloudkeeper_os/server.py b/cloudkeeper_os/server.py index ee0047f..c6e83a5 100644 --- a/cloudkeeper_os/server.py +++ b/cloudkeeper_os/server.py @@ -50,6 +50,11 @@ def PreAction(self, request, context): return cloudkeeper_pb2.Empty() def PostAction(self, request, context): + """Cleanup of images marked for removal + """ + LOG.info("Cleaning up appliances marked for removal") + manager = imagemanager.ApplianceManager() + manager.cleanup_appliances() metadata = ( ('status', 'SUCCESS'), ) @@ -98,9 +103,9 @@ def RemoveAppliance(self, request, context): metadata = ( ('status', 'SUCCESS'), ) - LOG.info("Removing appliance: %s" % request.identifier) + LOG.info("Marking appliances for removal: %s" % request.identifier) manager = imagemanager.ApplianceManager() - if not manager.remove_appliance(request): + if not manager.mark_appliance_for_removal(request): metadata = ( ('status', 'ERROR'), ) From 6a0239be303c8cd179b5ce422d577a208faacb79 Mon Sep 17 00:00:00 2001 From: Jerome Pansanel Date: Thu, 14 Dec 2017 11:43:45 +0100 Subject: [PATCH 2/3] Rebase --- cloudkeeper_os/imagemanager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cloudkeeper_os/imagemanager.py b/cloudkeeper_os/imagemanager.py index 8708fe3..9cbab08 100644 --- a/cloudkeeper_os/imagemanager.py +++ b/cloudkeeper_os/imagemanager.py @@ -87,7 +87,8 @@ def add_appliance(self, appliance): appliance.ClearField('image') properties = utils.extract_appliance_properties(appliance) - properties[IMAGE_STATUS_TAG] = 'Active' + min_ram = int(properties.get("APPLIANCE_RAM", 0)) + properties[IMAGE_STATUS_TAG] = 'ACTIVE' LOG.debug("Creating image '%s' (format: '%s', " "properties %s)" % (appliance.title, @@ -97,7 +98,9 @@ def add_appliance(self, appliance): glance_image = glance.images.create(name=appliance.title, disk_format=str.lower(image_format), - container_format="bare" + container_format="bare", + visibility=CONF.image_visibility, + min_ram=min_ram ) glance.images.upload(glance_image.id, image_data) glance.images.update(glance_image.id, **properties) From 2b79733c8085813e2d563e82e75810ebcb1f291b Mon Sep 17 00:00:00 2001 From: Jerome Pansanel Date: Thu, 14 Dec 2017 11:46:36 +0100 Subject: [PATCH 3/3] Rebase --- cloudkeeper_os/imagemanager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudkeeper_os/imagemanager.py b/cloudkeeper_os/imagemanager.py index 9cbab08..bd2058e 100644 --- a/cloudkeeper_os/imagemanager.py +++ b/cloudkeeper_os/imagemanager.py @@ -88,6 +88,7 @@ def add_appliance(self, appliance): properties = utils.extract_appliance_properties(appliance) min_ram = int(properties.get("APPLIANCE_RAM", 0)) + properties[IMAGE_STATUS_TAG] = 'ACTIVE' LOG.debug("Creating image '%s' (format: '%s', "