From f6d6793337e65a36ce66d727f2d74bc4db8a8f35 Mon Sep 17 00:00:00 2001 From: henrikek Date: Tue, 3 Sep 2024 22:28:24 +0200 Subject: [PATCH] Fix tape full verify --- ESSArch_Core/storage/backends/tape.py | 8 ++++---- ESSArch_Core/storage/models.py | 20 +++++++++++++++++--- requirements/base.txt | 4 ++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/ESSArch_Core/storage/backends/tape.py b/ESSArch_Core/storage/backends/tape.py index b84468a6f..ebc6213d4 100644 --- a/ESSArch_Core/storage/backends/tape.py +++ b/ESSArch_Core/storage/backends/tape.py @@ -278,14 +278,14 @@ def post_mark_as_full(cls, storage_medium): """Called after a medium has been successfully marked as full""" logger = logging.getLogger('essarch.storage.backends.tape') - drive = str(storage_medium.drive) + drive = storage_medium.tape_drive logger.debug('Release lock for drive {} and storage medium {} ({})'.format( - drive.pk, storage_medium.medium_id, str(storage_medium.pk))) + drive.pk, storage_medium.medium_id, storage_medium.pk)) cache.delete_pattern(drive.get_lock_key()) logger.debug('Queueing unmount of storage medium {} ({})'.format( - storage_medium.medium_id, str(storage_medium.pk))) + storage_medium.medium_id, storage_medium.pk)) rq, _ = RobotQueue.objects.get_or_create( user=User.objects.get(username='system'), storage_medium=storage_medium, @@ -295,5 +295,5 @@ def post_mark_as_full(cls, storage_medium): while RobotQueue.objects.filter(id=rq.id).exists(): logger.debug('Wait for the unmount request to complete for storage medium {} ({})'.format( - storage_medium.medium_id, str(storage_medium.pk))) + storage_medium.medium_id, storage_medium.pk)) time.sleep(1) diff --git a/ESSArch_Core/storage/models.py b/ESSArch_Core/storage/models.py index 901190881..f86877833 100644 --- a/ESSArch_Core/storage/models.py +++ b/ESSArch_Core/storage/models.py @@ -2,6 +2,7 @@ import logging import os import pickle +import shutil import tarfile import uuid from datetime import timedelta @@ -382,7 +383,8 @@ def _create_storage_medium(self): if storage_type == TAPE: slot = TapeSlot.objects.filter(status=20, storage_medium__isnull=True, - medium_id__startswith=self.target).exclude(medium_id__exact='').first() + medium_id__startswith=self.target + ).exclude(medium_id__exact='').natural_sort().first() if slot is None: raise ValueError("No tape available for allocation") medium = StorageMedium.objects.create(medium_id=slot.medium_id, storage_target=self, status=20, @@ -669,12 +671,13 @@ def deactivate(self) -> None: def mark_as_full(self): logger = logging.getLogger('essarch.storage.models') logger.debug('Marking storage medium as full: "{}"'.format(str(self.pk))) + logger.info('Storage medium is full, start to verify: "{}"'.format(self.medium_id)) objs = self.storage.annotate( content_location_value_int=Cast('content_location_value', models.IntegerField()) ).order_by('content_location_value_int') if objs.count() > 3: - objs = [objs.first(), objs[objs.count() / 2], objs.last()] + objs = [objs.first(), objs[int(objs.count() / 2)], objs.last()] try: for obj in objs: @@ -684,11 +687,15 @@ def mark_as_full(self): logger.exception('Failed to verify storage medium: "{}"'.format(str(self.pk))) raise else: + verifydir = Path.objects.get(entity='verify').value + tmppath = os.path.join(verifydir, self.storage_target.target) + shutil.rmtree(tmppath) self.status = 30 storage_backend = self.storage_target.get_storage_backend() storage_backend.post_mark_as_full(self) finally: self.save(update_fields=['status']) + logger.info('Storage medium is full, content verified success: "{}"'.format(self.medium_id)) class Meta: permissions = ( @@ -985,7 +992,7 @@ def verify(self): drive.last_change = timezone.now() drive.save(update_fields=['last_change']) - filename = os.path.join(tmppath, self.ip.object_identifier_value + '.tar'), + filename = os.path.join(tmppath, self.ip.object_identifier_value + '.tar') algorithm = self.ip.get_message_digest_algorithm_display() options = {'expected': self.ip.message_digest, 'algorithm': algorithm} @@ -1084,6 +1091,11 @@ def __str__(self): return self.device +class TapeSlotQueryset(models.QuerySet): + def natural_sort(self): + return natural_sort(self, 'medium_id') + + class TapeSlot(models.Model): STATUS_CHOICES = ( (0, 'Inactive'), @@ -1103,6 +1115,8 @@ class TapeSlot(models.Model): robot = models.ForeignKey('Robot', models.PROTECT, related_name='tape_slots') status = models.IntegerField(choices=STATUS_CHOICES, default=20) + objects = TapeSlotQueryset.as_manager() + @classmethod @transaction.atomic @retry(retry=retry_if_exception_type(RequestException), reraise=True, stop=stop_after_attempt(5), diff --git a/requirements/base.txt b/requirements/base.txt index 0ee116370..cbe0b869e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ asgiref==3.8.1 -boto3==1.34.160 +boto3==1.35.11 celery[tblib]==5.4.0 cffi==1.17.0 channels==4.1.0 @@ -9,7 +9,7 @@ click==8.1.3 cryptography==42.0.8 daphne==4.1.2 dj-rest-auth[with_social]==6.0.0 -django==5.0.8 +django==5.0.9 django-allauth==0.61.1 django-cors-headers==4.4.0 django-countries-plus==2.2.0