diff --git a/ESSArch_Core/storage/backends/s3.py b/ESSArch_Core/storage/backends/s3.py index c2b6a143e..82b02e77e 100644 --- a/ESSArch_Core/storage/backends/s3.py +++ b/ESSArch_Core/storage/backends/s3.py @@ -36,22 +36,25 @@ def open(self, storage_object, file, *args, **kwargs): return data def read(self, storage_object, dst, extract=False, include_xml=True, block_size=DEFAULT_BLOCK_SIZE): - ip = storage_object.ip - bucket_name, key = storage_object.content_location_value.split('/', 1) bucket = s3.Bucket(bucket_name) if storage_object.container: + ip = storage_object.ip + aic_xml = True if ip.aic else False + os.makedirs(dst, exist_ok=True) src_tar = key src_xml = os.path.splitext(key)[0] + '.xml' - src_aic_xml = str(ip.aic.pk) + '.xml' dst_tar = os.path.join(dst, os.path.basename(src_tar)) dst_xml = os.path.join(dst, os.path.basename(src_xml)) - dst_aic_xml = os.path.join(dst, os.path.basename(src_aic_xml)) + if aic_xml: + src_aic_xml = str(ip.aic.pk) + '.xml' + dst_aic_xml = os.path.join(dst, os.path.basename(src_aic_xml)) if include_xml: bucket.download_file(src_xml, dst_xml) - bucket.download_file(src_aic_xml, dst_aic_xml) + if aic_xml: + bucket.download_file(src_aic_xml, dst_aic_xml) if extract: return self._extract(storage_object, dst) else: @@ -59,7 +62,8 @@ def read(self, storage_object, dst, extract=False, include_xml=True, block_size= return dst_tar else: for object_summary in bucket.objects.filter(Prefix=key): - dst_file = os.path.join(dst, os.path.basename(object_summary.key)) + dst_file = os.path.join(dst, os.path.relpath(object_summary.key, key)) + os.makedirs(os.path.dirname(dst_file), exist_ok=True) bucket.download_file(object_summary.key, dst_file) return dst diff --git a/requirements/base.txt b/requirements/base.txt index 171c7f0e4..99afe0c80 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,7 +6,7 @@ channels==4.1.0 channels_redis==4.1.0 chardet==5.1.0 click==8.1.3 -cryptography==42.0.7 +cryptography==42.0.8 daphne==4.1.2 dj-rest-auth[with_social]==6.0.0 django==5.0.6 @@ -46,7 +46,7 @@ opf-fido==1.6.1 pyfakefs==5.5.0 python-dateutil==2.8.2 pywin32==306 ; platform_system=='Windows' -redis==5.0.4 +redis==5.0.5 regex==2024.5.15 requests==2.32.3 requests_toolbelt==1.0.0