Create folder structure when read from s3 (#2314)

ESSolutions · Jun 6, 2024 · abb2487 · abb2487
1 parent 15ffd80
commit abb2487
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 8 deletions.
diff --git a/ESSArch_Core/storage/backends/s3.py b/ESSArch_Core/storage/backends/s3.py
@@ -36,30 +36,34 @@ def open(self, storage_object, file, *args, **kwargs):
         return data
 
     def read(self, storage_object, dst, extract=False, include_xml=True, block_size=DEFAULT_BLOCK_SIZE):
-        ip = storage_object.ip
-
         bucket_name, key = storage_object.content_location_value.split('/', 1)
         bucket = s3.Bucket(bucket_name)
 
         if storage_object.container:
+            ip = storage_object.ip
+            aic_xml = True if ip.aic else False
+            os.makedirs(dst, exist_ok=True)
             src_tar = key
             src_xml = os.path.splitext(key)[0] + '.xml'
-            src_aic_xml = str(ip.aic.pk) + '.xml'
             dst_tar = os.path.join(dst, os.path.basename(src_tar))
             dst_xml = os.path.join(dst, os.path.basename(src_xml))
-            dst_aic_xml = os.path.join(dst, os.path.basename(src_aic_xml))
+            if aic_xml:
+                src_aic_xml = str(ip.aic.pk) + '.xml'
+                dst_aic_xml = os.path.join(dst, os.path.basename(src_aic_xml))
 
             if include_xml:
                 bucket.download_file(src_xml, dst_xml)
-                bucket.download_file(src_aic_xml, dst_aic_xml)
+                if aic_xml:
+                    bucket.download_file(src_aic_xml, dst_aic_xml)
             if extract:
                 return self._extract(storage_object, dst)
             else:
                 bucket.download_file(src_tar, dst_tar)
                 return dst_tar
         else:
             for object_summary in bucket.objects.filter(Prefix=key):
-                dst_file = os.path.join(dst, os.path.basename(object_summary.key))
+                dst_file = os.path.join(dst, os.path.relpath(object_summary.key, key))
+                os.makedirs(os.path.dirname(dst_file), exist_ok=True)
                 bucket.download_file(object_summary.key, dst_file)
             return dst
 

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -6,7 +6,7 @@ channels==4.1.0
 channels_redis==4.1.0
 chardet==5.1.0
 click==8.1.3
-cryptography==42.0.7
+cryptography==42.0.8
 daphne==4.1.2
 dj-rest-auth[with_social]==6.0.0
 django==5.0.6
@@ -46,7 +46,7 @@ opf-fido==1.6.1
 pyfakefs==5.5.0
 python-dateutil==2.8.2
 pywin32==306 ; platform_system=='Windows'
-redis==5.0.4
+redis==5.0.5
 regex==2024.5.15
 requests==2.32.3
 requests_toolbelt==1.0.0