From 88f61902244804faca693702dbf3919015181950 Mon Sep 17 00:00:00 2001 From: Wouter Claeys Date: Fri, 19 Oct 2018 13:23:45 +0200 Subject: [PATCH 1/5] Pyup scheduled update 2018 10 01 (#52) * Update pyramid-debugtoolbar from 4.4 to 4.5 * Update pytest from 3.6.0 to 3.8.1 * Update pytest-cov from 2.5.1 to 2.6.0 * Update py from 1.5.3 to 1.6.0 * Update coveralls from 1.3.0 to 1.5.1 * Update webtest from 2.0.29 to 2.0.30 * Update sphinx from 1.7.5 to 1.8.1 * Update sphinxcontrib-httpdomain from 1.6.1 to 1.7.0 * Update sphinx_rtd_theme from 0.3.1 to 0.4.1 * Update webob from 1.8.1 to 1.8.2 --- requirements-dev.txt | 18 +++++++++--------- requirements.txt | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7842fc7..29ebb2b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,21 +2,21 @@ --requirement requirements.txt # pyramid -pyramid-debugtoolbar==4.4 +pyramid-debugtoolbar==4.5 # Testing -pytest==3.6.0 -pytest-cov==2.5.1 -py==1.5.3 -coveralls==1.3.0 -webtest==2.0.29 +pytest==3.8.1 +pytest-cov==2.6.0 +py==1.6.0 +coveralls==1.5.1 +webtest==2.0.30 mock==2.0.0 tempdir==0.7.1 # Documentation -Sphinx==1.7.5 -sphinxcontrib-httpdomain==1.6.1 -sphinx_rtd_theme==0.3.1 +Sphinx==1.8.1 +sphinxcontrib-httpdomain==1.7.0 +sphinx_rtd_theme==0.4.1 # waitress waitress==1.1.0 diff --git a/requirements.txt b/requirements.txt index 975a1f7..e2c57b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ pairtree==0.8.1 python-magic==0.4.15 venusian==1.1.0 -webob==1.8.1 +webob==1.8.2 PasteDeploy==1.5.2 repoze.lru==0.7 From b034124f4a18d33a7470ee477c31c39c7b1ae368 Mon Sep 17 00:00:00 2001 From: Wim-De-Clercq Date: Fri, 19 Oct 2018 13:28:16 +0200 Subject: [PATCH 2/5] Download zip of entire container. (#50) Closes-Issue: #48 --- augeias/__init__.py | 6 ++++- augeias/stores/CephStore.py | 3 +++ augeias/stores/PairTreeFileSystemStore.py | 18 +++++++++++++++ augeias/stores/StoreInterface.py | 8 +++++++ augeias/views.py | 13 +++++++++++ tests/test_functional.py | 28 +++++++++++++++++++++++ 6 files changed, 75 insertions(+), 1 deletion(-) diff --git a/augeias/__init__.py b/augeias/__init__.py index ae62fee..3346658 100644 --- a/augeias/__init__.py +++ b/augeias/__init__.py @@ -18,7 +18,11 @@ def includeme(config): config.add_route('delete_container', pattern='/collections/{collection_key}/containers/{container_key}', request_method="DELETE") config.add_route('list_object_keys_for_container', - pattern='/collections/{collection_key}/containers/{container_key}', request_method="GET") + pattern='/collections/{collection_key}/containers/{container_key}', request_method="GET", + accept='application/json') + config.add_route('get_container_data', + pattern='/collections/{collection_key}/containers/{container_key}', + request_method="GET", accept='application/zip') config.add_route('create_object_and_id', pattern='/collections/{collection_key}/containers/{container_key}', request_method="POST") config.add_route('update_object', pattern='/collections/{collection_key}/containers/{container_key}/{object_key}', diff --git a/augeias/stores/CephStore.py b/augeias/stores/CephStore.py index d9d536b..f7c45cb 100644 --- a/augeias/stores/CephStore.py +++ b/augeias/stores/CephStore.py @@ -24,6 +24,9 @@ def delete_object(self, container_key, object_key): def update_object(self, container_key, object_key, object_data): pass + def get_container_data(self, container_key): + pass + def create_container(self, container_key): pass diff --git a/augeias/stores/PairTreeFileSystemStore.py b/augeias/stores/PairTreeFileSystemStore.py index 139858f..5df0a7c 100644 --- a/augeias/stores/PairTreeFileSystemStore.py +++ b/augeias/stores/PairTreeFileSystemStore.py @@ -2,6 +2,8 @@ ''' This module provide a simple filesystem based store ''' +from io import BytesIO +from zipfile import ZipFile from pairtree import PairtreeStorageFactory, PartNotFoundException, ObjectNotFoundException, id2path from augeias.stores.StoreInterface import IStore @@ -114,6 +116,22 @@ def delete_object(self, container_key, object_key): except PartNotFoundException: raise NotFoundException + def get_container_data(self, container_key): + ''' + Find a container and return a zip file of its contents. + + :param container_key: Key of the container which must be retrieved. + :return: a zip file containing all files of the container. + ''' + container = self.store.get_object(container_key, + create_if_doesnt_exist=False) + in_memory_file = BytesIO() + with ZipFile(in_memory_file, 'w') as zf: + for object_key in container.list_parts(): + zf.writestr(object_key, container.get_bytestream(object_key)) + in_memory_file.seek(0) + return in_memory_file + def create_container(self, container_key): ''' Create a new container in the data store. diff --git a/augeias/stores/StoreInterface.py b/augeias/stores/StoreInterface.py index dfa6aa7..9d86579 100644 --- a/augeias/stores/StoreInterface.py +++ b/augeias/stores/StoreInterface.py @@ -77,6 +77,14 @@ def list_object_keys_for_container(self, container_key): ''' @abstractmethod + def get_container_data(self, container_key): + ''' + Find a container and return a zip file of its contents. + + :param container_key: Key of the container which must be retrieved. + :return: a zip file containing all files of the container. + ''' + @abstractmethod def create_container(self, container_key): ''' Create a new container in the data store. diff --git a/augeias/views.py b/augeias/views.py index e6bf73f..d3429d1 100644 --- a/augeias/views.py +++ b/augeias/views.py @@ -134,6 +134,19 @@ def list_object_keys_for_container(self): res.json_body = collection.object_store.list_object_keys_for_container(container_key) return res + @view_config(route_name='get_container_data', permission='view') + def get_container_data(self): + '''get a container from the data store''' + collection = _retrieve_collection(self.request) + container_key = self.request.matchdict['container_key'] + zip_file = collection.object_store.get_container_data(container_key) + filename = str(container_key) + '.zip' + disposition = ('attachment; filename={}'.format(filename)) + res = Response(content_type='application/zip', status=200, + content_disposition=disposition) + res.body = zip_file.read() + return res + @view_config(route_name='create_container', permission='edit') def create_container(self): '''create a new container in the data store''' diff --git a/tests/test_functional.py b/tests/test_functional.py index 8282a61..76770fb 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -1,6 +1,9 @@ import os import re import unittest +from io import BytesIO +from zipfile import ZipFile + import tempdir from pyramid.paster import get_appsettings from webtest import TestApp @@ -161,6 +164,31 @@ def test_list_object_keys_for_container(self): l = [i.strip() for i in l] self.assertTrue('200x300' in l and '400x600' in l) + def test_download_container_zip(self): + # create container and add objects + cres = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID') + self.assertEqual('200 OK', cres.status) + testdata = os.path.join(here, '../', 'fixtures/kasteel.jpg') + with open(testdata, 'rb') as f: + bdata = f.read() + ores = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/200x300', bdata) + self.assertEqual('200 OK', ores.status) + testdata = os.path.join(here, '../', 'fixtures/brug.jpg') + with open(testdata, 'rb') as f: + bdata = f.read() + ores = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID/400x600', bdata) + self.assertEqual('200 OK', ores.status) + + res = self.testapp.get('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID', + headers={'Accept': 'application/zip'}) + self.assertEqual('200 OK', res.status) + with ZipFile(BytesIO(res.body)) as zf: + filenames = zf.namelist() + self.assertEqual(2, len(filenames)) + self.assertIn('200x300', filenames) + self.assertIn('400x600', filenames) + + def test_update_object(self): # create container and add object cres = self.testapp.put('/collections/TEST_COLLECTION/containers/TEST_CONTAINER_ID') From 8dd9348570f014a6940c7d49b3ff695b1f5145ba Mon Sep 17 00:00:00 2001 From: Wim-De-Clercq Date: Mon, 5 Nov 2018 14:01:06 +0100 Subject: [PATCH 3/5] Update list of supported python versions (#53) Closes-Issue: #51 --- .travis.yml | 1 + setup.py | 4 ++++ tox.ini | 4 ++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 57dc495..5101768 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ sudo: false python: - "2.7" - "3.5" + - "3.6" env: - PROJECT=augeias notifications: diff --git a/setup.py b/setup.py index 61d5dea..839fbce 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,11 @@ "Topic :: Internet :: WWW/HTTP", "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", ], + python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*', author='Flanders Heritage Agency', author_email='ict@onroerenderfgoed.be', url='https://augeias.readthedocs.org', diff --git a/tox.ini b/tox.ini index cd4587d..89591d3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py33,py34,cover +envlist = py27,py35,py36,py37,cover [testenv] deps = @@ -12,7 +12,7 @@ commands = [testenv:cover] basepython = - python3.4 + python3.5 deps = pytest pytest-cov From 2d08b9bb0ce51db929dd8a2a2f62dca56701f420 Mon Sep 17 00:00:00 2001 From: Wouter Claeys Date: Wed, 28 Nov 2018 12:49:56 +0100 Subject: [PATCH 4/5] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 839fbce..341e481 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ ] setup(name='augeias', - version='0.4.1', + version='0.5.0', description='Augeias. Stores your files.', long_description=README + '\n\n' + HISTORY, classifiers=[ From ee3b66163296ae448d9cbcd29b6bc8d167c18f89 Mon Sep 17 00:00:00 2001 From: Wouter Claeys Date: Wed, 28 Nov 2018 12:51:58 +0100 Subject: [PATCH 5/5] Update HISTORY.rst --- HISTORY.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index cfc2c3a..e086632 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,3 +1,9 @@ +0.5.0 (28-11-2018) +------------------ + +- Download zip of entire container (#48) +- Update list of supported python version (#51) + 0.4.1 (26-10-2017) ------------------