Merge pull request #18 from liip/tests/fix-tests

Tests/fix tests

.github/workflows/test.yml

@@ -7,7 +7,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.10'
       - name: Install requirements
         run: pip install flake8 pycodestyle black isort
       - name: Check syntax and complexity
@@ -47,6 +47,7 @@ jobs:
       CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
       CKAN_SOLR_URL: http://solr:8983/solr/ckan
       CKAN_REDIS_URL: redis://redis:6379/1
+      CKAN_SITE_URL: http://odp.test
 
     steps:
     - uses: actions/checkout@v2
@@ -55,14 +56,23 @@ jobs:
         pip install -r requirements.txt
         pip install -r dev-requirements.txt
         pip install -e .
+        pip install -e git+https://github.com/ckan/[email protected]#egg=ckanext-dcat
+        pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v1.5.1/requirements.txt
         pip install -e git+https://gitlab.liip.ch/odp_oev_schweiz/ckanext-harvest.git#egg=ckanext-harvest
         pip install -r https://gitlab.liip.ch/odp_oev_schweiz/ckanext-harvest/-/raw/main/requirements.txt
+        pip install -e git+https://github.com/ckan/[email protected]#egg=ckanext-scheming
+        pip install -e git+https://github.com/ckan/ckanext-fluent.git#egg=ckanext-fluent
+        pip install -r https://raw.githubusercontent.com/ckan/ckanext-fluent/master/requirements.txt
+        pip install -e git+https://gitlab.liip.ch/odp_oev_schweiz/[email protected]#egg=ckanext-discourse
+        pip install -e git+https://github.com/opendata-swiss/ckanext-ogdchcommands.git#egg=ckanext-ogdchcommands
+        pip install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-xloader'
+        pip install -r https://raw.githubusercontent.com/ckan/ckanext-xloader/1.0.1/requirements.txt
+        pip install -U requests[security]
     - name: Setup extension
       run: |
         # Replace default path to CKAN core config file with the one on the container
         sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
         ckan -c test.ini db init
         ckan -c test.ini harvester initdb
     - name: Run tests
-      run: nosetests --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.switzerland \
-        --cover-inclusive --cover-erase --cover-tests
+      run: pytest --ckan-ini=test.ini --disable-warnings --cov=ckanext.switzerland

.gitignore

@@ -35,8 +35,7 @@ htmlcov/
 .tox/
 .coverage
 .cache
-nosetests.xml
 coverage.xml
 
 # Sphinx documentation
-docs/_build/
+docs/_build/

ckanext/switzerland/presets.json

@@ -6,7 +6,7 @@
     {
       "preset_name": "title",
       "values": {
-        "validators": "if_empty_same_as(name) unicode",
+        "validators": "if_empty_same_as(name) unicode_safe",
         "form_snippet": "large_text.html",
         "form_attrs": {
           "data-module": "slug-preview-target"
@@ -52,14 +52,14 @@
     {
       "preset_name": "dataset_slug",
       "values": {
-        "validators": "not_empty unicode name_validator package_name_validator",
+        "validators": "not_empty unicode_safe name_validator package_name_validator",
         "form_snippet": "slug.html"
       }
     },
     {
       "preset_name": "group_slug",
       "values": {
-        "validators": "not_empty unicode name_validator group_name_validator",
+        "validators": "not_empty unicode_safe name_validator group_name_validator",
         "form_snipper": "slug.html"
       }
     },
@@ -77,7 +77,7 @@
     {
       "preset_name": "dataset_organization",
       "values": {
-        "validators": "owner_org_validator unicode",
+        "validators": "owner_org_validator unicode_safe",
         "form_snippet": "organization.html"
       }
     },
@@ -96,7 +96,7 @@
     {
       "preset_name": "resource_format_autocomplete",
       "values": {
-        "validators": "ignore_missing clean_format unicode",
+        "validators": "ignore_missing clean_format unicode_safe",
         "form_placeholder": "eg. CSV, XML or JSON",
         "form_attrs": {
           "data-module": "autocomplete",

ckanext/switzerland/tests/base_ftp_harvester_tests.py

@@ -1,14 +1,14 @@
 import json
 import os
 import shutil
+import unittest
 from datetime import datetime
 
 import ckan.model as model
-from ckan.lib import search, uploader
+from ckan.lib import uploader
 from ckan.lib.dictization.model_dictize import resource_dictize
 from ckan.logic import get_action
 from fs.memoryfs import MemoryFS
-from nose.tools import assert_equal
 
 from ckanext.harvest import model as harvester_model
 from ckanext.harvest.tests.factories import HarvestJobObj, HarvestSourceObj
@@ -17,7 +17,7 @@
 from . import data
 
 
-class BaseSBBHarvesterTests(object):
+class BaseSBBHarvesterTests(unittest.TestCase):
     harvester_class = None
 
     def run_harvester(
@@ -31,6 +31,7 @@ def run_harvester(
         max_revisions=None,
         infoplus=None,
         ist_file=None,
+        ftp_server=None,
     ):
         data.harvest_user()
         self.user = data.user()
@@ -59,6 +60,8 @@ def run_harvester(
             config["infoplus"] = infoplus
         if ist_file:
             config["ist_file"] = ist_file
+        if ftp_server:
+            config["ftp_server"] = ftp_server
 
         source = HarvestSourceObj(
             url="http://example.com/harvest",
@@ -70,8 +73,8 @@ def run_harvester(
         job = HarvestJobObj(source=source, run=False)
         run_harvest_job(job, harvester)
 
-        assert_equal(harvester_model.HarvestGatherError.count(), 0)
-        assert_equal(harvester_model.HarvestObjectError.count(), 0)
+        self.assertEqual(harvester_model.HarvestGatherError.count(), 0)
+        self.assertEqual(harvester_model.HarvestObjectError.count(), 0)
 
     def get_dataset(self, name=data.dataset_name):
         return get_action("ogdch_dataset_by_identifier")({}, {"identifier": name})
@@ -84,21 +87,21 @@ def get_filesystem(self, filename=data.filename):
         fs.makedir(data.environment)
         fs.makedir(os.path.join(data.environment, data.folder))
         path = os.path.join(data.environment, data.folder, filename)
-        fs.setcontents(path, data.dataset_content_1)
-        fs.settimes(path, modified_time=datetime(2000, 1, 1))
+        fs.writetext(path, data.dataset_content_1)
+        fs.settimes(path, modified=datetime(2000, 1, 1))
         return fs
 
     def assert_resource_data(self, resource_id, resource_data):
         resource_obj = model.Resource.get(resource_id)
         resource = resource_dictize(resource_obj, {"model": model})
         path = uploader.ResourceUpload(resource).get_path(resource_id)
         with open(path) as f:
-            assert_equal(f.read(), resource_data)
+            self.assertEqual(f.read(), resource_data)
 
     def assert_resource(self, resource_obj, exists):
         resource = resource_dictize(resource_obj, {"model": model})
         path = uploader.ResourceUpload(resource).get_path(resource_obj.id)
-        assert_equal(os.path.exists(path), exists)
+        self.assertEqual(os.path.exists(path), exists)
 
     def assert_resource_exists(self, resource_obj):
         self.assert_resource(resource_obj, True)
@@ -107,8 +110,6 @@ def assert_resource_deleted(self, resource_obj):
         self.assert_resource(resource_obj, False)
 
     def _cleanup(self):
-        model.repo.rebuild_db()  # clear database
-        search.clear_all()  # clear solr search index
         if os.path.exists("/tmp/ckan_storage_path/"):
             shutil.rmtree("/tmp/ckan_storage_path/")
 

ckanext/switzerland/tests/data.py

@@ -1,8 +1,6 @@
 # coding=utf-8
-from ckan import model
 from ckan.lib.munge import munge_filename, munge_name
 from ckan.tests import factories
-from paste.registry import Registry
 
 environment = "Test"
 folder = "Dataset"
@@ -26,9 +24,7 @@
 0000026   7.449206  46.943904 534    % Bern, Helvetiaplatz
 0000028   6.838953  46.949588 491    % Boudry Littorail
 0000030   8.711041  46.766423 831    % Golzern, Talstation Seilbahn
-0000033   9.680469  47.544341 0      % Lindau Hbf""".encode(
-    "iso-8859-1"
-)
+0000033   9.680469  47.544341 0      % Lindau Hbf"""
 
 bahnhof_file_csv = """StationID,Longitude,Latitude,Höhe,Remark
 0000006,7.549783,47.216111,441,St. Katharinen
@@ -44,11 +40,7 @@
 0000028,6.838953,46.949588,491,Boudry Littorail
 0000030,8.711041,46.766423,831,"Golzern, Talstation Seilbahn"
 0000033,9.680469,47.544341,0,Lindau Hbf
-""".replace(
-    "\n", "\r\n"
-).encode(
-    "utf-8"
-)
+"""
 
 infoplus_config = [
     {"from": 1, "to": 7, "name": "StationID"},
@@ -81,27 +73,15 @@
 25.09.2016;85:11:19667:001;85:11;SBB;Schweizerische Bundesbahnen SBB;Zug;19667;S16;;S;false;false;8503000;Zürich HB;25.09.2016 17:43;25.09.2016 17:43:13;GESCHAETZT;25.09.2016 17:45;25.09.2016 17:45:39;GESCHAETZT;false
 25.09.2016;85:11:19667:001;85:11;SBB;Schweizerische Bundesbahnen SBB;Zug;19667;S16;;S;false;false;8503003;Zürich Stadelhofen;25.09.2016 17:47;25.09.2016 17:48:09;GESCHAETZT;25.09.2016 17:48;25.09.2016 17:49:54;GESCHAETZT;false
 25.09.2016;85:11:19667:001;85:11;SBB;Schweizerische Bundesbahnen SBB;Zug;19667;S16;;S;false;false;8503004;Zürich Tiefenbrunnen;25.09.2016 17:50;25.09.2016 17:52:03;GESCHAETZT;25.09.2016 17:51;25.09.2016 17:52:59;GESCHAETZT;false
-""".replace(
-    "\n", "\r\n"
-)
+"""
 
 
 def user():
     return factories.User()
 
 
 def harvest_user():
-    factories.User(id="harvest", sysadmin=True)
-
-    # setup the pylons context object c, required by datapusher
-    registry = Registry()
-    registry.prepare()
-    import pylons
-
-    c = pylons.util.AttribSafeContextObj()
-    registry.register(pylons.c, c)
-    pylons.c.user = "harvest"
-    pylons.c.userobj = model.User.get("harvest")
+    return factories.User(id="harvest", sysadmin=True)
 
 
 def organization(user):

ckanext/switzerland/tests/fixtures/__init__.py

@@ -0,0 +1,16 @@
+# Copied from ckanext-harvester.
+
+import pytest
+
+import ckanext.harvest.model as harvest_model
+from ckanext.harvest import queue
+
+
+@pytest.fixture
+def harvest_setup():
+    harvest_model.setup()
+
+
+@pytest.fixture
+def clean_queues():
+    queue.purge_queues()