Merge pull request #98 from liip/migration

Migration

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        ckan-version: ["2.10"]
+        ckan-version: ["2.11"]
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
@@ -56,12 +56,12 @@ jobs:
           -v "${{ github.workspace }}/solr/stopwords_en.txt":"$SOLR_CONFIG_CKAN_DIR/stopwords_en.txt" \
           -v "${{ github.workspace }}/solr/stopwords_fr.txt":"$SOLR_CONFIG_CKAN_DIR/stopwords_fr.txt" \
           -v "${{ github.workspace }}/solr/stopwords_it.txt":"$SOLR_CONFIG_CKAN_DIR/stopwords_it.txt" \
-          ckan/ckan-solr:${{ matrix.ckan-version }}
+          ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
         docker start test_solr
     - name: Create ckan container
       run: |
         /usr/bin/docker create --name test_ckan --network ${{ job.container.network }} --network-alias ckan \
-          --env-file ${{ github.workspace }}/github-workflow.env \
+          --env-file ${{ github.workspace }}/github-workflow.env --user root \
           -v "/var/run/docker.sock":"/var/run/docker.sock" \
           -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" \
           -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" \

README.md

@@ -1,11 +1,12 @@
 ckanext-switzerland
 ===================
 
-CKAN extension for DCAT-AP Switzerland, templates and different plugins including FTP- and S3-Harvester for [opentransportdata.swiss](https://opentransportdata.swiss).
+CKAN extension for DCAT-AP Switzerland, templates and different plugins including FTP- and S3-Harvester for
+[opentransportdata.swiss](https://opentransportdata.swiss).
 
 ## Requirements
 
-- CKAN 2.8+
+- CKAN 2.10+
 - ckanext-fluent
 - ckanext-harvest
 - ckanext-scheming
@@ -35,15 +36,12 @@ To install ckanext-switzerland:
 
 This extension uses the following config options (.ini file)
 
-    # the URL of the WordPress AJAX interface
-    ckanext.switzerland.wp_ajax_url = https://wp/wp-admin/admin-ajax.php
-    ckanext.switzerland.wp_ajax_url = http://wp/cms/wp-admin/admin-ajax.php
-    ckanext.switzerland.wp_template_url = http://wp/cms/wp-admin/admin-post.php?action=get_nav
-    ckanext.switzerland.wp_url = http://wp
+    # For linking to the associated WordPress site from the header logo
+    ckanext.switzerland.wp_url
 
-    # matomo config
-    matomo.site_id = 1
-    matomo.url = stats.opentransportdata.swiss
+    # For managing cookie consent and tracking with OneTrust and Matomo
+    ckanext.switzerland.cookie_law_url
+    ckanext.switzerland.cookie_law_id
 
 ## Development Installation
 
@@ -183,7 +181,7 @@ The harvester will receive its configuration from the Database.
 This configuration will be passed to the `StorageAdapterFactory`. 
 This object has for sole responsibility to read the configuration, and decide which Storage Adapter to instantiate. 
 
-THe two possibilities are `S3StorageAdapter` or `FTPStorageAdapter`. 
+The two possibilities are `S3StorageAdapter` or `FTPStorageAdapter`. 
 Both classes extends the `StorageAdapterBase` class. 
 In this base class, designed as an abstract class (meaning all methods are there, 
 and the ones that have to be reimplement just throw `NotImplementedException`), 
@@ -193,3 +191,53 @@ but also some common function to manage the local folder used by the harvesters
 Each implementation (`S3StorageAdapter` and `FTPStorageAdapter`), are responsible to get their storage configuration, 
 from the storage identifier received from the harvester configuration. 
 Each implementation is also unit tested, see respectively `TestS3StorageAdapter` and `TestFTPStorageAdapter` classes.
+
+# Updating the translations
+
+The translation files for this ckanext are found in `i18n/`:
+    - `ckanext-switzerland.pot`: this is a template file containing all the translatable strings found in the code
+    - `{LANG}/LC_MESSAGES/ckanext-switzerland.po`: one translation file for each of our supported languages apart from
+English (German, French and Italian)
+    - `{LANG}/LC_MESSAGES/ckanext-switzerland.mo`: the compiled translations in binary format
+
+If you update the code and add or remove translatable strings, you will have to update the translations too.
+
+The following instructions assume you are working inside a docker container based on the official [Docker Compose setup
+for CKAN](https://github.com/ckan/ckan-docker/).
+
+1. Enter the container and go to the ckanext-switzerland directory:
+    ```shell
+    docker compose exec ckan bash
+    cd src_extensions/ckanext-switzerland/
+    ```
+2. Update the `.pot`  file:
+    ```shell
+    python setup.py extract_messages
+    ```
+3. *Optional* If you have copied templates from core CKAN to override them, you might want to copy the translations
+from CKAN too:
+   ```shell
+   apt-get install gettext
+   msgcat --use-first i18n/de/LC_MESSAGES/ckanext-switzerland.po ../../src/ckan/ckan/i18n/de/LC_MESSAGES/ckan.po > temp_de.po
+   cp temp_de.po i18n/de/LC_MESSAGES/ckanext-switzerland.po
+   msgcat --use-first i18n/fr/LC_MESSAGES/ckanext-switzerland.po ../../src/ckan/ckan/i18n/fr/LC_MESSAGES/ckan.po > temp_fr.po
+   cp temp_fr.po i18n/fr/LC_MESSAGES/ckanext-switzerland.po
+   msgcat --use-first i18n/it/LC_MESSAGES/ckanext-switzerland.po ../../src/ckan/ckan/i18n/it/LC_MESSAGES/ckan.po > temp_it.po
+   cp temp_it.po i18n/it/LC_MESSAGES/ckanext-switzerland.po
+   rm temp_de.po temp_fr.po temp_it.po
+   ```
+   This adds a lot of translations to our `.po`  files that we don't want, but they will be removed in the next step.
+4. Update the `.po` files:
+   ```shell
+   python setup.py update_catalog --ignore-obsolete=true --no-fuzzy-matching
+   ```
+   To update one language at a time, add `-l de` (etc.) to the command.
+5. Check the `.po` files and add any new translations that are needed.
+6. Update the `.mo` files:
+   ```shell
+   python setup.py compile_catalog
+   ```
+7. Restart the docker container:
+   ```shell
+   docker compose restart ckan
+   ```

bin/install_test_requirements.sh

@@ -6,18 +6,14 @@ pip install --ignore-installed -r /__w/ckanext-switzerland/ckanext-switzerland/d
 pip install -e /__w/ckanext-switzerland/ckanext-switzerland/
 
 # Install ckanext dependencies
-pip install -e git+https://github.com/ckan/ckanext-dcat.git@v1.5.1#egg=ckanext-dcat
-pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v1.5.1/requirements.txt
-pip install -e git+https://gitlab.liip.ch/odp_oev_schweiz/ckanext-harvest.git@v1.0.2#egg=ckanext-harvest
-pip install -r https://gitlab.liip.ch/odp_oev_schweiz/ckanext-harvest/-/raw/v1.0.2/requirements.txt
-pip install -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming
+pip install -e git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat
+pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/master/requirements.txt
+pip install -e git+https://gitlab.liip.ch/odp_oev_schweiz/ckanext-harvest.git#egg=ckanext-harvest
+pip install -r https://gitlab.liip.ch/odp_oev_schweiz/ckanext-harvest/-/raw/main/requirements.txt
+pip install -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming
 pip install -e git+https://github.com/ckan/ckanext-fluent.git#egg=ckanext-fluent
 pip install -r https://raw.githubusercontent.com/ckan/ckanext-fluent/master/requirements.txt
 
-# Replace default path to CKAN core config file with the one on the container
-sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' /__w/ckanext-switzerland/ckanext-switzerland/test.ini
-
 # Init db and re-enable required plugins
-ckan config-tool /__w/ckanext-switzerland/ckanext-switzerland/test.ini "ckan.plugins = "
 ckan -c /__w/ckanext-switzerland/ckanext-switzerland/test.ini db init
-ckan config-tool /__w/ckanext-switzerland/ckanext-switzerland/test.ini "ckan.plugins = ogdch ogdch_pkg ogdch_res ogdch_group ogdch_org harvest sbb_harvester timetable_harvester datastore fluent scheming_datasets scheming_groups scheming_organizations"
+ckan -c /__w/ckanext-switzerland/ckanext-switzerland/test.ini db pending-migrations --apply

ckanext/switzerland/assets/webassets.yml

@@ -15,11 +15,3 @@ relations:
   extra:
     preload:
       - base/main
-
-search:
-  output: switzerland/search.js
-  contents:
-    - a_vue.js
-    - b_vuevalidator.js
-    - c_search.js
-  filters: rjsmin