#45 Migrate to CKAN 2.10

.github/workflows/test.yml

@@ -5,9 +5,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v3
         with:
-          python-version: 3.9
+          python-version: "3.10"
       - name: Install dependencies
         run: pip install flake8 pytest
       - name: Lint with flake8
@@ -21,7 +21,7 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        ckan-version: [2.9]
+        ckan-version: ["2.10"]
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
@@ -30,7 +30,7 @@ jobs:
       image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
     services:
       solr:
-        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
       postgres:
         image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:
@@ -69,7 +69,7 @@ jobs:
       run: |
         crudini --set --verbose --list --list-sep=\  test.ini app:main ckan.plugins harvest ckan_harvester
         ckan -c test.ini db init
-        ckan -c test.ini harvester initdb
+        ckan -c test.ini db upgrade -p harvest
     - name: Run tests
       run: pytest --ckan-ini=test.ini --cov=ckanext.multilang --cov-report=xml --cov-append --disable-warnings ckanext/multilang/tests
 

README.md

@@ -49,6 +49,8 @@ To install ckanext-multilang:
     cd ckanext-multilang
 
     pip install -e .
+
+    python setup.py develop
 
 3. Initialize the DB with the mandatory Tables needed for localized records:
 

ckanext/multilang/harvesters/multilang.py

@@ -5,7 +5,7 @@
 
 import ckan.lib.dictization.model_dictize as model_dictize
 import ckan.lib.search as search
-from ckan.lib.base import model
+from ckan import model
 from ckan.common import config
 from ckan.model import Package, Session
 from ckan.plugins.core import SingletonPlugin
@@ -16,7 +16,7 @@
     TagMultilang,
 )
 from ckanext.spatial.harvesters.csw import CSWHarvester
-from ckanext.spatial.model import (
+from ckanext.spatial.harvested_metadata import (
     ISODocument,
     ISOElement,
     ISOKeyword,
@@ -160,43 +160,49 @@ def get_package_dict(self, iso_values, harvest_object):
         package_dict = super(MultilangHarvester, self).get_package_dict(iso_values, harvest_object)
 
         self._package_dict = {}
-
+        
         harvester_config = self.source_config.get('ckan_locales_mapping', {})
         if harvester_config:
             self._ckan_locales_mapping = harvester_config
             log.info('::::: ckan_locales_mapping entry found in harvester configuration :::::')
-
+    
         if iso_values['abstract-text'] and iso_values['title-text']:
             log.debug('::::: Collecting localised data from the metadata abstract :::::')
             localised_abstracts = []
             for abstract_entry in iso_values['abstract-text']:
                 if abstract_entry['text'] and abstract_entry['locale'].lower()[1:]:
-                    if self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]]:
-                        localised_abstracts.append({
-                            'text': abstract_entry['text'],
-                            'locale': self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]]
-                        })
-                    else:
-                        log.warning('Locale Mapping not found for metadata abstract, entry skipped!')
+                    # We comment out this if because abstract_entry['locale'] is equal with the CKAN
+                    # definition so we don't need the ckan_locales_mapping for now
+                    # if self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]]:
+                    localised_abstracts.append({
+                        'text': abstract_entry['text'],
+                        'locale': abstract_entry['locale'].lower()[1:]
+                        # 'locale': self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]]
+                    })
+                    #else:
+                    #   log.warning('Locale Mapping not found for metadata abstract, entry skipped!')
                 else:
                     log.warning('TextGroup data not available for metadata abstract, entry skipped!')
 
             log.debug('::::: Collecting localized data from the metadata title :::::')
             localised_titles = []
             for title_entry in iso_values['title-text']:
                 if title_entry['text'] and title_entry['locale'].lower()[1:]:
-                    if self._ckan_locales_mapping[title_entry['locale'].lower()[1:]]:
-                        localised_titles.append({
-                            'text': title_entry['text'],
-                            'locale': self._ckan_locales_mapping[title_entry['locale'].lower()[1:]]
-                        })
-                    else:
-                        log.warning('Locale Mapping not found for metadata title, entry skipped!')
+                    # the same as above
+                    # if self._ckan_locales_mapping[title_entry['locale'].lower()[1:]]:
+                    localised_titles.append({
+                        'text': title_entry['text'],
+                        # 'locale': self._ckan_locales_mapping[title_entry['locale'].lower()[1:]]
+                        'locale': title_entry['locale'].lower()[1:]
+                    })
+                    #else:
+                    #    log.warning('Locale Mapping not found for metadata title, entry skipped!')
                 else:
                     log.warning('TextGroup data not available for metadata title, entry skipped!')
 
             localised_titles.append({
                 'text': iso_values['title'],
+                # because iso_values['metadata-language']=ita we remove the last letter
                 'locale': self._ckan_locales_mapping[iso_values['metadata-language'].lower()]
             })
 
@@ -231,14 +237,15 @@ def get_package_dict(self, iso_values, harvest_object):
 
                 for tag_localized in tag_localized_entry:
                     if tag_localized['text'] and tag_localized['locale'].lower()[1:]:
-                        if self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]]:
-                            localized_tags.append({
-                                'text': tag_localized['name'],
-                                'localized_text': tag_localized['text'],
-                                'locale': self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]]
-                            })
-                        else:
-                            log.warning('Locale Mapping not found for metadata keyword: %r, entry skipped!', tag_localized['name'])
+                        # if self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]]:
+                        localized_tags.append({
+                            'text': tag_localized['name'],
+                            'localized_text': tag_localized['text'],
+                            # 'locale': self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]]
+                            'locale': tag_localized['locale'].lower()[1:]
+                        })
+                        #else:
+                        #    log.warning('Locale Mapping not found for metadata keyword: %r, entry skipped!', tag_localized['name'])
                     else:
                         log.warning('TextGroup data not available for metadata keyword: %r, entry skipped!', tag_localized['name'])
 
@@ -283,13 +290,14 @@ def handle_organization(self, harvest_object, organisation_mapping, values):
 
                             for entry in party['organisation-name-localized']:
                                 if entry['text'] and entry['locale'].lower()[1:]:
-                                    if self._ckan_locales_mapping[entry['locale'].lower()[1:]]:
-                                        localized_org.append({
-                                            'text': org.get('value_' + self._ckan_locales_mapping[entry['locale'].lower()[1:]]) or entry['text'],
-                                            'locale': self._ckan_locales_mapping[entry['locale'].lower()[1:]]
-                                        })
-                                    else:
-                                        log.warning('Locale Mapping not found for organization name, entry skipped!')
+                                    # if self._ckan_locales_mapping[entry['locale'].lower()[1:]]:
+                                    localized_org.append({
+                                        'text': org.get('value_' + self._ckan_locales_mapping[entry['locale'].lower()[1:]]) or entry['text'],
+                                        # 'locale': self._ckan_locales_mapping[entry['locale'].lower()[1:]]
+                                        'locale': entry['locale'].lower()[1:]
+                                    })
+                                    #else:
+                                    #    log.warning('Locale Mapping not found for organization name, entry skipped!')
                                 else:
                                     log.warning('TextGroup data not available for organization name, entry skipped!')
 

ckanext/multilang/tests/conftest.py

@@ -3,16 +3,28 @@
 import pytest
 
 from ckan.tests.pytest_ckan.fixtures import clean_db
-from ckanext.harvest.tests.fixtures import harvest_setup
-from ckanext.spatial.tests.conftest import clean_postgis, spatial_setup
+# the following functions doesn't exist in the new versions of ckanext-harvest and ckanext-spatial
+# from ckanext.harvest.tests.fixtures import harvest_setup
+# from ckanext.spatial.tests.conftest import clean_postgis, spatial_setup
 from ckanext.multilang.model import setup_db as multilang_setup_db
 
 
 @pytest.fixture
 def multilang_setup():
     multilang_setup_db()
 
+@pytest.fixture
+def clean_multilang_db(clean_db, multilang_setup):
+    return [
+        # clean_postgis,
+        clean_db,
+        # clean_index()
+        # harvest_setup,
+        # spatial_setup,
+        multilang_setup,
+        ]
 
+'''
 @pytest.fixture
 def clean_multilang_db(clean_postgis, clean_db, harvest_setup, spatial_setup, multilang_setup):
     return [
@@ -23,3 +35,4 @@ def clean_multilang_db(clean_postgis, clean_db, harvest_setup, spatial_setup, mu
         spatial_setup,
         multilang_setup,
         ]
+'''

setup.py

@@ -64,9 +64,10 @@
     # project is installed. For an analysis of "install_requires" vs pip's
     # requirements files see:
     # https://packaging.python.org/en/latest/technical.html#install-requires-vs-requirements-files
-    install_requires=[
-        'ckan>=2.9',
-    ],
+    # Uncomment the following three lines if CKAN isn't isntalled in your system
+    #install_requires=[
+    #    'ckan>=2.9',
+    #],
     python_requires=">=3.7",
 
     # If there are data files included in your packages that need to be