Feat/update dependencies (#498)

* Updating SqlAlchemy to 1.4 * Abandoning support for Debian 10 --------- Co-authored-by: Jacobe2169 <[email protected]>
PnX-SI · Dec 7, 2023 · b719991 · b719991
1 parent 0f7cea9
commit b719991
Show file tree

Hide file tree

Showing 11 changed files with 89 additions and 84 deletions.
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -22,12 +22,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        debian-version: [ '10', '11', '12' ]
+        debian-version: [ '11', '12' ]
         include:
-          - debian-version: '10'
-            python-version: "3.7"
-            postgres-version: '11'
-            postgis-version: '2.5'
           - debian-version: '11'
             python-version: '3.9'
             postgres-version: '13'
@@ -110,7 +106,8 @@ jobs:
         GEONATURE_CONFIG_FILE: dependencies/GeoNature/config/test_config.toml
     - name: Install import module backend
       run: |
-        pip install --editable .
+        pip install -r requirements-dev.in
+        pip install .
     - name: Install import module database
       run: |
         geonature upgrade-modules-db IMPORT

diff --git a/backend/gn_module_import/admin.py b/backend/gn_module_import/admin.py
@@ -2,7 +2,7 @@
 from itertools import groupby
 from pprint import pformat
 
-from flask import Markup
+from markupsafe import Markup
 from flask_admin.contrib.sqla import ModelView
 from flask_admin.form import BaseForm
 from wtforms.validators import StopValidation

diff --git a/backend/gn_module_import/checks/sql/__init__.py b/backend/gn_module_import/checks/sql/__init__.py
@@ -38,7 +38,7 @@ def do_nomenclatures_mapping(imprt, fields):
         source_field = getattr(ImportSyntheseData, field.source_field)
         # This CTE return the list of source value / cd_nomenclature for a given nomenclature type
         cte = (
-            select([column("key").label("value"), column("value").label("cd_nomenclature")])
+            select(column("key").label("value"), column("value").label("cd_nomenclature"))
             .select_from(sa.func.JSON_EACH_TEXT(TImports.contentmapping[field.mnemonique]))
             .where(TImports.id_import == imprt.id_import)
             .cte("cte")
@@ -54,7 +54,7 @@ def do_nomenclatures_mapping(imprt, fields):
             .where(TNomenclatures.id_type == BibNomenclaturesTypes.id_type)
             .values({field.synthese_field: TNomenclatures.id_nomenclature})
         )
-        db.session.execute(stmt)
+        db.session.execute(stmt, execution_options=dict({"synchronize_session": 'fetch'}))
 
     for field in BibFields.query.filter(BibFields.mnemonique != None).all():
         if (
@@ -181,7 +181,7 @@ def check_referential(imprt, field, reference_field, error_type, reference_table
     # We outerjoin the referential, and select rows where there is a value in synthese field
     # but no value in referential, which means no value in the referential matched synthese field.
     cte = (
-        select([ImportSyntheseData.line_no])
+        select(ImportSyntheseData.line_no)
         .select_from(
             join(
                 ImportSyntheseData,
@@ -234,22 +234,18 @@ def set_altitudes(imprt, fields):
         return
     altitudes = (
         select(
-            [
-                column("altitude_min"),
-                column("altitude_max"),
-            ]
+            column("altitude_min"),
+            column("altitude_max"),
         )
         .select_from(func.ref_geo.fct_get_altitude_intersection(ImportSyntheseData.the_geom_local))
         .lateral("altitudes")
     )
     cte = (
         select(
-            [
-                ImportSyntheseData.id_import,
-                ImportSyntheseData.line_no,
-                altitudes.c.altitude_min,
-                altitudes.c.altitude_max,
-            ]
+            ImportSyntheseData.id_import,
+            ImportSyntheseData.line_no,
+            altitudes.c.altitude_min,
+            altitudes.c.altitude_max,
         )
         .where(ImportSyntheseData.id_import == imprt.id_import)
         .where(ImportSyntheseData.the_geom_local != None)
@@ -270,27 +266,23 @@ def set_altitudes(imprt, fields):
         .values(
             {
                 ImportSyntheseData.altitude_min: sa.case(
-                    whens=[
-                        (
-                            sa.or_(
-                                ImportSyntheseData.src_altitude_min == None,
-                                ImportSyntheseData.src_altitude_min == "",
-                            ),
-                            cte.c.altitude_min,
+                    (
+                        sa.or_(
+                            ImportSyntheseData.src_altitude_min == None,
+                            ImportSyntheseData.src_altitude_min == "",
                         ),
-                    ],
+                        cte.c.altitude_min,
+                    ),
                     else_=ImportSyntheseData.altitude_min,
                 ),
                 ImportSyntheseData.altitude_max: sa.case(
-                    whens=[
-                        (
-                            sa.or_(
-                                ImportSyntheseData.src_altitude_max == None,
-                                ImportSyntheseData.src_altitude_max == "",
-                            ),
-                            cte.c.altitude_max,
+                    (
+                        sa.or_(
+                            ImportSyntheseData.src_altitude_max == None,
+                            ImportSyntheseData.src_altitude_max == "",
                         ),
-                    ],
+                        cte.c.altitude_max,
+                    ),
                     else_=ImportSyntheseData.altitude_max,
                 ),
             }
@@ -302,7 +294,7 @@ def set_altitudes(imprt, fields):
             "altitude_max": BibFields.query.filter_by(name_field="altitude_max").one(),
         }
     )
-    db.session.execute(stmt)
+    db.session.execute(stmt.execution_options(synchronize_session="fetch"))
 
 
 def get_duplicates_query(imprt, synthese_field, whereclause=sa.true()):
@@ -312,19 +304,17 @@ def get_duplicates_query(imprt, synthese_field, whereclause=sa.true()):
     )
     partitions = (
         select(
-            [
-                array_agg(ImportSyntheseData.line_no)
-                .over(
-                    partition_by=synthese_field,
-                )
-                .label("duplicate_lines")
-            ]
+            array_agg(ImportSyntheseData.line_no)
+            .over(
+                partition_by=synthese_field,
+            )
+            .label("duplicate_lines")
         )
         .where(whereclause)
         .alias("partitions")
     )
     duplicates = (
-        select([func.unnest(partitions.c.duplicate_lines).label("lines")])
+        select(func.unnest(partitions.c.duplicate_lines).label("lines"))
         .where(func.array_length(partitions.c.duplicate_lines, 1) > 1)
         .alias("duplicates")
     )
@@ -557,7 +547,6 @@ def set_geom_from_area_code(imprt, source_column, area_type_filter):
         )
         .join(
             LAreas.area_type,
-            aliased=True,
         )
         .filter(area_type_filter)
         .with_entities(
@@ -580,7 +569,7 @@ def set_geom_from_area_code(imprt, source_column, area_type_filter):
         .where(ImportSyntheseData.id_import == cte.c.id_import)
         .where(ImportSyntheseData.line_no == cte.c.line_no)
     )
-    db.session.execute(stmt)
+    db.session.execute(stmt.execution_options(synchronize_session="fetch"))
 
 
 def report_erroneous_rows(imprt, error_type, error_column, whereclause):
@@ -602,20 +591,18 @@ def report_erroneous_rows(imprt, error_type, error_column, whereclause):
         )
     else:
         cte = (
-            select([ImportSyntheseData.line_no])
+            select(ImportSyntheseData.line_no)
             .where(ImportSyntheseData.imprt == imprt)
             .where(whereclause)
             .cte("cte")
         )
     error = select(
-        [
-            literal(imprt.id_import).label("id_import"),
-            literal(error_type.pk).label("id_type"),
-            array_agg(
-                aggregate_order_by(cte.c.line_no, cte.c.line_no),
-            ).label("rows"),
-            literal(error_column).label("error_column"),
-        ]
+        literal(imprt.id_import).label("id_import"),
+        literal(error_type.pk).label("id_type"),
+        array_agg(
+            aggregate_order_by(cte.c.line_no, cte.c.line_no),
+        ).label("rows"),
+        literal(error_column).label("error_column"),
     ).alias("error")
     stmt = insert(ImportUserError).from_select(
         names=[
@@ -624,7 +611,7 @@ def report_erroneous_rows(imprt, error_type, error_column, whereclause):
             ImportUserError.rows,
             ImportUserError.column,
         ],
-        select=(select([error]).where(error.c.rows != None)),
+        select=(select(error).where(error.c.rows != None)),
     )
     db.session.execute(stmt)
 

diff --git a/backend/gn_module_import/routes/imports.py b/backend/gn_module_import/routes/imports.py
@@ -5,7 +5,10 @@
 
 from flask import request, current_app, jsonify, g, stream_with_context, send_file
 from werkzeug.exceptions import Conflict, BadRequest, Forbidden, Gone
-from werkzeug.urls import url_quote
+# url_quote was deprecated in werkzeug 3.0 https://stackoverflow.com/a/77222063/5807438
+from urllib.parse import (
+    quote as url_quote,
+)  
 from sqlalchemy import or_, func, desc
 from sqlalchemy.inspection import inspect
 from sqlalchemy.orm import joinedload, Load, load_only, undefer, contains_eager, class_mapper
@@ -120,7 +123,7 @@ def get_import_list(scope):
         .join(TImports.dataset, isouter=True)
         .join(TImports.authors, isouter=True)
         .filter_by_scope(scope)
-        .filter(or_(*filters))
+        .filter(or_(*filters) if len(filters) > 0 else True)
         .order_by(order_by)
         .paginate(page=page, error_out=False, max_per_page=limit)
     )
@@ -185,7 +188,7 @@ def upload_file(scope, import_id):
             dataset_id = int(request.form["datasetId"])
         except ValueError:
             raise BadRequest(description="'datasetId' must be an integer.")
-        dataset = TDatasets.query.get(dataset_id)
+        dataset = db.session.get(TDatasets, (dataset_id))
         if dataset is None:
             raise BadRequest(description=f"Dataset '{dataset_id}' does not exist.")
         if not dataset.has_instance_permission(scope):  # FIXME wrong scope
@@ -369,13 +372,13 @@ def get_import_values(scope, import_id):
             # the file do not contain this field expected by the mapping
             continue
         # TODO: vérifier que l’on a pas trop de valeurs différentes ?
-        column = field.source_column
+        column = getattr(ImportSyntheseData, field.source_column)
         values = [
-            getattr(data, column)
+            getattr(data, field.source_column)
             for data in (
                 ImportSyntheseData.query.filter_by(imprt=imprt)
                 .options(load_only(column))
-                .distinct(getattr(ImportSyntheseData, column))
+                .distinct(column)
                 .all()
             )
         ]
@@ -454,13 +457,14 @@ def preview_valid_data(scope, import_id):
         BibFields.name_field.in_(imprt.fieldmapping.keys()),
     ).all()
     columns = [field.name_field for field in fields]
+    columns_instance = [getattr(ImportSyntheseData, field.name_field) for field in fields]
     valid_data = (
         ImportSyntheseData.query.filter_by(
             imprt=imprt,
             valid=True,
         )
         .options(
-            load_only(*columns),
+            load_only(*columns_instance),
         )
         .limit(100)
     )

diff --git a/backend/gn_module_import/tasks.py b/backend/gn_module_import/tasks.py
@@ -46,7 +46,7 @@
 @celery_app.task(bind=True)
 def do_import_checks(self, import_id):
     logger.info(f"Starting verification of import {import_id}.")
-    imprt = TImports.query.get(import_id)
+    imprt = db.session.get(TImports, import_id)
     if imprt is None or imprt.task_id != self.request.id:
         logger.warning("Task cancelled, doing nothing.")
         return
@@ -131,7 +131,7 @@ def update_batch_progress(batch, step):
             progress = 0.4 + ((i + 1) / len(sql_checks)) * 0.6
             self.update_state(state="PROGRESS", meta={"progress": progress})
 
-    imprt = TImports.query.with_for_update(of=TImports).get(import_id)
+    imprt = db.session.get(TImports, import_id, with_for_update={"of": TImports})
     if imprt is None or imprt.task_id != self.request.id:
         logger.warning("Task cancelled, rollback changes.")
         db.session.rollback()
@@ -154,7 +154,7 @@ def update_batch_progress(batch, step):
 @celery_app.task(bind=True)
 def do_import_in_synthese(self, import_id):
     logger.info(f"Starting insertion in synthese of import {import_id}.")
-    imprt = TImports.query.get(import_id)
+    imprt = db.session.get(TImports, import_id)
     if imprt is None or imprt.task_id != self.request.id:
         logger.warning("Task cancelled, doing nothing.")
         return
@@ -170,7 +170,7 @@ def do_import_in_synthese(self, import_id):
         )
     import_data_to_synthese(imprt)
     ImportSyntheseData.query.filter_by(imprt=imprt).delete()
-    imprt = TImports.query.with_for_update(of=TImports).get(import_id)
+    imprt = db.session.get(TImports, import_id, with_for_update={"of": TImports})
     if imprt is None or imprt.task_id != self.request.id:
         logger.warning("Task cancelled, rollback changes.")
         db.session.rollback()

diff --git a/backend/gn_module_import/tests/conftest.py b/backend/gn_module_import/tests/conftest.py
@@ -1,2 +1,3 @@
 from geonature.tests.fixtures import *
 from geonature.tests.fixtures import app, _session, users
+from pypnusershub.tests.fixtures import teardown_logout_user