diff --git a/backend/gn_module_import/blueprint.py b/backend/gn_module_import/blueprint.py index 35e7ce0f..9f1d5c8a 100644 --- a/backend/gn_module_import/blueprint.py +++ b/backend/gn_module_import/blueprint.py @@ -1,14 +1,27 @@ -from flask import Blueprint +from flask import Blueprint, current_app, g +from geonature.core.gn_commons.models import TModules + +from gn_module_import.models import Destination import gn_module_import.admin # noqa: F401 blueprint = Blueprint("import", __name__, template_folder="templates") + +@blueprint.url_value_preprocessor +def set_current_destination(endpoint, values): + if current_app.url_map.is_endpoint_expecting(endpoint, "destination"): + g.destination = values["destination"] = Destination.query.filter( + Destination.code == values["destination"] + ).first_or_404() + + from .routes import ( imports, mappings, + fields, ) - from .commands import fix_mappings + blueprint.cli.add_command(fix_mappings) diff --git a/backend/gn_module_import/migrations/2b0b3bd0248c_multidest.py b/backend/gn_module_import/migrations/2b0b3bd0248c_multidest.py new file mode 100644 index 00000000..19052acf --- /dev/null +++ b/backend/gn_module_import/migrations/2b0b3bd0248c_multidest.py @@ -0,0 +1,239 @@ +"""multidest + +Revision ID: 2b0b3bd0248c +Revises: 2896cf965dd6 +Create Date: 2023-10-20 09:05:49.973738 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.schema import Table, MetaData + + +# revision identifiers, used by Alembic. +revision = "2b0b3bd0248c" +down_revision = "2896cf965dd6" +branch_labels = None +depends_on = None + + +def upgrade(): + meta = MetaData(bind=op.get_bind()) + module = Table("t_modules", meta, autoload=True, schema="gn_commons") + destination = op.create_table( + "bib_destinations", + sa.Column("id_destination", sa.Integer, primary_key=True), + sa.Column( + "id_module", + sa.Integer, + sa.ForeignKey("gn_commons.t_modules.id_module", ondelete="CASCADE"), + ), + sa.Column("code", sa.String(64), unique=True), + sa.Column("label", sa.String(128)), + sa.Column("table_name", sa.String(64)), + schema="gn_imports", + ) + id_synthese_module = ( + op.get_bind() + .execute(sa.select([module.c.id_module]).where(module.c.module_code == "SYNTHESE")) + .scalar() + ) + op.bulk_insert( + destination, + [ + { + "id_module": id_synthese_module, + "code": "synthese", + "label": "synthèse", + "table_name": "t_imports_synthese", + }, + ], + ) + id_synthese_dest = ( + op.get_bind() + .execute( + sa.select([destination.c.id_destination]).where( + destination.c.id_module == id_synthese_module + ) + ) + .scalar() + ) + op.add_column( + "bib_fields", + sa.Column( + "id_destination", + sa.Integer, + sa.ForeignKey("gn_imports.bib_destinations.id_destination"), + nullable=True, + ), + schema="gn_imports", + ) + field = Table("bib_fields", meta, autoload=True, schema="gn_imports") + op.execute(field.update().values({"id_destination": id_synthese_dest})) + op.alter_column( + table_name="bib_fields", column_name="id_destination", nullable=False, schema="gn_imports" + ) + op.add_column( + "t_imports", + sa.Column( + "id_destination", + sa.Integer, + sa.ForeignKey("gn_imports.bib_destinations.id_destination"), + nullable=True, + ), + schema="gn_imports", + ) + imprt = Table("t_imports", meta, autoload=True, schema="gn_imports") + op.execute(imprt.update().values({"id_destination": id_synthese_dest})) + op.alter_column( + table_name="t_imports", column_name="id_destination", nullable=False, schema="gn_imports" + ) + op.add_column( + "t_mappings", + sa.Column( + "id_destination", + sa.Integer, + sa.ForeignKey("gn_imports.bib_destinations.id_destination"), + nullable=True, + ), + schema="gn_imports", + ) + mapping = Table("t_mappings", meta, autoload=True, schema="gn_imports") + op.execute(mapping.update().values({"id_destination": id_synthese_dest})) + op.alter_column( + table_name="t_mappings", column_name="id_destination", nullable=False, schema="gn_imports" + ) + entity = op.create_table( + "bib_entities", + sa.Column("id_entity", sa.Integer, primary_key=True), + sa.Column("id_destination", sa.Integer, sa.ForeignKey(destination.c.id_destination)), + sa.Column("label", sa.String(64)), + sa.Column("order", sa.Integer), + sa.Column("validity_column", sa.String(64)), + schema="gn_imports", + ) + op.bulk_insert( + entity, + [ + { + "id_destination": id_synthese_dest, + "label": "Observations", + "order": 1, + "validity_column": "gn_is_valid", + }, + ], + ) + op.create_table( + "cor_entity_field", + sa.Column( + "id_entity", + sa.Integer, + sa.ForeignKey("gn_imports.bib_entities.id_entity", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "id_field", + sa.Integer, + sa.ForeignKey("gn_imports.bib_fields.id_field", ondelete="CASCADE"), + primary_key=True, + ), + schema="gn_imports", + ) + op.execute( + """ + INSERT INTO + gn_imports.cor_entity_field (id_entity, id_field) + SELECT + e.id_entity, + f.id_field + FROM + gn_commons.t_modules m + JOIN + gn_imports.bib_destinations d ON d.id_module = m.id_module + JOIN + gn_imports.bib_entities e ON e.id_destination = d.id_destination + JOIN + gn_imports.bib_fields f ON f.id_destination = d.id_destination + WHERE m.module_code = 'SYNTHESE'; + """ + ) + op.execute( + """ + INSERT INTO + gn_permissions.t_permissions_available (id_module, id_object, id_action, label, scope_filter) + SELECT + m.id_module, o.id_object, a.id_action, 'Importer des observations', TRUE + FROM + gn_commons.t_modules m, + gn_permissions.t_objects o, + gn_permissions.bib_actions a + WHERE + m.module_code = 'SYNTHESE' + AND + o.code_object = 'ALL' + AND + a.code_action = 'C' + """ + ) + op.execute( + """ + INSERT INTO + gn_permissions.t_permissions (id_role, id_module, id_object, id_action, scope_value) + SELECT + p.id_role, new_module.id_module, new_object.id_object, p.id_action, p.scope_value + FROM + gn_permissions.t_permissions p + JOIN gn_permissions.bib_actions a USING(id_action) + JOIN gn_commons.t_modules m USING(id_module) + JOIN gn_permissions.t_objects o USING(id_object) + JOIN utilisateurs.t_roles r USING(id_role), + gn_commons.t_modules new_module, + gn_permissions.t_objects new_object + WHERE + a.code_action = 'C' AND m.module_code = 'IMPORT' AND o.code_object = 'IMPORT' + AND + new_module.module_code = 'SYNTHESE' AND new_object.code_object = 'ALL'; + """ + ) + # TODO unique constraint + + +def downgrade(): + op.execute( + """ + DELETE FROM + gn_permissions.t_permissions p + USING + gn_permissions.bib_actions a, + gn_commons.t_modules m, + gn_permissions.t_objects o + WHERE + p.id_action = a.id_action AND a.code_action = 'C' + AND + p.id_module = m.id_module AND m.module_code = 'SYNTHESE' + AND + p.id_object = o.id_object AND o.code_object = 'ALL'; + """ + ) + op.execute( + """ + DELETE FROM + gn_permissions.t_permissions_available pa + USING + gn_permissions.bib_actions a, + gn_commons.t_modules m, + gn_permissions.t_objects o + WHERE + pa.id_action = a.id_action AND a.code_action = 'C' + AND + pa.id_module = m.id_module AND m.module_code = 'SYNTHESE' + AND + pa.id_object = o.id_object AND o.code_object = 'ALL'; + """ + ) + op.drop_table("cor_entity_field", schema="gn_imports") + op.drop_table("bib_entities", schema="gn_imports") + op.drop_column(schema="gn_imports", table_name="bib_fields", column_name="id_destination") + op.drop_column(schema="gn_imports", table_name="t_mappings", column_name="id_destination") + op.drop_column(schema="gn_imports", table_name="t_imports", column_name="id_destination") + op.drop_table("bib_destinations", schema="gn_imports") diff --git a/backend/gn_module_import/models.py b/backend/gn_module_import/models.py index 39505a25..e7d60a0c 100644 --- a/backend/gn_module_import/models.py +++ b/backend/gn_module_import/models.py @@ -106,6 +106,52 @@ def __str__(self): return f"" +class Destination(db.Model): + __tablename__ = "bib_destinations" + __table_args__ = {"schema": "gn_imports"} + + id_destination = db.Column(db.Integer, primary_key=True, autoincrement=True) + id_module = db.Column(db.Integer, ForeignKey(TModules.id_module), nullable=True) + code = db.Column(db.String(64), unique=True) + label = db.Column(db.String(128)) + table_name = db.Column(db.String(64)) + + module = relationship(TModules) + + +cor_entity_field = db.Table( + "cor_entity_field", + db.Column( + "id_entity", + db.Integer, + db.ForeignKey("gn_imports.bib_entities.id_entity"), + primary_key=True, + ), + db.Column( + "id_field", + db.Integer, + db.ForeignKey("gn_imports.bib_fields.id_field"), + primary_key=True, + ), + schema="gn_imports", +) + + +@serializable +class Entity(db.Model): + __tablename__ = "bib_entities" + __table_args__ = {"schema": "gn_imports"} + + id_entity = db.Column(db.Integer, primary_key=True, autoincrement=True) + id_destination = db.Column(db.Integer, ForeignKey(Destination.id_destination)) + destination = relationship(Destination) + label = db.Column(db.String(64)) + order = db.Column(db.Integer) + validity_column = db.Column(db.String(64)) + + fields = relationship("BibFields", secondary=cor_entity_field) + + class InstancePermissionMixin: def get_instance_permissions(self, scopes, user=None): if user is None: @@ -167,6 +213,8 @@ class TImports(InstancePermissionMixin, db.Model): AVAILABLE_SEPARATORS = [",", ";"] id_import = db.Column(db.Integer, primary_key=True, autoincrement=True) + id_destination = db.Column(db.Integer, ForeignKey(Destination.id_destination)) + destination = relationship(Destination) format_source_file = db.Column(db.Unicode, nullable=True) srid = db.Column(db.Integer, nullable=True) separator = db.Column(db.Unicode, nullable=True) @@ -531,6 +579,8 @@ class BibFields(db.Model): __table_args__ = {"schema": "gn_imports"} id_field = db.Column(db.Integer, primary_key=True) + id_destination = db.Column(db.Integer, ForeignKey(Destination.id_destination)) + destination = relationship(Destination) name_field = db.Column(db.Unicode, nullable=False, unique=True) source_field = db.Column(db.Unicode, unique=True) synthese_field = db.Column(db.Unicode, unique=True) @@ -602,6 +652,8 @@ class MappingTemplate(db.Model): query_class = MappingQuery id = db.Column(db.Integer, primary_key=True) + id_destination = db.Column(db.Integer, ForeignKey(Destination.id_destination)) + destination = relationship(Destination) label = db.Column(db.Unicode(255), nullable=False) type = db.Column(db.Unicode(10), nullable=False) active = db.Column(db.Boolean, nullable=False, default=True, server_default="true") @@ -655,7 +707,7 @@ class FieldMapping(MappingTemplate): @staticmethod def validate_values(values): fields = ( - BibFields.query.filter_by(display=True) + BibFields.query.filter_by(destination=g.destination, display=True) .with_entities( BibFields.name_field, BibFields.autogenerated, @@ -698,7 +750,9 @@ class ContentMapping(MappingTemplate): @staticmethod def validate_values(values): nomenclature_fields = ( - BibFields.query.filter(BibFields.nomenclature_type != None) + BibFields.query.filter( + BibFields.destination == g.destination, BibFields.nomenclature_type != None + ) .options( joinedload(BibFields.nomenclature_type).joinedload( BibNomenclaturesTypes.nomenclatures diff --git a/backend/gn_module_import/routes/fields.py b/backend/gn_module_import/routes/fields.py new file mode 100644 index 00000000..e29153a3 --- /dev/null +++ b/backend/gn_module_import/routes/fields.py @@ -0,0 +1,106 @@ +from itertools import groupby + +from flask import jsonify +from sqlalchemy.orm import joinedload + +from geonature.core.gn_permissions import decorators as permissions +from pypnnomenclature.models import BibNomenclaturesTypes + +from gn_module_import.models import ( + Entity, + BibFields, + BibThemes, +) + +from gn_module_import.blueprint import blueprint + + +@blueprint.route("//fields", methods=["GET"]) +@permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="IMPORT") +def get_fields(scope, destination): + """ + .. :quickref: Import; Get synthesis fields. + + Get all synthesis fields + Use in field mapping steps + You can find a jsonschema of the returned data in the associated test. + """ + # TODO use selectinload + fields = ( + BibFields.query.filter_by(destination=destination, display=True) + .options(joinedload(BibFields.theme)) + .join(BibThemes) + .order_by(BibThemes.order_theme, BibFields.order_field) + .all() + ) + entities = Entity.query.order_by(Entity.order).all() + data = [] + for entity in entities: + themes = [] + for id_theme, fields in groupby( + filter(lambda field: field in entity.fields, fields), lambda field: field.id_theme + ): + fields = list(fields) + theme = fields[0].theme + themes.append( + { + "theme": theme.as_dict( + fields=[ + "id_theme", + "name_theme", + "fr_label_theme", + "eng_label_theme", + "desc_theme", + ], + ), + "fields": [ + field.as_dict( + fields=[ + "id_field", + "name_field", + "fr_label", + "eng_label", + "desc_field", + "mandatory", + "autogenerated", + "comment", + "multi", + ], + ) + for field in fields + ], + } + ) + data.append( + { + "entity": entity.as_dict(fields=["label"]), + "themes": themes, + } + ) + return jsonify(data) + + +@blueprint.route("//nomenclatures", methods=["GET"]) +def get_nomenclatures(destination): + nomenclature_fields = ( + BibFields.query.filter(BibFields.destination == destination) + .filter(BibFields.nomenclature_type != None) + .options( + joinedload(BibFields.nomenclature_type).joinedload( + BibNomenclaturesTypes.nomenclatures + ), + ) + .all() + ) + return jsonify( + { + field.nomenclature_type.mnemonique: { + "nomenclature_type": field.nomenclature_type.as_dict(), + "nomenclatures": { + nomenclature.cd_nomenclature: nomenclature.as_dict() + for nomenclature in field.nomenclature_type.nomenclatures + }, + } + for field in nomenclature_fields + } + ) diff --git a/backend/gn_module_import/routes/imports.py b/backend/gn_module_import/routes/imports.py index 9749dacc..b89150cf 100644 --- a/backend/gn_module_import/routes/imports.py +++ b/backend/gn_module_import/routes/imports.py @@ -4,24 +4,23 @@ import unicodedata from flask import request, current_app, jsonify, g, stream_with_context, send_file -from werkzeug.exceptions import Conflict, BadRequest, Forbidden, Gone +from werkzeug.exceptions import Conflict, BadRequest, Forbidden, Gone, NotFound from werkzeug.urls import url_quote from sqlalchemy import or_, func, desc from sqlalchemy.inspection import inspect -from sqlalchemy.orm import joinedload, Load, load_only, undefer, contains_eager, class_mapper +from sqlalchemy.orm import joinedload, Load, load_only, undefer, contains_eager from sqlalchemy.orm.attributes import set_committed_value from sqlalchemy.sql.expression import collate from geonature.utils.env import db from geonature.utils.sentry import start_sentry_child from geonature.core.gn_permissions import decorators as permissions -from geonature.core.gn_synthese.models import ( - Synthese, - TSources, -) +from geonature.core.gn_permissions.decorators import login_required +from geonature.core.gn_permissions.tools import get_scopes_by_action +from geonature.core.gn_synthese.models import Synthese from geonature.core.gn_meta.models import TDatasets -from pypnnomenclature.models import BibNomenclaturesTypes, TNomenclatures +from pypnnomenclature.models import TNomenclatures from gn_module_import.models import ( TImports, @@ -48,34 +47,22 @@ IMPORTS_PER_PAGE = 15 -@blueprint.route("/nomenclatures", methods=["GET"]) -def get_nomenclatures(): - nomenclature_fields = ( - BibFields.query.filter(BibFields.nomenclature_type != None) - .options( - joinedload(BibFields.nomenclature_type).joinedload( - BibNomenclaturesTypes.nomenclatures - ), - ) - .all() - ) - return jsonify( - { - field.nomenclature_type.mnemonique: { - "nomenclature_type": field.nomenclature_type.as_dict(), - "nomenclatures": { - nomenclature.cd_nomenclature: nomenclature.as_dict() - for nomenclature in field.nomenclature_type.nomenclatures - }, - } - for field in nomenclature_fields - } - ) +@blueprint.url_value_preprocessor +def resolve_import(endpoint, values): + if current_app.url_map.is_endpoint_expecting(endpoint, "import_id"): + import_id = values.pop("import_id") + if import_id is not None: + imprt = TImports.query.get_or_404(import_id) + if imprt.destination != values.pop("destination"): + raise NotFound + else: + imprt = None + values["imprt"] = imprt -@blueprint.route("/imports/", methods=["GET"]) +@blueprint.route("//imports/", methods=["GET"]) @permissions.check_cruved_scope("R", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def get_import_list(scope): +def get_import_list(scope, destination): """ .. :quickref: Import; Get all imports. @@ -120,6 +107,7 @@ def get_import_list(scope): .join(TImports.dataset, isouter=True) .join(TImports.authors, isouter=True) .filter_by_scope(scope) + .filter(TImports.destination == destination) .filter(or_(*filters)) .order_by(order_by) .paginate(page=page, error_out=False, max_per_page=limit) @@ -134,25 +122,24 @@ def get_import_list(scope): return jsonify(data) -@blueprint.route("/imports//", methods=["GET"]) +@blueprint.route("//imports//", methods=["GET"]) @permissions.check_cruved_scope("R", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def get_one_import(scope, import_id): +def get_one_import(scope, imprt): """ .. :quickref: Import; Get an import. Get an import. """ - imprt = TImports.query.get_or_404(import_id) # check that the user has read permission to this particular import instance: if not imprt.has_instance_permission(scope): raise Forbidden return jsonify(imprt.as_dict()) -@blueprint.route("/imports/upload", defaults={"import_id": None}, methods=["POST"]) -@blueprint.route("/imports//upload", methods=["PUT"]) +@blueprint.route("//imports/upload", defaults={"import_id": None}, methods=["POST"]) +@blueprint.route("//imports//upload", methods=["PUT"]) @permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def upload_file(scope, import_id): +def upload_file(scope, imprt, destination=None): # destination is set when imprt is None """ .. :quickref: Import; Add an import or update an existing import. @@ -161,15 +148,16 @@ def upload_file(scope, import_id): :form file: file to import :form int datasetId: dataset ID to which import data """ - author = g.current_user - if import_id: - imprt = TImports.query.get_or_404(import_id) + if destination is None: + destination = imprt.destination + if imprt: if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.dataset.active: raise Forbidden("Le jeu de données est fermé.") else: - imprt = None + assert destination + author = g.current_user f = request.files["file"] size = get_file_size(f) # value in config file is in Mo @@ -188,11 +176,15 @@ def upload_file(scope, import_id): dataset = TDatasets.query.get(dataset_id) if dataset is None: raise BadRequest(description=f"Dataset '{dataset_id}' does not exist.") - if not dataset.has_instance_permission(scope): # FIXME wrong scope + ds_scope = get_scopes_by_action( + module_code=destination.module.module_code, + object_code="ALL", # TODO object_code should be configurable by destination + )["C"] + if not dataset.has_instance_permission(ds_scope): raise Forbidden(description="Vous n’avez pas les permissions sur ce jeu de données.") if not dataset.active: raise Forbidden("Le jeu de données est fermé.") - imprt = TImports(dataset=dataset) + imprt = TImports(destination=destination, dataset=dataset) imprt.authors.append(author) db.session.add(imprt) else: @@ -211,10 +203,9 @@ def upload_file(scope, import_id): return jsonify(imprt.as_dict()) -@blueprint.route("/imports//decode", methods=["POST"]) +@blueprint.route("//imports//decode", methods=["POST"]) @permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def decode_file(scope, import_id): - imprt = TImports.query.get_or_404(import_id) +def decode_file(scope, imprt): if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.dataset.active: @@ -262,7 +253,7 @@ def decode_file(scope, import_id): columns = next(csvreader) while True: # read full file to ensure that no encoding errors occur next(csvreader) - except UnicodeError as e: + except UnicodeError: raise BadRequest( description="Erreur d’encodage lors de la lecture du fichier source. " "Avez-vous sélectionné le bon encodage de votre fichier ?" @@ -278,10 +269,9 @@ def decode_file(scope, import_id): return jsonify(imprt.as_dict()) -@blueprint.route("/imports//fieldmapping", methods=["POST"]) +@blueprint.route("//imports//fieldmapping", methods=["POST"]) @permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def set_import_field_mapping(scope, import_id): - imprt = TImports.query.get_or_404(import_id) +def set_import_field_mapping(scope, imprt): if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.dataset.active: @@ -296,10 +286,9 @@ def set_import_field_mapping(scope, import_id): return jsonify(imprt.as_dict()) -@blueprint.route("/imports//load", methods=["POST"]) +@blueprint.route("//imports//load", methods=["POST"]) @permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def load_import(scope, import_id): - imprt = TImports.query.get_or_404(import_id) +def load_import(scope, imprt): if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.dataset.active: @@ -319,15 +308,14 @@ def load_import(scope, import_id): return jsonify(imprt.as_dict()) -@blueprint.route("/imports//columns", methods=["GET"]) +@blueprint.route("//imports//columns", methods=["GET"]) @permissions.check_cruved_scope("R", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def get_import_columns_name(scope, import_id): +def get_import_columns_name(scope, imprt): """ .. :quickref: Import; Return all the columns of the file of an import """ - imprt = TImports.query.get_or_404(import_id) if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.columns: @@ -335,15 +323,14 @@ def get_import_columns_name(scope, import_id): return jsonify(imprt.columns) -@blueprint.route("/imports//values", methods=["GET"]) +@blueprint.route("//imports//values", methods=["GET"]) @permissions.check_cruved_scope("R", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def get_import_values(scope, import_id): +def get_import_values(scope, imprt): """ .. :quickref: Import; Return all values present in imported file for nomenclated fields """ - imprt = TImports.query.get_or_404(import_id) # check that the user has read permission to this particular import instance: if not imprt.has_instance_permission(scope): raise Forbidden @@ -394,10 +381,9 @@ def get_import_values(scope, import_id): return jsonify(response) -@blueprint.route("/imports//contentmapping", methods=["POST"]) +@blueprint.route("//imports//contentmapping", methods=["POST"]) @permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def set_import_content_mapping(scope, import_id): - imprt = TImports.query.get_or_404(import_id) +def set_import_content_mapping(scope, imprt): if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.dataset.active: @@ -412,13 +398,12 @@ def set_import_content_mapping(scope, import_id): return jsonify(imprt.as_dict()) -@blueprint.route("/imports//prepare", methods=["POST"]) +@blueprint.route("//imports//prepare", methods=["POST"]) @permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def prepare_import(scope, import_id): +def prepare_import(scope, imprt): """ Prepare data to be imported: apply all checks and transformations. """ - imprt = TImports.query.get_or_404(import_id) if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.dataset.active: @@ -441,10 +426,9 @@ def prepare_import(scope, import_id): return jsonify(imprt.as_dict()) -@blueprint.route("/imports//preview_valid_data", methods=["GET"]) +@blueprint.route("//imports//preview_valid_data", methods=["GET"]) @permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def preview_valid_data(scope, import_id): - imprt = TImports.query.get_or_404(import_id) +def preview_valid_data(scope, imprt): if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.processed: @@ -484,24 +468,22 @@ def preview_valid_data(scope, import_id): ) -@blueprint.route("/imports//errors", methods=["GET"]) +@blueprint.route("//imports//errors", methods=["GET"]) @permissions.check_cruved_scope("R", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def get_import_errors(scope, import_id): +def get_import_errors(scope, imprt): """ .. :quickref: Import; Get errors of an import. Get errors of an import. """ - imprt = TImports.query.options(joinedload("errors")).get_or_404(import_id) if not imprt.has_instance_permission(scope): raise Forbidden return jsonify([error.as_dict(fields=["type"]) for error in imprt.errors]) -@blueprint.route("/imports//source_file", methods=["GET"]) +@blueprint.route("//imports//source_file", methods=["GET"]) @permissions.check_cruved_scope("R", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def get_import_source_file(scope, import_id): - imprt = TImports.query.options(undefer("source_file")).get_or_404(import_id) +def get_import_source_file(scope, imprt): if not imprt.has_instance_permission(scope): raise Forbidden if imprt.source_file is None: @@ -514,15 +496,14 @@ def get_import_source_file(scope, import_id): ) -@blueprint.route("/imports//invalid_rows", methods=["GET"]) +@blueprint.route("//imports//invalid_rows", methods=["GET"]) @permissions.check_cruved_scope("R", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def get_import_invalid_rows_as_csv(scope, import_id): +def get_import_invalid_rows_as_csv(scope, imprt): """ .. :quickref: Import; Get invalid rows of an import as CSV. Export invalid data in CSV. """ - imprt = TImports.query.get_or_404(import_id) if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.processed: @@ -565,15 +546,14 @@ def generate_invalid_rows_csv(): return response -@blueprint.route("/imports//import", methods=["POST"]) +@blueprint.route("//imports//import", methods=["POST"]) @permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def import_valid_data(scope, import_id): +def import_valid_data(scope, imprt): """ .. :quickref: Import; Import the valid data. Import valid data in GeoNature synthese. """ - imprt = TImports.query.get_or_404(import_id) if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.dataset.active: @@ -595,15 +575,14 @@ def import_valid_data(scope, import_id): return jsonify(imprt.as_dict()) -@blueprint.route("/imports//", methods=["DELETE"]) +@blueprint.route("//imports//", methods=["DELETE"]) @permissions.check_cruved_scope("D", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def delete_import(scope, import_id): +def delete_import(scope, imprt): """ .. :quickref: Import; Delete an import. Delete an import. """ - imprt = TImports.query.get_or_404(import_id) if not imprt.has_instance_permission(scope): raise Forbidden if not imprt.dataset.active: @@ -618,18 +597,12 @@ def delete_import(scope, import_id): return jsonify() -@blueprint.route("/export_pdf/", methods=["POST"]) +@blueprint.route("//export_pdf/", methods=["POST"]) @permissions.check_cruved_scope("R", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def export_pdf(scope, import_id): +def export_pdf(scope, imprt): """ Downloads the report in pdf format """ - imprt = TImports.query.options( - Load(TImports).raiseload("*"), - joinedload("authors"), - joinedload("dataset"), - joinedload("errors"), - ).get_or_404(import_id) if not imprt.has_instance_permission(scope): raise Forbidden ctx = imprt.as_dict(fields=["errors", "errors.type", "dataset.dataset_name"]) diff --git a/backend/gn_module_import/routes/mappings.py b/backend/gn_module_import/routes/mappings.py index 86135057..4fd07884 100644 --- a/backend/gn_module_import/routes/mappings.py +++ b/backend/gn_module_import/routes/mappings.py @@ -1,16 +1,11 @@ -from itertools import groupby - from flask import request, jsonify, current_app, g from werkzeug.exceptions import Forbidden, Conflict, BadRequest, NotFound -from sqlalchemy.orm import joinedload from sqlalchemy.orm.attributes import flag_modified from geonature.utils.env import db from geonature.core.gn_permissions import decorators as permissions from gn_module_import.models import ( - BibFields, - BibThemes, MappingTemplate, FieldMapping, ContentMapping, @@ -27,14 +22,16 @@ def check_mapping_type(endpoint, values): values["mappingtype"] = values["mappingtype"].upper() if current_app.url_map.is_endpoint_expecting(endpoint, "id_mapping"): mapping = MappingTemplate.query.get_or_404(values.pop("id_mapping")) - if mapping.type != values["mappingtype"]: + if mapping.destination != values.pop("destination"): + raise NotFound + if mapping.type != values.pop("mappingtype"): raise NotFound values["mapping"] = mapping -@blueprint.route("/mappings/", methods=["GET"]) +@blueprint.route("//mappings/", methods=["GET"]) @permissions.check_cruved_scope("R", get_scope=True, module_code="IMPORT", object_code="MAPPING") -def list_mappings(mappingtype, scope): +def list_mappings(destination, mappingtype, scope): """ .. :quickref: Import; Return all active named mappings. @@ -44,16 +41,17 @@ def list_mappings(mappingtype, scope): :type type: str """ mappings = ( - MappingTemplate.query.filter(MappingTemplate.type == mappingtype) - .filter(MappingTemplate.active == True) + MappingTemplate.query.filter(MappingTemplate.destination == destination) + .filter(MappingTemplate.type == mappingtype) + .filter(MappingTemplate.active == True) # noqa: E712 .filter_by_scope(scope) ) return jsonify([mapping.as_dict() for mapping in mappings]) -@blueprint.route("/mappings//", methods=["GET"]) +@blueprint.route("//mappings//", methods=["GET"]) @permissions.check_cruved_scope("R", get_scope=True, module_code="IMPORT", object_code="MAPPING") -def get_mapping(mappingtype, mapping, scope): +def get_mapping(mapping, scope): """ .. :quickref: Import; Return a mapping. @@ -66,9 +64,9 @@ def get_mapping(mappingtype, mapping, scope): return jsonify(mapping.as_dict()) -@blueprint.route("/mappings/", methods=["POST"]) +@blueprint.route("//mappings/", methods=["POST"]) @permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="MAPPING") -def add_mapping(mappingtype, scope): +def add_mapping(destination, mappingtype, scope): """ .. :quickref: Import; Add a mapping. """ @@ -78,7 +76,9 @@ def add_mapping(mappingtype, scope): # check if name already exists if db.session.query( - MappingTemplate.query.filter_by(type=mappingtype, label=label).exists() + MappingTemplate.query.filter_by( + destination=destination, type=mappingtype, label=label + ).exists() ).scalar(): raise Conflict(description="Un mapping de ce type portant ce nom existe déjà") @@ -89,16 +89,20 @@ def add_mapping(mappingtype, scope): raise BadRequest(*e.args) mapping = MappingClass( - type=mappingtype, label=label, owners=[g.current_user], values=request.json + destination=destination, + type=mappingtype, + label=label, + owners=[g.current_user], + values=request.json, ) db.session.add(mapping) db.session.commit() return jsonify(mapping.as_dict()) -@blueprint.route("/mappings//", methods=["POST"]) +@blueprint.route("//mappings//", methods=["POST"]) @permissions.check_cruved_scope("U", get_scope=True, module_code="IMPORT", object_code="MAPPING") -def update_mapping(mappingtype, mapping, scope): +def update_mapping(mapping, scope): """ .. :quickref: Import; Update a mapping (label and/or content). """ @@ -109,7 +113,7 @@ def update_mapping(mappingtype, mapping, scope): if label: # check if name already exists if db.session.query( - MappingTemplate.query.filter_by(type=mappingtype, label=label).exists() + MappingTemplate.query.filter_by(type=mapping.type, label=label).exists() ).scalar(): raise Conflict(description="Un mapping de ce type portant ce nom existe déjà") mapping.label = label @@ -118,9 +122,9 @@ def update_mapping(mappingtype, mapping, scope): mapping.validate_values(request.json) except ValueError as e: raise BadRequest(*e.args) - if mappingtype == "FIELD": + if mapping.type == "FIELD": mapping.values.update(request.json) - elif mappingtype == "CONTENT": + elif mapping.type == "CONTENT": for key, value in request.json.items(): if key not in mapping.values: mapping.values[key] = value @@ -133,9 +137,9 @@ def update_mapping(mappingtype, mapping, scope): return jsonify(mapping.as_dict()) -@blueprint.route("/mappings//", methods=["DELETE"]) +@blueprint.route("//mappings//", methods=["DELETE"]) @permissions.check_cruved_scope("D", get_scope=True, module_code="IMPORT", object_code="MAPPING") -def delete_mapping(mappingtype, mapping, scope): +def delete_mapping(mapping, scope): """ .. :quickref: Import; Delete a mapping. """ @@ -144,57 +148,3 @@ def delete_mapping(mappingtype, mapping, scope): db.session.delete(mapping) db.session.commit() return "", 204 - - -@blueprint.route("/synthesis/fields", methods=["GET"]) -@permissions.check_cruved_scope("C", get_scope=True, module_code="IMPORT", object_code="IMPORT") -def get_synthesis_fields(scope): - """ - .. :quickref: Import; Get synthesis fields. - - Get all synthesis fields - Use in field mapping steps - You can find a jsonschema of the returned data in the associated test. - """ - # TODO use selectinload - fields = ( - BibFields.query.filter_by(display=True) - .options(joinedload(BibFields.theme)) - .join(BibThemes) - .order_by(BibThemes.order_theme, BibFields.order_field) - .all() - ) - data = [] - for id_theme, fields in groupby(fields, lambda field: field.id_theme): - fields = list(fields) - theme = fields[0].theme - data.append( - { - "theme": theme.as_dict( - fields=[ - "id_theme", - "name_theme", - "fr_label_theme", - "eng_label_theme", - "desc_theme", - ], - ), - "fields": [ - field.as_dict( - fields=[ - "id_field", - "name_field", - "fr_label", - "eng_label", - "desc_field", - "mandatory", - "autogenerated", - "comment", - "multi", - ], - ) - for field in fields - ], - } - ) - return jsonify(data) diff --git a/backend/gn_module_import/tests/conftest.py b/backend/gn_module_import/tests/conftest.py index 0f3b5091..09ed7f53 100644 --- a/backend/gn_module_import/tests/conftest.py +++ b/backend/gn_module_import/tests/conftest.py @@ -1,2 +1,3 @@ from geonature.tests.fixtures import * from geonature.tests.fixtures import app, _session, users +from .fixtures import * diff --git a/backend/gn_module_import/tests/fixtures.py b/backend/gn_module_import/tests/fixtures.py new file mode 100644 index 00000000..d0e005fc --- /dev/null +++ b/backend/gn_module_import/tests/fixtures.py @@ -0,0 +1,23 @@ +import pytest + +from geonature.core.gn_commons.models import TModules + +from gn_module_import.models import Destination + + +@pytest.fixture(scope="session") +def synthese_destination(): + return Destination.query.filter( + Destination.module.has(TModules.module_code == "SYNTHESE") + ).one() + + +@pytest.fixture(scope="session") +def default_synthese_destination(app, synthese_destination): + """ + This fixture set "synthese" as default destination when not specified in call to url_for. + """ + @app.url_defaults + def set_synthese_destination(endpoint, values): + if app.url_map.is_endpoint_expecting(endpoint, "destination") and "destination" not in values: + values["destination"] = "synthese" diff --git a/backend/gn_module_import/tests/jsonschema_definitions.py b/backend/gn_module_import/tests/jsonschema_definitions.py index cae0ba47..59c93d3f 100644 --- a/backend/gn_module_import/tests/jsonschema_definitions.py +++ b/backend/gn_module_import/tests/jsonschema_definitions.py @@ -1,5 +1,17 @@ jsonschema_definitions = { - "synthesis_field": { + "entity": { + "type": "object", + "properties": { + "label": { + "type": "string", + }, + }, + "required": [ + "label", + ], + "additionalProperties": False, + }, + "fields": { "type": "object", "properties": { "id_field": { @@ -66,7 +78,7 @@ ], "additionalProperties": False, }, - "synthesis_theme": { + "theme": { "type": "object", "properties": { "id_theme": { @@ -270,6 +282,7 @@ "type": "object", "properties": { "id": {"type": "integer"}, + "id_destination": {"type": "integer"}, "label": {"type": "string"}, "type": { "type": "string", diff --git a/backend/gn_module_import/tests/test_fields.py b/backend/gn_module_import/tests/test_fields.py new file mode 100644 index 00000000..853d461a --- /dev/null +++ b/backend/gn_module_import/tests/test_fields.py @@ -0,0 +1,79 @@ +import pytest +from flask import url_for +from werkzeug.exceptions import Unauthorized +from jsonschema import validate as validate_json + +from geonature.tests.utils import set_logged_user_cookie +from geonature.core.gn_commons.models import TModules + +from gn_module_import.models import ( + Destination, + BibThemes, +) + +from .jsonschema_definitions import jsonschema_definitions + + +@pytest.fixture() +def dest(): + return Destination.query.filter( + Destination.module.has(TModules.module_code == "SYNTHESE") + ).one() + + +@pytest.mark.usefixtures("client_class", "temporary_transaction", "default_synthese_destination") +class TestFields: + def test_fields(self, users): + assert ( + self.client.get( + url_for("import.get_fields") + ).status_code + == Unauthorized.code + ) + set_logged_user_cookie(self.client, users["admin_user"]) + r = self.client.get(url_for("import.get_fields")) + assert r.status_code == 200 + data = r.get_json() + themes_count = BibThemes.query.count() + schema = { + "definitions": jsonschema_definitions, + "type": "array", + "items": { + "type": "object", + "properties": { + "entity": {"$ref": "#/definitions/entity"}, + "themes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "theme": {"$ref": "#/definitions/theme"}, + "fields": { + "type": "array", + "items": {"$ref": "#/definitions/fields"}, + "uniqueItems": True, + "minItems": 1, + }, + }, + "required": [ + "theme", + "fields", + ], + "additionalProperties": False, + }, + "minItems": themes_count, + "maxItems": themes_count, + } + } + } + } + validate_json(data, schema) + + def test_get_nomenclatures(self): + resp = self.client.get(url_for("import.get_nomenclatures")) + + assert resp.status_code == 200 + assert all( + set(nomenclature.keys()) == {"nomenclature_type", "nomenclatures"} + for nomenclature in resp.json.values() + ) diff --git a/backend/gn_module_import/tests/test_imports.py b/backend/gn_module_import/tests/test_imports_synthese.py similarity index 99% rename from backend/gn_module_import/tests/test_imports.py rename to backend/gn_module_import/tests/test_imports_synthese.py index d32f193c..b5ac62dd 100644 --- a/backend/gn_module_import/tests/test_imports.py +++ b/backend/gn_module_import/tests/test_imports_synthese.py @@ -109,10 +109,10 @@ def sample_area(): @pytest.fixture(scope="function") -def imports(users): +def imports(synthese_destination, users): def create_import(authors=[]): with db.session.begin_nested(): - imprt = TImports(authors=authors) + imprt = TImports(destination=synthese_destination, authors=authors) db.session.add(imprt) return imprt @@ -168,9 +168,10 @@ def import_dataset(datasets, import_file_name): @pytest.fixture() -def new_import(users, import_dataset): +def new_import(synthese_destination, users, import_dataset): with db.session.begin_nested(): imprt = TImports( + destination=synthese_destination, authors=[users["user"]], id_dataset=import_dataset.id_dataset, ) @@ -296,9 +297,9 @@ def change_id_list_conf(monkeypatch, sample_taxhub_list): ) -@pytest.mark.usefixtures("client_class", "temporary_transaction", "celery_eager") -class TestImports: - def test_import_permissions(self, g_permissions): +@pytest.mark.usefixtures("client_class", "temporary_transaction", "celery_eager", "default_synthese_destination") +class TestImportsSynthese: + def test_import_permissions(self, g_permissions, synthese_destination): with db.session.begin_nested(): organisme = Organisme(nom_organisme="test_import") db.session.add(organisme) @@ -309,7 +310,7 @@ def test_import_permissions(self, g_permissions): other_user = User(groupe=False, organisme=organisme) db.session.add(other_user) user.groups.append(group) - imprt = TImports() + imprt = TImports(destination=synthese_destination) db.session.add(imprt) get_scopes_by_action = partial( diff --git a/backend/gn_module_import/tests/test_mappings.py b/backend/gn_module_import/tests/test_mappings.py index 79dd1355..6f2f02e3 100644 --- a/backend/gn_module_import/tests/test_mappings.py +++ b/backend/gn_module_import/tests/test_mappings.py @@ -1,9 +1,7 @@ -from pathlib import Path from copy import deepcopy import pytest -from flask import testing, url_for -from werkzeug.datastructures import Headers +from flask import url_for from werkzeug.exceptions import Unauthorized, Forbidden, BadRequest, Conflict, NotFound from jsonschema import validate as validate_json from sqlalchemy import func @@ -11,43 +9,21 @@ from geonature.utils.env import db from geonature.tests.utils import set_logged_user_cookie -from geonature.core.gn_permissions.models import ( - Permission, -) -from geonature.core.gn_commons.models import TModules -from geonature.core.gn_meta.models import ( - TNomenclatures, - TAcquisitionFramework, - TDatasets, -) from pypnnomenclature.models import BibNomenclaturesTypes -from pypnusershub.db.models import ( - User, - Organisme, - Application, - Profils as Profil, - UserApplicationRight, -) from gn_module_import.models import ( MappingTemplate, FieldMapping, ContentMapping, - BibThemes, BibFields, - ImportUserError, - ImportUserErrorType, ) from .jsonschema_definitions import jsonschema_definitions -tests_path = Path(__file__).parent - - @pytest.fixture() -def mappings(users): +def mappings(synthese_destination, users): mappings = {} fieldmapping_values = { field.name_field: True @@ -74,34 +50,44 @@ def mappings(users): } with db.session.begin_nested(): mappings["content_public"] = ContentMapping( + destination=synthese_destination, label="Content Mapping", active=True, public=True, values=contentmapping_values, ) mappings["field_public"] = FieldMapping( + destination=synthese_destination, label="Public Field Mapping", active=True, public=True, values=fieldmapping_values, ) - mappings["field"] = FieldMapping(label="Private Field Mapping", active=True, public=False) + mappings["field"] = FieldMapping( + destination=synthese_destination, label="Private Field Mapping", active=True, public=False + ) mappings["field_public_disabled"] = FieldMapping( - label="Disabled Public Field Mapping", active=False, public=True + destination=synthese_destination, + label="Disabled Public Field Mapping", + active=False, + public=True, ) mappings["self"] = FieldMapping( + destination=synthese_destination, label="Self’s Mapping", active=True, public=False, owners=[users["self_user"]], ) mappings["stranger"] = FieldMapping( + destination=synthese_destination, label="Stranger’s Mapping", active=True, public=False, owners=[users["stranger_user"]], ) mappings["associate"] = FieldMapping( + destination=synthese_destination, label="Associate’s Mapping", active=True, public=False, @@ -111,7 +97,7 @@ def mappings(users): return mappings -@pytest.mark.usefixtures("client_class", "temporary_transaction") +@pytest.mark.usefixtures("client_class", "temporary_transaction", "default_synthese_destination") class TestMappings: def test_list_mappings(self, users, mappings): set_logged_user_cookie(self.client, users["noright_user"]) @@ -134,13 +120,14 @@ def test_list_mappings(self, users, mappings): ) def test_get_mapping(self, users, mappings): - get_mapping = lambda mapping: self.client.get( - url_for( - "import.get_mapping", - mappingtype=mapping.type.lower(), - id_mapping=mapping.id, + def get_mapping(mapping): + return self.client.get( + url_for( + "import.get_mapping", + mappingtype=mapping.type.lower(), + id_mapping=mapping.id, + ) ) - ) assert get_mapping(mappings["field_public"]).status_code == Unauthorized.code @@ -170,7 +157,11 @@ def test_get_mapping(self, users, mappings): unexisting_id = db.session.query(func.max(MappingTemplate.id)).scalar() + 1 r = self.client.get( - url_for("import.get_mapping", mappingtype="field", id_mapping=unexisting_id) + url_for( + "import.get_mapping", + mappingtype="field", + id_mapping=unexisting_id, + ) ) assert r.status_code == NotFound.code @@ -256,7 +247,11 @@ def test_add_field_mapping(self, users, mappings): assert self.client.post(url, data=fieldmapping).status_code == BadRequest.code # label already exist - url = url_for("import.add_mapping", mappingtype="field", label=mappings["field"].label) + url = url_for( + "import.add_mapping", + mappingtype="field", + label=mappings["field"].label, + ) assert self.client.post(url, data=fieldmapping).status_code == Conflict.code # label may be reused between field and content @@ -286,7 +281,11 @@ def test_add_field_mapping(self, users, mappings): assert mapping.owners == [users["user"]] def test_add_content_mapping(self, users, mappings): - url = url_for("import.add_mapping", mappingtype="content", label="test content mapping") + url = url_for( + "import.add_mapping", + mappingtype="content", + label="test content mapping", + ) set_logged_user_cookie(self.client, users["user"]) contentmapping = { @@ -313,11 +312,6 @@ def test_add_content_mapping(self, users, mappings): def test_update_mapping_label(self, users, mappings): mapping = mappings["associate"] - url = url_for( - "import.update_mapping", - mappingtype=mapping.type.lower(), - id_mapping=mapping.id, - ) r = self.client.post( url_for( @@ -381,7 +375,11 @@ def test_update_field_mapping_values(self, users, mappings): fieldvalues_should = deepcopy(fieldvalues_update) del fieldvalues_update["validator"] # should not removed from mapping! r = self.client.post( - url_for("import.update_mapping", mappingtype=fm.type.lower(), id_mapping=fm.id), + url_for( + "import.update_mapping", + mappingtype=fm.type.lower(), + id_mapping=fm.id, + ), data=fieldvalues_update, ) assert r.status_code == 200 @@ -389,7 +387,11 @@ def test_update_field_mapping_values(self, users, mappings): fieldvalues_update = deepcopy(fm.values) fieldvalues_update["unexisting"] = "unexisting" r = self.client.post( - url_for("import.update_mapping", mappingtype=fm.type.lower(), id_mapping=fm.id), + url_for( + "import.update_mapping", + mappingtype=fm.type.lower(), + id_mapping=fm.id, + ), data=fieldvalues_update, ) assert r.status_code == BadRequest.code @@ -405,7 +407,11 @@ def test_update_content_mapping_values(self, users, mappings): contentvalues_should = deepcopy(contentvalues_update) del contentvalues_update["NAT_OBJ_GEO"]["St"] # should not be removed! r = self.client.post( - url_for("import.update_mapping", mappingtype=cm.type.lower(), id_mapping=cm.id), + url_for( + "import.update_mapping", + mappingtype=cm.type.lower(), + id_mapping=cm.id, + ), data=contentvalues_update, ) assert r.status_code == 200 @@ -413,7 +419,11 @@ def test_update_content_mapping_values(self, users, mappings): contentvalues_update = deepcopy(cm.values) contentvalues_update["NAT_OBJ_GEO"] = "invalid" r = self.client.post( - url_for("import.update_mapping", mappingtype=cm.type.lower(), id_mapping=cm.id), + url_for( + "import.update_mapping", + mappingtype=cm.type.lower(), + id_mapping=cm.id, + ), data=contentvalues_update, ) assert r.status_code == BadRequest.code @@ -444,7 +454,11 @@ def test_delete_mapping(self, users, mappings): set_logged_user_cookie(self.client, users["user"]) r = self.client.delete( - url_for("import.delete_mapping", mappingtype="content", id_mapping=mapping.id) + url_for( + "import.delete_mapping", + mappingtype="content", + id_mapping=mapping.id, + ) ) assert r.status_code == NotFound.code assert MappingTemplate.query.get(mapping.id) is not None @@ -458,38 +472,3 @@ def test_delete_mapping(self, users, mappings): ) assert r.status_code == 204 assert MappingTemplate.query.get(mapping.id) is None - - def test_synthesis_fields(self, users): - assert ( - self.client.get(url_for("import.get_synthesis_fields")).status_code - == Unauthorized.code - ) - set_logged_user_cookie(self.client, users["admin_user"]) - r = self.client.get(url_for("import.get_synthesis_fields")) - assert r.status_code == 200 - data = r.get_json() - themes_count = BibThemes.query.count() - schema = { - "definitions": jsonschema_definitions, - "type": "array", - "items": { - "type": "object", - "properties": { - "theme": {"$ref": "#/definitions/synthesis_theme"}, - "fields": { - "type": "array", - "items": {"$ref": "#/definitions/synthesis_field"}, - "uniqueItems": True, - "minItems": 1, - }, - }, - "required": [ - "theme", - "fields", - ], - "additionalProperties": False, - }, - "minItems": themes_count, - "maxItems": themes_count, - } - validate_json(data, schema) diff --git a/frontend/app/components/import_list/import-list.component.ts b/frontend/app/components/import_list/import-list.component.ts index fabab0c8..ca509759 100644 --- a/frontend/app/components/import_list/import-list.component.ts +++ b/frontend/app/components/import_list/import-list.component.ts @@ -87,11 +87,6 @@ export class ImportListComponent implements OnInit { this.limit = res["limit"] this.offset = res["offset"] }, - error => { - if (error.status === 404) { - this._commonService.regularToaster("warning", "Aucun import trouvé"); - } - } ); } private getImportsStatus() { diff --git a/frontend/app/components/import_process/fields-mapping-step/fields-mapping-step.component.html b/frontend/app/components/import_process/fields-mapping-step/fields-mapping-step.component.html index 4b8affcb..8254b129 100644 --- a/frontend/app/components/import_process/fields-mapping-step/fields-mapping-step.component.html +++ b/frontend/app/components/import_process/fields-mapping-step/fields-mapping-step.component.html @@ -168,7 +168,9 @@
Correspondance des champs avec le modèle
-
+
+

{{ entitythemes.entity.label }}

+
{{themefields.theme.fr_label_theme}} @@ -245,6 +247,7 @@
Correspondance des champs avec le modèle
+

diff --git a/frontend/app/components/import_process/fields-mapping-step/fields-mapping-step.component.ts b/frontend/app/components/import_process/fields-mapping-step/fields-mapping-step.component.ts index b528554a..bcfadec4 100644 --- a/frontend/app/components/import_process/fields-mapping-step/fields-mapping-step.component.ts +++ b/frontend/app/components/import_process/fields-mapping-step/fields-mapping-step.component.ts @@ -17,7 +17,7 @@ import { CruvedStoreService } from "@geonature_common/service/cruved-store.servi import { DataService } from "../../../services/data.service"; import { FieldMappingService } from "../../../services/mappings/field-mapping.service"; -import { Import, SynthesisThemeFields } from "../../../models/import.model"; +import { Import, EntitiesThemesFields } from "../../../models/import.model"; import { ImportProcessService } from "../import-process.service"; import {ContentMapping, FieldMapping, FieldMappingValues} from "../../../models/mapping.model"; import { Step } from "../../../models/enums.model"; @@ -36,7 +36,7 @@ export class FieldsMappingStepComponent implements OnInit { public spinner: boolean = false; public userFieldMappings: Array; // all field mapping accessible by the users - public targetFields: Array; // list of target fields, i.e. fields of synthesis, ordered by theme + public targetFields: Array; // list of target fields, i.e. fields, ordered by theme, grouped by entities public mappedTargetFields: Set; public unmappedTargetFields: Set; @@ -99,10 +99,12 @@ export class FieldsMappingStepComponent implements OnInit { this.syntheseForm.updateValueAndValidity(); this.sourceFields = sourceFields; - for (let fieldsTheme of this.targetFields) { - for (let field of fieldsTheme.fields) { - if (field.autogenerated) { - this.autogeneratedFields.push(field.name_field); + for (let entity of this.targetFields) { + for (let theme of entity.themes) { + for (let field of theme.fields) { + if (field.autogenerated) { + this.autogeneratedFields.push(field.name_field); + } } } } @@ -136,22 +138,25 @@ export class FieldsMappingStepComponent implements OnInit { } populateSyntheseForm() { let validators: Array; - for (let themefields of this.targetFields) { - for (let field of themefields.fields) { - if (!field.autogenerated) { // autogenerated = checkbox - this.unmappedTargetFields.add(field.name_field); - } - if (field.mandatory) { - validators = [Validators.required]; - } else { - validators = []; + // TODO: use the same form control for fields shared between two entities + for (let entity of this.targetFields) { + for (let theme of entity.themes) { + for (let field of theme.fields) { + if (!field.autogenerated) { // autogenerated = checkbox + this.unmappedTargetFields.add(field.name_field); + } + if (field.mandatory) { + validators = [Validators.required]; + } else { + validators = []; + } + let control = new FormControl(null, validators); + control.valueChanges + .subscribe(value => { + this.onFieldMappingChange(field.name_field, value); + }); + this.syntheseForm.addControl(field.name_field, control); } - let control = new FormControl(null, validators); - control.valueChanges - .subscribe(value => { - this.onFieldMappingChange(field.name_field, value); - }); - this.syntheseForm.addControl(field.name_field, control); } } } @@ -308,18 +313,20 @@ export class FieldsMappingStepComponent implements OnInit { this.mappedSourceFields.clear(); this.unmappedSourceFields = new Set(this.sourceFields); - for (let themefields of this.targetFields) { - for (let targetField of themefields.fields) { - let sourceField = this.syntheseForm.get(targetField.name_field).value; - if (sourceField != null) { - if (Array.isArray(sourceField)) { - sourceField.forEach(sf => { - this.unmappedSourceFields.delete(sf); - this.mappedSourceFields.add(sf); - }) - } else { - this.unmappedSourceFields.delete(sourceField); - this.mappedSourceFields.add(sourceField); + for (let entity of this.targetFields) { + for (let theme of entity.themes) { + for (let targetField of theme.fields) { + let sourceField = this.syntheseForm.get(targetField.name_field).value; + if (sourceField != null) { + if (Array.isArray(sourceField)) { + sourceField.forEach(sf => { + this.unmappedSourceFields.delete(sf); + this.mappedSourceFields.add(sf); + }) + } else { + this.unmappedSourceFields.delete(sourceField); + this.mappedSourceFields.add(sourceField); + } } } } diff --git a/frontend/app/components/modal_dataset/import-modal-dataset.component.html b/frontend/app/components/modal_dataset/import-modal-dataset.component.html index fbb70a8c..bf610905 100644 --- a/frontend/app/components/modal_dataset/import-modal-dataset.component.html +++ b/frontend/app/components/modal_dataset/import-modal-dataset.component.html @@ -18,8 +18,8 @@
Sélection du jeu de données
Sélection du jeu de données \ No newline at end of file + diff --git a/frontend/app/models/import.model.ts b/frontend/app/models/import.model.ts index 8cc019dd..0bd0d820 100644 --- a/frontend/app/models/import.model.ts +++ b/frontend/app/models/import.model.ts @@ -99,7 +99,11 @@ export interface ImportValues { } } -interface SynthesisTheme { +interface Entity { + label: string, +} + +interface Theme { id_theme: number, name_theme: string, fr_label_theme: string, @@ -107,7 +111,7 @@ interface SynthesisTheme { desc_theme: string, } -interface SynthesisField { +interface Field { id_field: number, name_field: string, fr_label: string, @@ -118,9 +122,14 @@ interface SynthesisField { comment: string, } -export interface SynthesisThemeFields { - theme: SynthesisTheme, - fields: [SynthesisField], +interface ThemesFields { + theme: Theme, + fields: [Field], +} + +export interface EntitiesThemesFields { + entity: Entity, + themes: [ThemesFields], } export interface TaxaDistribution { @@ -131,4 +140,4 @@ export interface TaxaDistribution { // minimal dataset model export interface Dataset { dataset_name: string -} \ No newline at end of file +} diff --git a/frontend/app/services/data.service.ts b/frontend/app/services/data.service.ts index 990fdd77..3cbff5fd 100644 --- a/frontend/app/services/data.service.ts +++ b/frontend/app/services/data.service.ts @@ -1,7 +1,7 @@ import { Injectable } from "@angular/core"; import { Observable } from "rxjs"; import { HttpClient, HttpParams } from "@angular/common/http"; -import { Dataset, Import, ImportError, ImportValues, SynthesisThemeFields, TaxaDistribution } from "../models/import.model"; +import { Dataset, Import, ImportError, ImportValues, EntitiesThemesFields, TaxaDistribution } from "../models/import.model"; import { FieldMapping, FieldMappingValues, ContentMapping, ContentMappingValues } from "../models/mapping.model"; import { ConfigService } from '@geonature/services/config.service'; @@ -11,7 +11,7 @@ export class DataService { private urlApi = null; constructor(private _http: HttpClient, public config: ConfigService) { - this.urlApi = `${this.config.API_ENDPOINT}/${this.config.IMPORT.MODULE_URL}`; + this.urlApi = `${this.config.API_ENDPOINT}/import/synthese`; } getNomenclatures(): Observable { @@ -141,8 +141,8 @@ export class DataService { return this._http.get(`${this.urlApi}/imports/${idImport}/values`); } - getBibFields(): Observable> { - return this._http.get>(`${this.urlApi}/synthesis/fields`); + getBibFields(): Observable> { + return this._http.get>(`${this.urlApi}/fields`); } setImportFieldMapping(idImport: number, values: FieldMappingValues): Observable {