From 3d4f69399cc5f594003ffb4d1401886a6d9ebff2 Mon Sep 17 00:00:00 2001 From: Etienne Delclaux Date: Thu, 12 Dec 2024 12:33:50 +0100 Subject: [PATCH] feat: remove dataset selection from upload phase and set it as other fieldmapping --- .../core/imports/checks/dataframe/core.py | 4 +- .../geonature/core/imports/config_schema.py | 6 --- backend/geonature/core/imports/models.py | 5 --- .../geonature/core/imports/routes/imports.py | 42 +------------------ .../templates/import_template_pdf.html | 11 ++--- .../delete-modal/delete-modal.component.html | 2 +- .../import_errors.component.html | 4 -- .../import_list/import-list.component.html | 12 +----- .../import_list/import-list.component.ts | 6 --- .../header-stepper.component.html | 7 ---- .../header-stepper.component.scss | 5 --- .../header-stepper.component.ts | 18 ++++---- .../upload-file-step.component.html | 15 ------- .../upload-file-step.component.ts | 18 +++----- .../import_report.component.html | 9 ---- .../import_report/import_report.component.ts | 10 ----- .../import-modal-destination.component.html | 2 +- .../import-modal-destination.component.scss | 2 +- .../modules/imports/models/import.model.ts | 11 ----- .../modules/imports/services/data.service.ts | 10 +---- 20 files changed, 27 insertions(+), 172 deletions(-) diff --git a/backend/geonature/core/imports/checks/dataframe/core.py b/backend/geonature/core/imports/checks/dataframe/core.py index f20c1f6b14..29a3338211 100644 --- a/backend/geonature/core/imports/checks/dataframe/core.py +++ b/backend/geonature/core/imports/checks/dataframe/core.py @@ -230,7 +230,7 @@ def check_datasets( uuid = df.loc[has_uuid_mask, uuid_col].unique().tolist() datasets = { - ds.unique_dataset_id.hex: ds + str(ds.unique_dataset_id): ds for ds in TDatasets.query.filter(TDatasets.unique_dataset_id.in_(uuid)) .options(sa.orm.joinedload(TDatasets.nomenclature_data_origin)) .options(sa.orm.raiseload("*")) @@ -256,7 +256,7 @@ def check_datasets( # Warning: we check only permissions of first author, but currently there it only one author per import. authorized_datasets = { - ds.unique_dataset_id.hex: ds + str(ds.unique_dataset_id): ds for ds in db.session.execute( TDatasets.filter_by_creatable( user=imprt.authors[0], module_code=module_code, object_code=object_code diff --git a/backend/geonature/core/imports/config_schema.py b/backend/geonature/core/imports/config_schema.py index 2a77982e3f..5b5164b670 100644 --- a/backend/geonature/core/imports/config_schema.py +++ b/backend/geonature/core/imports/config_schema.py @@ -18,12 +18,6 @@ "show": True, "filter": True, }, - { - "prop": "dataset.dataset_name", - "name": "Jeu de données", - "show": True, - "filter": False, - }, { "prop": "statistics_rows", "name": "Lignes importées", diff --git a/backend/geonature/core/imports/models.py b/backend/geonature/core/imports/models.py index e95ff8ac6e..74ad5d4c64 100644 --- a/backend/geonature/core/imports/models.py +++ b/backend/geonature/core/imports/models.py @@ -29,7 +29,6 @@ from geonature.utils.celery import celery_app from geonature.core.gn_permissions.tools import get_scopes_by_action from geonature.core.gn_commons.models import TModules -from geonature.core.gn_meta.models import TDatasets from pypnnomenclature.models import BibNomenclaturesTypes from pypnusershub.db.models import User @@ -316,8 +315,6 @@ def get_instance_permissions(self, scopes, user=None): @serializable( fields=[ "authors.nom_complet", - "dataset.dataset_name", - "dataset.active", "destination.code", "destination.label", "destination.statistics_labels", @@ -349,7 +346,6 @@ class TImports(InstancePermissionMixin, db.Model): detected_encoding = db.Column(db.Unicode, nullable=True) # import_table = db.Column(db.Unicode, nullable=True) full_file_name = db.Column(db.Unicode, nullable=True) - id_dataset = db.Column(db.Integer, ForeignKey("gn_meta.t_datasets.id_dataset"), nullable=True) date_create_import = db.Column(db.DateTime, default=datetime.now) date_update_import = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now) date_end_import = db.Column(db.DateTime, nullable=True) @@ -369,7 +365,6 @@ class TImports(InstancePermissionMixin, db.Model): ) loaded = db.Column(db.Boolean, nullable=False, default=False) processed = db.Column(db.Boolean, nullable=False, default=False) - dataset = db.relationship(TDatasets, lazy="joined") source_file = deferred(db.Column(db.LargeBinary)) columns = db.Column(ARRAY(db.Unicode)) # keys are target names, values are source names diff --git a/backend/geonature/core/imports/routes/imports.py b/backend/geonature/core/imports/routes/imports.py index 92b1dbb819..63840f373b 100644 --- a/backend/geonature/core/imports/routes/imports.py +++ b/backend/geonature/core/imports/routes/imports.py @@ -23,7 +23,6 @@ from geonature.core.gn_permissions import decorators as permissions from geonature.core.gn_permissions.decorators import login_required from geonature.core.gn_permissions.tools import get_scopes_by_action -from geonature.core.gn_meta.models import TDatasets from pypnnomenclature.models import TNomenclatures @@ -85,11 +84,6 @@ def get_import_list(scope, destination=None): filters = [] if search: filters.append(TImports.full_file_name.ilike(f"%{search}%")) - filters.append( - TImports.dataset.has( - func.lower(TDatasets.dataset_name).contains(func.lower(search)), - ) - ) filters.append( TImports.authors.any( or_( @@ -114,11 +108,9 @@ def get_import_list(scope, destination=None): query = ( select(TImports) .options( - contains_eager(TImports.dataset), contains_eager(TImports.authors), contains_eager(TImports.destination).contains_eager(Destination.module), ) - .join(TImports.dataset, isouter=True) .join(TImports.authors, isouter=True) .join(Destination) .join(TModules) @@ -165,13 +157,10 @@ def upload_file(scope, imprt, destination=None): # destination is set when impr Add an import or update an existing import. :form file: file to import - :form int datasetId: dataset ID to which import data """ if imprt: if not imprt.has_instance_permission(scope, action_code="C"): raise Forbidden - if not imprt.dataset.active: - raise Forbidden("Le jeu de données est fermé.") destination = imprt.destination else: assert destination @@ -187,22 +176,7 @@ def upload_file(scope, imprt, destination=None): # destination is set when impr if size == 0: raise BadRequest(description="Impossible to upload empty files") if imprt is None: - try: - dataset_id = int(request.form["datasetId"]) - except ValueError: - raise BadRequest(description="'datasetId' must be an integer.") - dataset = db.session.get(TDatasets, dataset_id) - if dataset is None: - raise BadRequest(description=f"Dataset '{dataset_id}' does not exist.") - ds_scope = get_scopes_by_action( - module_code=destination.module.module_code, - object_code="ALL", # TODO object_code should be configurable by destination - )["C"] - if not dataset.has_instance_permission(ds_scope): - raise Forbidden(description="Vous n’avez pas les permissions sur ce jeu de données.") - if not dataset.active: - raise Forbidden("Le jeu de données est fermé.") - imprt = TImports(destination=destination, dataset=dataset) + imprt = TImports(destination=destination) imprt.authors.append(author) db.session.add(imprt) else: @@ -226,8 +200,6 @@ def upload_file(scope, imprt, destination=None): # destination is set when impr def decode_file(scope, imprt): if not imprt.has_instance_permission(scope, action_code="C"): raise Forbidden - if not imprt.dataset.active: - raise Forbidden("Le jeu de données est fermé.") if imprt.source_file is None: raise BadRequest(description="A file must be first uploaded.") if "encoding" not in request.json: @@ -292,8 +264,6 @@ def decode_file(scope, imprt): def set_import_field_mapping(scope, imprt): if not imprt.has_instance_permission(scope, action_code="C"): raise Forbidden - if not imprt.dataset.active: - raise Forbidden("Le jeu de données est fermé.") try: FieldMapping.validate_values(request.json) except ValueError as e: @@ -309,8 +279,6 @@ def set_import_field_mapping(scope, imprt): def load_import(scope, imprt): if not imprt.has_instance_permission(scope, action_code="C"): raise Forbidden - if not imprt.dataset.active: - raise Forbidden("Le jeu de données est fermé.") if imprt.source_file is None: raise BadRequest(description="A file must be first uploaded.") if imprt.fieldmapping is None: @@ -404,8 +372,6 @@ def get_import_values(scope, imprt): def set_import_content_mapping(scope, imprt): if not imprt.has_instance_permission(scope, action_code="C"): raise Forbidden - if not imprt.dataset.active: - raise Forbidden("Le jeu de données est fermé.") try: ContentMapping.validate_values(request.json) except ValueError as e: @@ -424,8 +390,6 @@ def prepare_import(scope, imprt): """ if not imprt.has_instance_permission(scope, action_code="C"): raise Forbidden - if not imprt.dataset.active: - raise Forbidden("Le jeu de données est fermé.") # Check preconditions to execute this action if not imprt.loaded: @@ -627,8 +591,6 @@ def import_valid_data(scope, imprt): """ if not imprt.has_instance_permission(scope, action_code="C"): raise Forbidden - if not imprt.dataset.active: - raise Forbidden("Le jeu de données est fermé.") if not imprt.processed: raise Forbidden("L’import n’a pas été préalablement vérifié.") transient_table = imprt.destination.get_transient_table() @@ -662,8 +624,6 @@ def delete_import(scope, imprt): """ if not imprt.has_instance_permission(scope, action_code="C"): raise Forbidden - if not imprt.dataset.active: - raise Forbidden("Le jeu de données est fermé.") ImportUserError.query.filter_by(imprt=imprt).delete() transient_table = imprt.destination.get_transient_table() db.session.execute( diff --git a/backend/geonature/core/imports/templates/import_template_pdf.html b/backend/geonature/core/imports/templates/import_template_pdf.html index 467849f33b..093034fb4b 100644 --- a/backend/geonature/core/imports/templates/import_template_pdf.html +++ b/backend/geonature/core/imports/templates/import_template_pdf.html @@ -47,9 +47,6 @@

{% endfor %} - {% if data.dataset: %} -
  • Jeu de données : {{ data.dataset.dataset_name }}
  • - {% endif %} {% if data.keywords: %}
    @@ -74,7 +71,7 @@

    {% endif %} - + {% if data.date_end_import is not none %}
    @@ -97,7 +94,7 @@

    {% endfor %} - +

    - +
    {% endif %} {% if data.processed %} @@ -137,7 +134,7 @@

    - + {% endif %} diff --git a/frontend/src/app/modules/imports/components/delete-modal/delete-modal.component.html b/frontend/src/app/modules/imports/components/delete-modal/delete-modal.component.html index 8766f11d0d..64289dcdc7 100644 --- a/frontend/src/app/modules/imports/components/delete-modal/delete-modal.component.html +++ b/frontend/src/app/modules/imports/components/delete-modal/delete-modal.component.html @@ -14,7 +14,7 @@

    -
    - -
    - -
    -
    { this.ds.getDestination(params['destination']).subscribe((dest) => { this.destination = dest; }); }); + this.step = this.route.snapshot.data.step; + this.importData = this.importProcessService.getImportData(); + if (this.importData) { + this.fileName = this.importData.full_file_name; + } } isNextStepAvailable() { @@ -93,7 +87,7 @@ export class UploadFileStepComponent implements OnInit { if (this.importData) { return this.ds.updateFile(this.importData.id_import, this.file); } else { - return this.ds.addFile(this.uploadForm.get('dataset').value, this.file); + return this.ds.addFile(this.file); } } onNextStep() { diff --git a/frontend/src/app/modules/imports/components/import_report/import_report.component.html b/frontend/src/app/modules/imports/components/import_report/import_report.component.html index ec71838a7b..b2bc8156aa 100644 --- a/frontend/src/app/modules/imports/components/import_report/import_report.component.html +++ b/frontend/src/app/modules/imports/components/import_report/import_report.component.html @@ -33,15 +33,6 @@
    Description de l'import
    -

    - Jeu de données : - - {{ datasetName }} - -

    Fichier : = []; public importWarnings: Array = []; public nbTotalErrors: number = 0; - public datasetName: string = ''; public rank: string = null; public loadingChart: boolean; public options: any = { @@ -94,7 +93,6 @@ export class ImportReportComponent implements OnInit { this.importData = this.importProcessService.getImportData(); // Load additionnal data if imported data this.loadValidData(this.importData); - this.loadDatasetName(); // Add property to show errors lines. Need to do this to // show line per line... this.loadErrors(); @@ -130,14 +128,6 @@ export class ImportReportComponent implements OnInit { } } - loadDatasetName() { - if (this.importData) { - this._dataService.getDatasetFromId(this.importData.id_dataset).subscribe((data) => { - this.datasetName = data.dataset_name; - }); - } - } - loadErrors() { if (this.importData) { this._dataService.getImportErrors(this.importData.id_import).subscribe((errors) => { diff --git a/frontend/src/app/modules/imports/components/modal_destination/import-modal-destination.component.html b/frontend/src/app/modules/imports/components/modal_destination/import-modal-destination.component.html index ce6540c857..d602fb57e5 100644 --- a/frontend/src/app/modules/imports/components/modal_destination/import-modal-destination.component.html +++ b/frontend/src/app/modules/imports/components/modal_destination/import-modal-destination.component.html @@ -28,7 +28,7 @@

    *ngIf="isUserDestinationError" > warning diff --git a/frontend/src/app/modules/imports/components/modal_destination/import-modal-destination.component.scss b/frontend/src/app/modules/imports/components/modal_destination/import-modal-destination.component.scss index 75342dd005..23feb336d1 100644 --- a/frontend/src/app/modules/imports/components/modal_destination/import-modal-destination.component.scss +++ b/frontend/src/app/modules/imports/components/modal_destination/import-modal-destination.component.scss @@ -1,4 +1,4 @@ -#warningDataset { +#warningDestination { color: red; } diff --git a/frontend/src/app/modules/imports/models/import.model.ts b/frontend/src/app/modules/imports/models/import.model.ts index 0a67f06c72..f2a74ca296 100644 --- a/frontend/src/app/modules/imports/models/import.model.ts +++ b/frontend/src/app/modules/imports/models/import.model.ts @@ -21,10 +21,6 @@ export interface ImportError { show?: boolean; } -export interface Dataset { - dataset_name: string; - active: boolean; -} export interface ImportStatistics { import_count: number; [propName: string]: any; @@ -40,7 +36,6 @@ export interface Import { detected_encoding: string; import_table: string; full_file_name: string; - id_dataset: number; date_create_import: string; date_update_import: string; date_end_import: null | string; @@ -65,7 +60,6 @@ export interface Import { task_progress?: number; task_id?: string; errors?: [ImportError]; - dataset?: Dataset; id_source?: number; id_destination: number; destination?: Destination; @@ -141,11 +135,6 @@ export interface TaxaDistribution { group: string; } -// minimal dataset model -export interface Dataset { - dataset_name: string; -} - export interface ImportPreview { valid_bbox: any; entities: [ diff --git a/frontend/src/app/modules/imports/services/data.service.ts b/frontend/src/app/modules/imports/services/data.service.ts index 8d799f2c7f..03e6dd779a 100644 --- a/frontend/src/app/modules/imports/services/data.service.ts +++ b/frontend/src/app/modules/imports/services/data.service.ts @@ -2,7 +2,6 @@ import { Injectable } from '@angular/core'; import { Observable } from 'rxjs'; import { HttpClient, HttpParams } from '@angular/common/http'; import { - Dataset, Import, ImportError, ImportValues, @@ -55,10 +54,9 @@ export class ImportDataService { return this._http.get(`${this.getUrlApiForADestination()}/imports/${id_import}/`); } - addFile(datasetId: number, file: File): Observable { - let fd = new FormData(); + addFile(file: File): Observable { + const fd = new FormData(); fd.append('file', file, file.name); - fd.append('datasetId', String(datasetId)); const url = `${this.getUrlApiForADestination()}/imports/upload`; return this._http.post(url, fd); } @@ -288,10 +286,6 @@ export class ImportDataService { ); } - getDatasetFromId(datasetId: number) { - return this._http.get(`${this.config.API_ENDPOINT}/meta/dataset/${datasetId}`); - } - getPdf(importId, mapImg, chartImg) { const formData = new FormData(); if (mapImg) {