Merge pull request #1625 from BLSQ/IA-3420-add-progress-message-on-ge…

…opackage-import

IA-3420 Add progress message when importing geopackage

iaso/gpkg/import_gpkg.py

@@ -159,7 +159,9 @@ def import_gpkg_file(filename, project_id, source_name, version_number, validati
     # to our project via the tenant.
     source.projects.add(project_id)
     project = Project.objects.get(id=project_id)
-    import_gpkg_file2(filename, project, source, version_number, validation_status, user=None, description=description)
+    import_gpkg_file2(
+        filename, project, source, version_number, validation_status, user=None, description=description, task=None
+    )
 
 
 @transaction.atomic
@@ -171,6 +173,7 @@ def import_gpkg_file2(
     validation_status,
     user: Optional[User],
     description,
+    task,
 ):
     if version_number is None:
         last_version = source.versions.all().order_by("number").last()
@@ -217,6 +220,10 @@ def import_gpkg_file2(
     # Layer are OrgUnit's Type
     layers_name = fiona.listlayers(filename)
     for layer_name in layers_name:
+        if task:
+            task.report_progress_and_stop_if_killed(
+                progress_message=f"processing layer : {layer_name} total_org_unit : {total_org_unit}"
+            )
         # layers to import must be named level-{depth}-{name}
         if not layer_name.startswith("level-"):
             continue
@@ -234,6 +241,12 @@ def import_gpkg_file2(
 
             existing_ou = ref_ou.get(ref, None)
             orgunit = create_or_update_orgunit(existing_ou, row, version, validation_status, ref_group)
+
+            if task and total_org_unit % 500 == 0:
+                task.report_progress_and_stop_if_killed(
+                    progress_message=f"processing layer : {layer_name} {orgunit.name} total_org_unit : {total_org_unit}"
+                )
+
             ref = get_ref(orgunit)  # if ref was null in gpkg
             ref_ou[ref] = orgunit
 
@@ -244,16 +257,28 @@ def import_gpkg_file2(
                 modifications_to_log.append((existing_ou, orgunit))
 
             total_org_unit += 1
-
+    if task:
+        task.report_progress_and_stop_if_killed(
+            progress_message=f"processing parents : total_org_unit : {total_org_unit} to_update_with_parent : {len(to_update_with_parent)}"
+        )
+    parent_count = 0
     for ref, parent_ref in to_update_with_parent:
+        parent_count += 1
         ou = ref_ou[ref]
         if parent_ref and parent_ref not in ref_ou:
             raise ValueError(f"Bad GPKG parent {parent_ref} for {ou} don't exist in input or SourceVersion")
 
+        if task and parent_count % 1000 == 0:
+            task.report_progress_and_stop_if_killed(
+                progress_message=f"processing parent : parent_count : #{parent_count}"
+            )
         parent_ou = ref_ou[parent_ref] if parent_ref else None
         ou.parent = parent_ou
         ou.save()
-
+    if task:
+        task.report_progress_and_stop_if_killed(
+            progress_message=f"storing log_modifications total_org_unit : {total_org_unit}"
+        )
     for old_ou, new_ou in modifications_to_log:
         # Possible optimisation, crate a bulk update
         audit_models.log_modification(old_ou, new_ou, source=audit_models.GPKG_IMPORT, user=user)

iaso/tasks/import_gpkg_task.py

@@ -29,6 +29,7 @@ def import_gpkg_task(import_gpkg_id: int, task: Task):
             validation_status="NEW",
             user=user,
             description=ig.description,
+            task=task,
         )
 
         task.report_success(message=f"Imported {total} OrgUnits")