Merge pull request #882 from NASA-IMPACT/dev-enhanced-merged

Recent changes to the dev-enhanced-merged branch

config/settings/base.py

@@ -339,3 +339,7 @@
 SINEQUA_CONFIGS_REPO_DEV_BRANCH = env("SINEQUA_CONFIGS_REPO_DEV_BRANCH")
 SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH = env("SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH")
 SLACK_WEBHOOK_URL = env("SLACK_WEBHOOK_URL")
+XLI_USER = env("XLI_USER")
+XLI_PASSWORD = env("XLI_PASSWORD")
+LRM_USER = env("LRM_USER")
+LRM_PASSWORD = env("LRM_PASSWORD")

sde_collections/migrations/0056_alter_candidateurl_document_type_and_more.py

@@ -0,0 +1,54 @@
+# Generated by Django 4.2.9 on 2024-06-24 16:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sde_collections", "0055_alter_workflowhistory_old_status_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="candidateurl",
+            name="document_type",
+            field=models.IntegerField(
+                choices=[
+                    (1, "Images"),
+                    (2, "Data"),
+                    (3, "Documentation"),
+                    (4, "Software and Tools"),
+                    (5, "Missions and Instruments"),
+                ],
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="collection",
+            name="document_type",
+            field=models.IntegerField(
+                choices=[
+                    (1, "Images"),
+                    (2, "Data"),
+                    (3, "Documentation"),
+                    (4, "Software and Tools"),
+                    (5, "Missions and Instruments"),
+                ],
+                default=3,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="documenttypepattern",
+            name="document_type",
+            field=models.IntegerField(
+                choices=[
+                    (1, "Images"),
+                    (2, "Data"),
+                    (3, "Documentation"),
+                    (4, "Software and Tools"),
+                    (5, "Missions and Instruments"),
+                ]
+            ),
+        ),
+    ]

sde_collections/models/collection_choice_fields.py

@@ -30,7 +30,6 @@ class DocumentTypes(models.IntegerChoices):
     DOCUMENTATION = 3, "Documentation"
     SOFTWARETOOLS = 4, "Software and Tools"
     MISSIONSINSTRUMENTS = 5, "Missions and Instruments"
-    TRAININGANDEDUCATION = 6, "Training and Education"
 
     @classmethod
     def lookup_by_text(cls, text: str) -> int | None:

sde_collections/models/pattern.py

@@ -2,11 +2,7 @@
 
 from django.apps import apps
 from django.core.exceptions import ValidationError
-from django.db import models, transaction
-from django.db.models.signals import post_save
-from django.dispatch import receiver
-
-from sde_collections.tasks import resolve_title_pattern
+from django.db import models
 
 from ..utils.title_resolver import (
     is_valid_fstring,
@@ -175,10 +171,8 @@ def apply(self) -> None:
                 "title": candidate_url.scraped_title,
                 "collection": self.collection.name,
             }
-
             try:
-                # generated_title = resolve_title(self.title_pattern, context)
-                generated_title = self.title_pattern
+                generated_title = resolve_title(self.title_pattern, context)
 
                 # check to see if the candidate url has an existing resolved title and delete it
                 ResolvedTitle.objects.filter(candidate_url=candidate_url).delete()
@@ -190,6 +184,7 @@ def apply(self) -> None:
 
                 candidate_url.generated_title = generated_title
                 candidate_url.save()
+                updated_urls.append(candidate_url)
 
             except (ValueError, ValidationError) as e:
                 message = str(e)
@@ -210,7 +205,15 @@ def apply(self) -> None:
         TitlePatternCandidateURL.objects.bulk_create(pattern_url_associations, ignore_conflicts=True)
 
     def unapply(self) -> None:
-        self.candidate_urls.update(generated_title="")
+        candidate_urls = self.candidate_urls.all()
+        for candidate_url in candidate_urls:
+            candidate_url.generated_title = ""
+            candidate_url.save()
+        self.candidate_urls.clear()
+
+    def delete(self, *args, **kwargs):
+        self.unapply()
+        super().delete(*args, **kwargs)
 
     class Meta:
         """Meta definition for TitlePattern."""

sde_collections/sinequa_api.py

@@ -2,6 +2,7 @@
 
 import requests
 import urllib3
+from django.conf import settings
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -50,6 +51,10 @@ def __init__(self, server_name: str) -> None:
         self.app_name: str = server_configs[server_name]["app_name"]
         self.query_name: str = server_configs[server_name]["query_name"]
         self.base_url: str = server_configs[server_name]["base_url"]
+        self.xli_user = settings.XLI_USER
+        self.xli_password = settings.XLI_PASSWORD
+        self.lrm_user = settings.LRM_USER
+        self.lrm_password = settings.LRM_PASSWORD
 
     def process_response(self, url: str, payload: dict[str, Any]) -> Any:
         response = requests.post(url, headers={}, json=payload, verify=False)
@@ -63,9 +68,9 @@ def process_response(self, url: str, payload: dict[str, Any]) -> Any:
 
     def query(self, page: int, collection_config_folder: str = "") -> Any:
         if self.server_name == "lis_server":
-            url = f"{self.base_url}/api/v1/search.query?Password=admin&User=admin"
+            url = f"{self.base_url}/api/v1/search.query?Password={self.xli_password}&User={self.xli_user}"
         elif self.server_name == "lrm_dev_server":
-            url = f"{self.base_url}/api/v1/search.query?Password=QDZ8ASZagUpRCHR&User=lrmdev"
+            url = f"{self.base_url}/api/v1/search.query?Password={self.lrm_password}&User={self.lrm_user}"
         else:
             url = f"{self.base_url}/api/v1/search.query"
         payload = {

sde_collections/tasks.py

@@ -10,7 +10,7 @@
 
 from config import celery_app
 
-from .models.collection import Collection
+from .models.collection import Collection, WorkflowStatusChoices
 from .sinequa_api import Api
 from .utils.github_helper import GitHubHandler
 
@@ -90,6 +90,14 @@ def import_candidate_urls_from_api(server_name="test", collection_ids=[]):
         print("Applying existing patterns; this may take a while")
         collection.apply_all_patterns()
 
+        if collection.workflow_status == WorkflowStatusChoices.READY_FOR_ENGINEERING:
+            collection.workflow_status = WorkflowStatusChoices.ENGINEERING_IN_PROGRESS
+            collection.save()
+
+        # Finally set the status to READY_FOR_CURATION
+        collection.workflow_status = WorkflowStatusChoices.READY_FOR_CURATION
+        collection.save()
+
     print("Deleting temp files")
     shutil.rmtree(TEMP_FOLDER_NAME)
 

sde_indexing_helper/static/css/candidate_url_list.css

@@ -90,11 +90,11 @@
     border-color:  #fafafa;
     font-size: 0.6875rem;
     box-shadow: 0 2px 2px 0 rgba(153, 153, 153, 0.14), 0 3px 1px -2px rgba(153, 153, 153, 0.2), 0 1px 5px 0 rgba(153, 153, 153, 0.12);  }
-  
+
   .select-dropdown:hover {
     box-shadow: 0 14px 26px -12px rgba(250, 250, 250, 0.42), 0 4px 23px 0px rgba(0, 0, 0, 0.12), 0 8px 10px -5px rgba(250, 250, 250, 0.2);
   }
-  
+
   .select-dropdown:focus,
   .select-dropdown.focus {
     box-shadow: none, 0 0 0 0.2rem rgba(76, 175, 80, 0.5);
@@ -194,7 +194,7 @@ letter-spacing: -0.02em;
     display: flex;
     align-items: baseline;
   }
-  
+
   .checkbox-wrapper label {
     font-weight: 600;
     font-size: 16px;
@@ -228,7 +228,7 @@ letter-spacing: -0.02em;
     width: 600px;
     color: #65B1EF;
   }
-  
+
 .title-dropdown {
     width: fit-content !important;
     margin-top:20px;
@@ -237,7 +237,7 @@ letter-spacing: -0.02em;
 .table tbody tr:nth-child(odd) {
     background-color: #050E19 !important;
    }
-  
+
    .table tbody tr:nth-child(even) {
     background-color: #3F4A58 !important;
    }
@@ -247,7 +247,7 @@ letter-spacing: -0.02em;
    }
 
 
-   
+
    .custom-select, .buttons-csv, .customizeColumns, .addPattern{
     border-style: solid !important;
     border-color: #A7BACD !important;
@@ -346,7 +346,7 @@ div.dt-buttons .btn.processing:after {
     align-items: center;
     /* justify-content: space-between; */
  }
- 
+
 .headerDiv{
     display: flex;
     justify-content: space-between;
@@ -356,6 +356,12 @@ div.dt-buttons .btn.processing:after {
     display:flex;
     align-items: center;
     justify-content: space-between;
+    word-wrap: break-word;
+    word-break: break-all;
+    white-space: normal;
+    overflow-wrap: break-word;
+    min-width: 100%;
+    max-width: 100%;
  }
 
  .url-icon {
@@ -415,4 +421,10 @@ div.dt-buttons .btn.processing:after {
 div.dt-container div.dt-paging ul.pagination {
     position: absolute;
     right: 60px;
-}
+}
+
+.individual_title_input {
+    width: 100%;
+    max-width: 100%;
+    min-width: 100%;
+}