diff --git a/config/settings/base.py b/config/settings/base.py index 616b9c92..46dfe692 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -339,3 +339,7 @@ SINEQUA_CONFIGS_REPO_DEV_BRANCH = env("SINEQUA_CONFIGS_REPO_DEV_BRANCH") SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH = env("SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH") SLACK_WEBHOOK_URL = env("SLACK_WEBHOOK_URL") +XLI_USER = env("XLI_USER") +XLI_PASSWORD = env("XLI_PASSWORD") +LRM_USER = env("LRM_USER") +LRM_PASSWORD = env("LRM_PASSWORD") diff --git a/sde_collections/migrations/0056_alter_candidateurl_document_type_and_more.py b/sde_collections/migrations/0056_alter_candidateurl_document_type_and_more.py new file mode 100644 index 00000000..50168981 --- /dev/null +++ b/sde_collections/migrations/0056_alter_candidateurl_document_type_and_more.py @@ -0,0 +1,54 @@ +# Generated by Django 4.2.9 on 2024-06-24 16:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sde_collections", "0055_alter_workflowhistory_old_status_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="candidateurl", + name="document_type", + field=models.IntegerField( + choices=[ + (1, "Images"), + (2, "Data"), + (3, "Documentation"), + (4, "Software and Tools"), + (5, "Missions and Instruments"), + ], + null=True, + ), + ), + migrations.AlterField( + model_name="collection", + name="document_type", + field=models.IntegerField( + choices=[ + (1, "Images"), + (2, "Data"), + (3, "Documentation"), + (4, "Software and Tools"), + (5, "Missions and Instruments"), + ], + default=3, + ), + ), + migrations.AlterField( + model_name="documenttypepattern", + name="document_type", + field=models.IntegerField( + choices=[ + (1, "Images"), + (2, "Data"), + (3, "Documentation"), + (4, "Software and Tools"), + (5, "Missions and Instruments"), + ] + ), + ), + ] diff --git a/sde_collections/models/collection_choice_fields.py b/sde_collections/models/collection_choice_fields.py index 37ac9412..9ab5b2e7 100644 --- a/sde_collections/models/collection_choice_fields.py +++ b/sde_collections/models/collection_choice_fields.py @@ -30,7 +30,6 @@ class DocumentTypes(models.IntegerChoices): DOCUMENTATION = 3, "Documentation" SOFTWARETOOLS = 4, "Software and Tools" MISSIONSINSTRUMENTS = 5, "Missions and Instruments" - TRAININGANDEDUCATION = 6, "Training and Education" @classmethod def lookup_by_text(cls, text: str) -> int | None: diff --git a/sde_collections/models/pattern.py b/sde_collections/models/pattern.py index 29248e50..0849c6fd 100644 --- a/sde_collections/models/pattern.py +++ b/sde_collections/models/pattern.py @@ -2,11 +2,7 @@ from django.apps import apps from django.core.exceptions import ValidationError -from django.db import models, transaction -from django.db.models.signals import post_save -from django.dispatch import receiver - -from sde_collections.tasks import resolve_title_pattern +from django.db import models from ..utils.title_resolver import ( is_valid_fstring, @@ -175,10 +171,8 @@ def apply(self) -> None: "title": candidate_url.scraped_title, "collection": self.collection.name, } - try: - # generated_title = resolve_title(self.title_pattern, context) - generated_title = self.title_pattern + generated_title = resolve_title(self.title_pattern, context) # check to see if the candidate url has an existing resolved title and delete it ResolvedTitle.objects.filter(candidate_url=candidate_url).delete() @@ -190,6 +184,7 @@ def apply(self) -> None: candidate_url.generated_title = generated_title candidate_url.save() + updated_urls.append(candidate_url) except (ValueError, ValidationError) as e: message = str(e) @@ -210,7 +205,15 @@ def apply(self) -> None: TitlePatternCandidateURL.objects.bulk_create(pattern_url_associations, ignore_conflicts=True) def unapply(self) -> None: - self.candidate_urls.update(generated_title="") + candidate_urls = self.candidate_urls.all() + for candidate_url in candidate_urls: + candidate_url.generated_title = "" + candidate_url.save() + self.candidate_urls.clear() + + def delete(self, *args, **kwargs): + self.unapply() + super().delete(*args, **kwargs) class Meta: """Meta definition for TitlePattern.""" diff --git a/sde_collections/sinequa_api.py b/sde_collections/sinequa_api.py index 192a8176..17c03b33 100644 --- a/sde_collections/sinequa_api.py +++ b/sde_collections/sinequa_api.py @@ -2,6 +2,7 @@ import requests import urllib3 +from django.conf import settings urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -50,6 +51,10 @@ def __init__(self, server_name: str) -> None: self.app_name: str = server_configs[server_name]["app_name"] self.query_name: str = server_configs[server_name]["query_name"] self.base_url: str = server_configs[server_name]["base_url"] + self.xli_user = settings.XLI_USER + self.xli_password = settings.XLI_PASSWORD + self.lrm_user = settings.LRM_USER + self.lrm_password = settings.LRM_PASSWORD def process_response(self, url: str, payload: dict[str, Any]) -> Any: response = requests.post(url, headers={}, json=payload, verify=False) @@ -63,9 +68,9 @@ def process_response(self, url: str, payload: dict[str, Any]) -> Any: def query(self, page: int, collection_config_folder: str = "") -> Any: if self.server_name == "lis_server": - url = f"{self.base_url}/api/v1/search.query?Password=admin&User=admin" + url = f"{self.base_url}/api/v1/search.query?Password={self.xli_password}&User={self.xli_user}" elif self.server_name == "lrm_dev_server": - url = f"{self.base_url}/api/v1/search.query?Password=QDZ8ASZagUpRCHR&User=lrmdev" + url = f"{self.base_url}/api/v1/search.query?Password={self.lrm_password}&User={self.lrm_user}" else: url = f"{self.base_url}/api/v1/search.query" payload = { diff --git a/sde_collections/tasks.py b/sde_collections/tasks.py index 659f4cc8..fa754efc 100644 --- a/sde_collections/tasks.py +++ b/sde_collections/tasks.py @@ -10,7 +10,7 @@ from config import celery_app -from .models.collection import Collection +from .models.collection import Collection, WorkflowStatusChoices from .sinequa_api import Api from .utils.github_helper import GitHubHandler @@ -90,6 +90,14 @@ def import_candidate_urls_from_api(server_name="test", collection_ids=[]): print("Applying existing patterns; this may take a while") collection.apply_all_patterns() + if collection.workflow_status == WorkflowStatusChoices.READY_FOR_ENGINEERING: + collection.workflow_status = WorkflowStatusChoices.ENGINEERING_IN_PROGRESS + collection.save() + + # Finally set the status to READY_FOR_CURATION + collection.workflow_status = WorkflowStatusChoices.READY_FOR_CURATION + collection.save() + print("Deleting temp files") shutil.rmtree(TEMP_FOLDER_NAME) diff --git a/sde_indexing_helper/static/css/candidate_url_list.css b/sde_indexing_helper/static/css/candidate_url_list.css index 304887af..004da9b8 100644 --- a/sde_indexing_helper/static/css/candidate_url_list.css +++ b/sde_indexing_helper/static/css/candidate_url_list.css @@ -90,11 +90,11 @@ border-color: #fafafa; font-size: 0.6875rem; box-shadow: 0 2px 2px 0 rgba(153, 153, 153, 0.14), 0 3px 1px -2px rgba(153, 153, 153, 0.2), 0 1px 5px 0 rgba(153, 153, 153, 0.12); } - + .select-dropdown:hover { box-shadow: 0 14px 26px -12px rgba(250, 250, 250, 0.42), 0 4px 23px 0px rgba(0, 0, 0, 0.12), 0 8px 10px -5px rgba(250, 250, 250, 0.2); } - + .select-dropdown:focus, .select-dropdown.focus { box-shadow: none, 0 0 0 0.2rem rgba(76, 175, 80, 0.5); @@ -194,7 +194,7 @@ letter-spacing: -0.02em; display: flex; align-items: baseline; } - + .checkbox-wrapper label { font-weight: 600; font-size: 16px; @@ -228,7 +228,7 @@ letter-spacing: -0.02em; width: 600px; color: #65B1EF; } - + .title-dropdown { width: fit-content !important; margin-top:20px; @@ -237,7 +237,7 @@ letter-spacing: -0.02em; .table tbody tr:nth-child(odd) { background-color: #050E19 !important; } - + .table tbody tr:nth-child(even) { background-color: #3F4A58 !important; } @@ -247,7 +247,7 @@ letter-spacing: -0.02em; } - + .custom-select, .buttons-csv, .customizeColumns, .addPattern{ border-style: solid !important; border-color: #A7BACD !important; @@ -346,7 +346,7 @@ div.dt-buttons .btn.processing:after { align-items: center; /* justify-content: space-between; */ } - + .headerDiv{ display: flex; justify-content: space-between; @@ -356,6 +356,12 @@ div.dt-buttons .btn.processing:after { display:flex; align-items: center; justify-content: space-between; + word-wrap: break-word; + word-break: break-all; + white-space: normal; + overflow-wrap: break-word; + min-width: 100%; + max-width: 100%; } .url-icon { @@ -415,4 +421,10 @@ div.dt-buttons .btn.processing:after { div.dt-container div.dt-paging ul.pagination { position: absolute; right: 60px; -} \ No newline at end of file +} + +.individual_title_input { + width: 100%; + max-width: 100%; + min-width: 100%; +} diff --git a/sde_indexing_helper/static/js/candidate_url_list.js b/sde_indexing_helper/static/js/candidate_url_list.js index a4184802..4e9b0ae6 100644 --- a/sde_indexing_helper/static/js/candidate_url_list.js +++ b/sde_indexing_helper/static/js/candidate_url_list.js @@ -225,7 +225,6 @@ function initializeDataTable() { 3: "Documentation", 4: "Software and Tools", 5: "Missions and Instruments", - 6: "Training and Education", }; this.api() .columns() @@ -604,7 +603,6 @@ function initializeDataTable() { Documentation: 3, "Software and Tools": 4, "Missions and Instruments": 5, - "Training and Education": 6, }, }, }; @@ -707,10 +705,11 @@ function setupClickHandlers() { function getURLColumn() { return { data: "url", + width: "30%", render: function (data, type, row) { return `
${remove_protocol( data - )} + )} open_in_new
`; @@ -721,6 +720,7 @@ function getURLColumn() { function getScrapedTitleColumn() { return { data: "scraped_title", + width: "30%", render: function (data, type, row) { return `${data}`; }, @@ -730,6 +730,7 @@ function getScrapedTitleColumn() { function getGeneratedTitleColumn() { return { data: "generated_title", + width: "20%", render: function (data, type, row) { return `Documentation Software and Tools Missions and Instruments - Training and Education `; }, @@ -806,7 +807,7 @@ function addEnterEscapeKeypress(modalID, formID) { modal.modal("hide"); } }) -} +} function handleHideorShowKeypress() { $("body").on("keydown", function () { @@ -997,16 +998,14 @@ function postDocumentTypePatterns( success: function (data) { $("#candidate_urls_table").DataTable().ajax.reload(null, false); $("#document_type_patterns_table").DataTable().ajax.reload(null, false); - if (currentTab === "") { - //Only add a notification if we are on the first tab - newDocumentTypePatternsCount = newDocumentTypePatternsCount + 1; - $("#documentTypePatternsTab").html( - `Document Type Patterns ` + - newDocumentTypePatternsCount + - " new" + - `` - ); - } + if(currentTab === ""){ //Only add a notification if we are on the first tab + newDocumentTypePatternsCount = newDocumentTypePatternsCount + 1; + $("#documentTypePatternsTab").html( + `Document Type Patterns ` + + newDocumentTypePatternsCount + " new" + + `` + ); + } }, error: function (xhr, status, error) { var errorMessage = xhr.responseText; @@ -1050,16 +1049,14 @@ function postExcludePatterns(match_pattern, match_pattern_type = 0, force) { success: function (data) { $("#candidate_urls_table").DataTable().ajax.reload(null, false); $("#exclude_patterns_table").DataTable().ajax.reload(null, false); - if (currentTab === "") { - //Only add a notification if we are on the first tab - newExcludePatternsCount = newExcludePatternsCount + 1; - $("#excludePatternsTab").html( - `Exclude Patterns ` + - newExcludePatternsCount + - " new" + - `` - ); - } + if(currentTab === ""){ //Only add a notification if we are on the first tab + newExcludePatternsCount = newExcludePatternsCount + 1; + $("#excludePatternsTab").html( + `Exclude Patterns ` + + newExcludePatternsCount + " new" + + `` + ); + } }, error: function (xhr, status, error) { var errorMessage = xhr.responseText; @@ -1094,16 +1091,14 @@ function postIncludePatterns(match_pattern, match_pattern_type = 0) { success: function (data) { $("#candidate_urls_table").DataTable().ajax.reload(null, false); $("#include_patterns_table").DataTable().ajax.reload(null, false); - if (currentTab === "") { - //Only add a notification if we are on the first tab - newIncludePatternsCount = newIncludePatternsCount + 1; - $("#includePatternsTab").html( - `Include Patterns ` + - newIncludePatternsCount + - " new" + - `` - ); - } + if(currentTab === ""){ //Only add a notification if we are on the first tab + newIncludePatternsCount = newIncludePatternsCount + 1; + $("#includePatternsTab").html( + `Include Patterns ` + + newIncludePatternsCount + " new" + + `` + ); + } }, error: function (xhr, status, error) { var errorMessage = xhr.responseText; @@ -1122,45 +1117,40 @@ function postTitlePatterns( return; } - $.ajax({ - url: "/api/title-patterns/", - type: "POST", - data: { - collection: collection_id, - match_pattern: match_pattern, - match_pattern_type: match_pattern_type, - title_pattern: title_pattern, - csrfmiddlewaretoken: csrftoken, - }, - success: function (data) { - $("#candidate_urls_table").DataTable().ajax.reload(null, false); - $("#title_patterns_table").DataTable().ajax.reload(null, false); - if (currentTab === "") { - //Only add a notification if we are on the first tab - newTitlePatternsCount = newTitlePatternsCount + 1; - $("#titlePatternsTab").html( - `Title Patterns ` + - newTitlePatternsCount + - " new" + - `` - ); - } - }, - error: function (xhr, status, error) { - var errorMessage = xhr.responseText; - if ( - errorMessage == - '{"error":{"non_field_errors":["The fields collection, match_pattern must make a unique set."]},"status_code":400}' - ) { - toastr.success("Pattern already exists"); - return; - } - var errorMessages = JSON.parse(errorMessage); - Object.entries(errorMessages.error).forEach(([key, value]) => { - toastr.error(value, key); - }); - }, - }); + $.ajax({ + url: '/api/title-patterns/', + type: "POST", + data: { + collection: collection_id, + match_pattern: match_pattern, + match_pattern_type: match_pattern_type, + title_pattern: title_pattern, + csrfmiddlewaretoken: csrftoken + }, + success: function (data) { + $('#candidate_urls_table').DataTable().ajax.reload(null, false); + $('#title_patterns_table').DataTable().ajax.reload(null, false); + if(currentTab === ""){ //Only add a notification if we are on the first tab + newTitlePatternsCount = newTitlePatternsCount + 1; + $("#titlePatternsTab").html( + `Title Patterns ` + + newTitlePatternsCount + " new" + + `` + ); + } + }, + error: function (xhr, status, error) { + var errorMessage = xhr.responseText; + if (errorMessage == '{"error":{"non_field_errors":["The fields collection, match_pattern must make a unique set."]},"status_code":400}') { + toastr.success("Pattern already exists"); + return; + } + var errorMessages = JSON.parse(errorMessage); + Object.entries(errorMessages.error).forEach(([key, value]) => { + toastr.error(value, key); + }); + } + }); } function postVisited(url) { diff --git a/sde_indexing_helper/static/js/collection_list.js b/sde_indexing_helper/static/js/collection_list.js index de8ed457..5a00c1a4 100644 --- a/sde_indexing_helper/static/js/collection_list.js +++ b/sde_indexing_helper/static/js/collection_list.js @@ -391,6 +391,9 @@ function postCurator(collection_id, curator_id) { $(document).ready(function () { setupClickHandlers(); + // Clear search values and redraw table + clearSearchValues(); + // Remove the search input and add custom titles var paneTitles = [ null, @@ -424,3 +427,8 @@ function setupClickHandlers() { handleWorkflowStatusSelect(); handleCuratorSelect(); } + +function clearSearchValues() { + let table = $("#collection_table").DataTable(); + table.columns().search("").draw(); +} diff --git a/sde_indexing_helper/templates/sde_collections/candidate_urls_list.html b/sde_indexing_helper/templates/sde_collections/candidate_urls_list.html index da0871df..e4867159 100644 --- a/sde_indexing_helper/templates/sde_collections/candidate_urls_list.html +++ b/sde_indexing_helper/templates/sde_collections/candidate_urls_list.html @@ -106,7 +106,6 @@

- @@ -253,7 +252,6 @@

- @@ -476,4 +474,4 @@

Are you sure?
-{% endblock content %} \ No newline at end of file +{% endblock content %} diff --git a/sde_indexing_helper/templates/sde_collections/collection_detail.html b/sde_indexing_helper/templates/sde_collections/collection_detail.html index e957e832..c57ab212 100644 --- a/sde_indexing_helper/templates/sde_collections/collection_detail.html +++ b/sde_indexing_helper/templates/sde_collections/collection_detail.html @@ -40,7 +40,7 @@

{{ colle data-match-pattern remove_protocol row - url> + url> - + {{ entry.curated_by }} {{entry.created_at|timesince}} ago @@ -324,8 +322,8 @@
Are you sure?
- - + +