From 528f10876238a9438e5e7760ebae0ff3e3fbb482 Mon Sep 17 00:00:00 2001
From: Ashish Acharya <ashish.acharya14@gmail.com>
Date: Tue, 17 Oct 2023 11:04:14 -0500
Subject: [PATCH 01/10] Remove the push to github button since it's broken

---
 .../static/js/collection_list.js              | 28 -------------------
 1 file changed, 28 deletions(-)

diff --git a/sde_indexing_helper/static/js/collection_list.js b/sde_indexing_helper/static/js/collection_list.js
index cd2f127b..ad3273c6 100644
--- a/sde_indexing_helper/static/js/collection_list.js
+++ b/sde_indexing_helper/static/js/collection_list.js
@@ -16,34 +16,6 @@ let table = $('#collection_table').DataTable({
                     'collections.json'
                 );
             }
-        },
-        {
-            text: 'Push selected collections to GitHub',
-            action: function (e, dt, node, config) {
-                var collection_ids = [];
-                $('#collection_table').DataTable().rows({ selected: true }).every(function (rowIdx, tableLoop, rowLoop) {
-                    var data = this.data();
-                    var collection_name = $(data[1]).text().slice(0, -14); // remove " chevron_right" from end of string
-                    var collection_id = $(data[1]).attr('href').slice(1, -1); // we get /932/ from href="/932/"
-                    var curation_status = $(data[6]).find('button').text();
-                    var workflow_status = $(data[7]).find('button').text();
-
-                    if (workflow_status !== "Curated") {
-                        toastr.error(`Can't push <strong>${collection_name}</strong> because workflow status is not "Curated".`);
-                    } else {
-                        collection_ids.push(collection_id);
-                        toastr.success(`Started pushing <strong>${collection_name}</strong> to GitHub...`);
-                    }
-                });
-                $.ajax({
-                    url: '/api/collections/push_to_github/',
-                    type: "POST",
-                    data: {
-                        collection_ids: collection_ids,
-                        csrfmiddlewaretoken: csrftoken
-                    },
-                });
-            }
         }
     ],
     "columnDefs": [

From 61c625431cd27a0a8ea317984959299fc72c64e6 Mon Sep 17 00:00:00 2001
From: Ashish Acharya <ashish.acharya14@gmail.com>
Date: Tue, 17 Oct 2023 11:18:44 -0500
Subject: [PATCH 02/10] Get rid of extraneous columns and filters for
 collections

---
 .../static/js/collection_list.js              | 19 ++---------
 .../sde_collections/collection_list.html      | 33 -------------------
 2 files changed, 2 insertions(+), 50 deletions(-)

diff --git a/sde_indexing_helper/static/js/collection_list.js b/sde_indexing_helper/static/js/collection_list.js
index ad3273c6..844da442 100644
--- a/sde_indexing_helper/static/js/collection_list.js
+++ b/sde_indexing_helper/static/js/collection_list.js
@@ -1,5 +1,4 @@
 let table = $('#collection_table').DataTable({
-    "order": [[0, 'asc']],
     "paging": false,
     "stateSave": true,
     "dom": 'BPfritip',
@@ -19,14 +18,6 @@ let table = $('#collection_table').DataTable({
         }
     ],
     "columnDefs": [
-        {
-            target: -1,
-            visible: false,
-        },
-        {
-            target: 3,
-            sortable: false,
-        },
         {
             searchPanes: {
                 options: [
@@ -74,14 +65,8 @@ let table = $('#collection_table').DataTable({
                     }
                 ]
             },
-            targets: [4]
-        },
-        {
-            searchPanes: {
-                show: true
-            },
-            targets: [-2]
-        },
+            targets: [3]
+        }
     ]
 });
 
diff --git a/sde_indexing_helper/templates/sde_collections/collection_list.html b/sde_indexing_helper/templates/sde_collections/collection_list.html
index 74036b77..2c5a8e7a 100644
--- a/sde_indexing_helper/templates/sde_collections/collection_list.html
+++ b/sde_indexing_helper/templates/sde_collections/collection_list.html
@@ -15,16 +15,12 @@ <h2>SDE Collections</h2>
     <table id="collection_table" class="table table-striped">
         <thead>
             <tr>
-                <th class="text-center">Priority</th>
                 <th class="text-center">Name</th>
                 <th class="text-center">URL</th>
                 <th class="text-center">Division</th>
                 <th class="text-center">Candidate URLs</th>
-                <th class="text-center">New?</th>
-                <th class="text-center">Status</th>
                 <th class="text-center">Workflow Status</th>
                 <th class="text-center">Curator</th>
-                <th class="text-center">Has Config?</th>
                 <th class="text-center">Connector Type</th>
                 <th class="text-center">Config Folder</th>
             </tr>
@@ -32,7 +28,6 @@ <h2>SDE Collections</h2>
         <tbody>
             {% for collection in collections %}
                 <tr>
-                    <td>{{ collection.cleaning_order }}</td>
                     <td>
                         <a href="{% url 'sde_collections:detail' collection.pk %}">{{ collection.name }} <i class="material-icons">chevron_right</i></a>
                     </td>
@@ -47,33 +42,6 @@ <h2>SDE Collections</h2>
                            class="btn btn-sm {% if collection.num_candidate_urls > 0 %}btn-primary {% else %}disabled{% endif %}"
                            role="button">{{ collection.num_candidate_urls }}</a>
                     </td>
-                    <td>
-                        {% if collection.new_collection %}
-                            <i class="material-icons" style="color:green;">check</i>
-                        {% else %}
-                            <i class="material-icons" style="color:red;">close</i>
-                        {% endif %}
-                    </td>
-                    <td>
-                        <div class="dropdown curation_status_dropdown"
-                             data-match-pattern
-                             remove_protocol
-                             row
-                             url>
-                            <button class="btn {{ collection.curation_status_button_color }} btn-sm dropdown-toggle"
-                                    type="button"
-                                    id="curation-status-button-{{ collection.id }}"
-                                    data-toggle="dropdown"
-                                    aria-haspopup="true"
-                                    aria-expanded="false">{{ collection.get_curation_status_display }}</button>
-                            <div class="dropdown-menu"
-                                 aria-labelledby="curation-status-button-{{ collection.id }}">
-                                {% for choice in curation_status_choices %}
-                                    <a class="dropdown-item curation_status_select" value="{{ choice }}" data-collection-id={{ collection.id }}>{{ choice.label }}</a>
-                                {% endfor %}
-                            </div>
-                        </div>
-                    </td>
                     <td>
                         <div class="dropdown workflow_status_dropdown"
                              data-match-pattern
@@ -114,7 +82,6 @@ <h2>SDE Collections</h2>
                             </div>
                         </div>
                     </td>
-                    <td>{{ collection.has_sinequa_config }}</td>
                     <td>{{ collection.get_connector_display }}</td>
                     <td>{{ collection.config_folder }}</td>
                 </tr>

From d15080af6c02378e112cc03878c147ea911e1c22 Mon Sep 17 00:00:00 2001
From: Ashish Acharya <ashish.acharya14@gmail.com>
Date: Tue, 17 Oct 2023 11:19:18 -0500
Subject: [PATCH 03/10] Turn off select for collections

---
 sde_indexing_helper/static/js/collection_list.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sde_indexing_helper/static/js/collection_list.js b/sde_indexing_helper/static/js/collection_list.js
index 844da442..5898a3b4 100644
--- a/sde_indexing_helper/static/js/collection_list.js
+++ b/sde_indexing_helper/static/js/collection_list.js
@@ -2,7 +2,6 @@ let table = $('#collection_table').DataTable({
     "paging": false,
     "stateSave": true,
     "dom": 'BPfritip',
-    "select": true,
     "buttons": [
         'csv',
         {

From 690b76d4dc8b05130e563759cd595fcb35de4a5a Mon Sep 17 00:00:00 2001
From: Ashish Acharya <ashish.acharya14@gmail.com>
Date: Tue, 17 Oct 2023 11:45:22 -0500
Subject: [PATCH 04/10] Revert frontend changes from recent merges

---
 .../static/js/candidate_url_list.js           | 63 ++++---------------
 .../sde_collections/candidate_urls_list.html  | 21 -------
 2 files changed, 13 insertions(+), 71 deletions(-)

diff --git a/sde_indexing_helper/static/js/candidate_url_list.js b/sde_indexing_helper/static/js/candidate_url_list.js
index 5295e22e..4f574069 100644
--- a/sde_indexing_helper/static/js/candidate_url_list.js
+++ b/sde_indexing_helper/static/js/candidate_url_list.js
@@ -4,8 +4,7 @@ var collection_id = getCollectionId();
 var selected_text = "";
 var INDIVIDUAL_URL = 1
 var MULTI_URL_PATTERN = 2
-var candidate_urls_table;
-var url_lists=null;
+
 $(document).ready(function () {
     handleAjaxStartAndStop();
     initializeDataTable();
@@ -20,7 +19,7 @@ function initializeDataTable() {
     var true_icon = '<i class="material-icons" style="color: green">check</i>';
     var false_icon = '<i class="material-icons" style="color: red">close</i>';
 
-    candidate_urls_table = $('#candidate_urls_table').DataTable({
+    var candidate_urls_table = $('#candidate_urls_table').DataTable({
         "scrollY": true,
         "serverSide": true,
         "stateSave": true,
@@ -50,9 +49,6 @@ function initializeDataTable() {
             { "data": "generated_title_id", "visible": false, "searchable": false },
             { "data": "match_pattern_type", "visible": false, "searchable": false },
             { "data": "candidate_urls_count", "visible": false, "searchable": false },
-            { "data": "inferenced_by", "visible": false, "searchable": false },
-            { "data": "is_pdf", "visible": false, "searchable": false }
-
         ],
         "createdRow": function (row, data, dataIndex) {
             if (data['excluded']) {
@@ -60,7 +56,7 @@ function initializeDataTable() {
             }
         }
     });
-}
+
     var exclude_patterns_table = $('#exclude_patterns_table').DataTable({
         "scrollY": true,
         "serverSide": true,
@@ -123,9 +119,9 @@ function initializeDataTable() {
             { "data": "id", "visible": false, "searchable": false },
         ]
     });
+}
 
 function setupClickHandlers() {
-    handleInferenceButton();
     handleAddNewPatternClick();
 
     handleCreateDocumentTypePatternButton();
@@ -136,7 +132,7 @@ function setupClickHandlers() {
     handleDeleteExcludePatternButtonClick();
     handleDeleteTitlePatternButtonClick();
 
-    handleDocumentTypeSelect();
+    handleDocumentTypeSelect()
     handleExcludeIndividualUrlClick();
     handleNewTitleChange();
 
@@ -193,11 +189,10 @@ function getDocumentTypeColumn() {
                 3: 'Documentation',
                 4: 'Software and Tools',
                 5: 'Missions and Instruments',
-                6: 'Training and Education'
+                6: 'Training and Education',
             };
-            var inferenceValue = row['inferenced_by'];
             button_text = data ? dict[data] : 'Select';
-            button_color = inferenceValue === 'user' ? 'btn-success' : (inferenceValue === 'model' ? 'btn-primary' : 'btn-secondary');
+            button_color = data ? 'btn-success' : 'btn-secondary';
             return `
             <div class="dropdown document_type_dropdown" data-match-pattern=${remove_protocol(row['url'])}>
               <button class="btn ${button_color} btn-sm dropdown-toggle" type="button" id="dropdownMenuButton" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
@@ -216,7 +211,6 @@ function getDocumentTypeColumn() {
         }
     }
 }
-
 function handleCreateDocumentTypePatternButton() {
     $("body").on("click", ".create_document_type_pattern_button", function () {
         $modal = $('#documentTypePatternModal').modal();
@@ -235,18 +229,10 @@ function handleCreateTitlePatternButton() {
     });
 }
 
-
-//handling action when inference button is hit
-function handleInferenceButton() {
-    $("body").on("click", ".create_model_button", function (e) {
-        e.preventDefault(); // Prevent the default button behavior
-        postInferenceButton(collection_id);
-});}
-
 function handleDocumentTypeSelect() {
     $("body").on("click", ".document_type_select", function () {
         $match_pattern = $(this).parents(".document_type_dropdown").data('match-pattern');
-        postDocumentTypePatterns($match_pattern, match_pattern_type = 1, document_type = $(this).attr("value"),inferencer="user");
+        postDocumentTypePatterns($match_pattern, match_pattern_type = 1, document_type = $(this).attr("value"));
     });
 }
 
@@ -313,11 +299,12 @@ function handleUrlLinkClick() {
     });
 }
 
-function postDocumentTypePatterns(match_pattern, match_pattern_type, document_type,inferencer) {
+function postDocumentTypePatterns(match_pattern, match_pattern_type, document_type) {
     if (!match_pattern) {
         toastr.error('Please highlight a pattern to add document type.');
         return;
     }
+
     $.ajax({
         url: '/api/document-type-patterns/',
         type: "POST",
@@ -325,8 +312,7 @@ function postDocumentTypePatterns(match_pattern, match_pattern_type, document_ty
             collection: collection_id,
             match_pattern: match_pattern,
             match_pattern_type: match_pattern_type,
-            document_type: Number(document_type),
-            inferencer:inferencer,
+            document_type: document_type,
             csrfmiddlewaretoken: csrftoken
         },
         success: function (data) {
@@ -337,28 +323,7 @@ function postDocumentTypePatterns(match_pattern, match_pattern_type, document_ty
             var errorMessage = xhr.responseText;
             toastr.error(errorMessage);
         }
-    });}
-
-// runs the script to get predictions for given list of uninferenced urls
-function postInferenceButton(collection_id){
-    $.blockUI({ message: '<div id="loadingMessage"><h3><b>Model Inference taking place...</b></h3></div>'});
-    $.ajax({
-    url: '/api/model_inference',
-    type: 'POST',
-    data: {
-        csrfmiddlewaretoken: csrftoken,collection_id
-    },
-    success: function() {
-        $.unblockUI();
-        location.reload();
-        // Unblock the UI after the AJAX call is complete
-    },
-    error: function(xhr, status, error) {
-        var errorMessage = xhr.responseText;
-        toastr.error(errorMessage);
-        console.log("I am in some error")
-    }
-});
+    });
 }
 
 
@@ -463,7 +428,6 @@ function getCollectionId() {
     return collection_id;
 }
 
-
 function getParameterByName(name, url) {
     if (!url) url = window.location.href;
     name = name.replace(/[\[\]]/g, "\\$&");
@@ -599,13 +563,12 @@ $('.document_type_form_select').on('click', function (e) {
         inputs[field.name] = field.value;
     });
 
-    postDocumentTypePatterns(inputs.match_pattern, 2, inputs.document_type_pattern,inferencer="user");
+    postDocumentTypePatterns(inputs.match_pattern, 2, inputs.document_type_pattern);
 
     // close the modal if it is open
     $('#documentTypePatternModal').modal('hide');
 });
 
-
 $('#filter-checkbox').on('change', function () {
     $('#candidate_urls_table').DataTable().ajax.reload();
 });
diff --git a/sde_indexing_helper/templates/sde_collections/candidate_urls_list.html b/sde_indexing_helper/templates/sde_collections/candidate_urls_list.html
index 8329dfd0..041d8e65 100644
--- a/sde_indexing_helper/templates/sde_collections/candidate_urls_list.html
+++ b/sde_indexing_helper/templates/sde_collections/candidate_urls_list.html
@@ -27,13 +27,6 @@ <h3>
         <br>
         <small class="muted">Base URL: <a href="{{ collection.url }}" target="_blank">{{ collection.url }}</a></small>
     </h3>
-    <div class="row">
-        <button style="background-color: #4CAF50; text-align: center;">
-            <a href="{% url 'sde_collections:health-check' collection.pk %}" style="color:white;">Health Check</button>
-        </button>
-    </div>
-    
-
     <div class="row pull-right">
         <div class="form-group">
             <input type="checkbox" id="filter-checkbox">
@@ -41,20 +34,7 @@ <h3>
         </div>
     </div>
     <br>
-    <style>
-        .right-align {
-            text-align: right;
-        }
-    </style>
     <table class="table" id="candidate_urls_table">
-        <div class="right-align" style="margin-top: 50px";>
-            <h3><small class="muted">Document Type Inference</small></h3>
-        </div>
-        <div class="right-align">
-            User Inference:<span style="background-color: green; color: green; width: 200px; padding: 0px; margin-left: 25px; border-radius: 0px; font-size: 11px;">Press here for </span>
-        </div>
-        <div class="right-align">
-            Model's Inference:<button type="submit" id="create_model_button" class="create_model_button" style="background-color: purple; color: white; width: 70px; border-radius: 0px; margin-left: 4px; font-size: 11px; height: 20px;"><strong>Press here</strong></button>
         <thead>
             <tr>
                 <th scope="col" class="text-center col-1">URL</th>
@@ -285,7 +265,6 @@ <h5 class="modal-title" id="documentTypePatternModalLabel">Document Type Pattern
                                         <a class="dropdown-item document_type_form_select" value="3">Documentation</a>
                                         <a class="dropdown-item document_type_form_select" value="4">Software and Tools</a>
                                         <a class="dropdown-item document_type_form_select" value="5">Missions and Instruments</a>
-                                        <a class="dropdown-item document_type_form_select" value="6">Training and Education</a>
                                     </div>
                                 </div>
                             </div>

From 32eb1c89c9b1f075fde52c24cdc2f1846dc6450d Mon Sep 17 00:00:00 2001
From: Ashish Acharya <ashish.acharya14@gmail.com>
Date: Tue, 17 Oct 2023 11:57:05 -0500
Subject: [PATCH 05/10] Make gitignore consistent

---
 .gitignore | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index f601cebf..74a95675 100644
--- a/.gitignore
+++ b/.gitignore
@@ -286,13 +286,11 @@ sde_indexing_helper/media/
 **/.ipynb_checkpoints/
 **/*.xlsx
 
-
-# config details for the api access
+# Config details for the api access
 config_generation/config.py
 
-
-#model's inference files
+# Model's inference files
 Document_Classifier_inference/model.pt
 
-#Database Backup
+# Database backup
 backup.json

From 82f2911dcafba9234af94463d12fd276996080d5 Mon Sep 17 00:00:00 2001
From: Ashish Acharya <ashish.acharya14@gmail.com>
Date: Tue, 17 Oct 2023 11:58:37 -0500
Subject: [PATCH 06/10] Improve document classifier README

---
 Document_Classifier_inference/README.md | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/Document_Classifier_inference/README.md b/Document_Classifier_inference/README.md
index 601e34f9..fefa838b 100644
--- a/Document_Classifier_inference/README.md
+++ b/Document_Classifier_inference/README.md
@@ -1,18 +1,18 @@
 # Automated Document Tagging
 
-
 # Project Description:
-This purpose of this tag the content of a given url onto one of the five classes "Image","Documentation","Software and Tools",
-"Mission and Instruments", and "Data".
 
-#Datasets:
-Reference link for datasets: https://docs.google.com/spreadsheets/d/1rK7hvb_HRd-sqL3jrSYll5BiDvwnzQY2qVWDmpg6Bbk/edit#gid=1560325588
+This purpose of this is to tag the content of a given url onto one of the six classes "Image","Documentation","Software and Tools",
+"Mission and Instruments", "Training and Education", and "Data".
 
-# to run the repository:
-* location for saved model in drive: https://drive.google.com/drive/u/1/folders/1jkJSpN3ZuXhZIis4dSc-v0LkSV3pMrcs
-* saved weight_name: model.pt
-* prediction sample:python3 main.py predicts --config_file config.json --url "url_link"
+# Datasets:
+
+Reference link for datasets: https://docs.google.com/spreadsheets/d/1rK7hvb_HRd-sqL3jrSYll5BiDvwnzQY2qVWDmpg6Bbk/edit#gid=1560325588
 
+# To run the inference pipeline:
 
+- location for saved model in drive: https://drive.google.com/drive/u/1/folders/1jkJSpN3ZuXhZIis4dSc-v0LkSV3pMrcs
+- saved weight_name: model.pt
+- prediction sample: `python3 main.py predicts --config_file config.json --url "url_link"`
 
 For more details: contact rd0081@uah.edu

From 561eddf4247f2ef9b6050b6149971514c435bff9 Mon Sep 17 00:00:00 2001
From: Ashish Acharya <ashish.acharya14@gmail.com>
Date: Tue, 17 Oct 2023 12:03:53 -0500
Subject: [PATCH 07/10] Remove unused URLs

---
 sde_collections/urls.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/sde_collections/urls.py b/sde_collections/urls.py
index 13c1fbea..81f9da48 100644
--- a/sde_collections/urls.py
+++ b/sde_collections/urls.py
@@ -33,11 +33,6 @@
         PushToGithubView.as_view(),
         name="push-to-github",
     ),
-    path(
-        "api/health-check/<int:pk>",
-        view=HealthCheckView.as_view(),
-        name="health-check"
-    ),
     path(
         "delete-required-url/<int:pk>",
         view=RequiredUrlsDeleteView.as_view(),
@@ -54,5 +49,4 @@
     # Update an existing CandidateURL instance: /candidate-urls/{id}/
     # Delete an existing CandidateURL instance: /candidate-urls/{id}/
     path("api/", include(router.urls)),
-    path("api/model_inference", views.model_inference, name="model_inference"),
 ]

From 27214aa77092b99149183be8b19fd7b2bfbaa7e5 Mon Sep 17 00:00:00 2001
From: Ashish Acharya <ashish.acharya14@gmail.com>
Date: Tue, 17 Oct 2023 12:04:00 -0500
Subject: [PATCH 08/10] Remove unused serializers

---
 sde_collections/serializers.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/sde_collections/serializers.py b/sde_collections/serializers.py
index 745fcc52..8d4805e1 100644
--- a/sde_collections/serializers.py
+++ b/sde_collections/serializers.py
@@ -40,8 +40,6 @@ class CandidateURLSerializer(serializers.ModelSerializer):
     generated_title_id = serializers.SerializerMethodField(read_only=True)
     match_pattern_type = serializers.SerializerMethodField(read_only=True)
     candidate_urls_count = serializers.SerializerMethodField(read_only=True)
-    inferenced_by = serializers.CharField(read_only=True)
-    is_pdf = serializers.BooleanField(required=False)
 
     def get_candidate_urls_count(self, obj):
         titlepattern = obj.titlepattern_urls.last()
@@ -69,8 +67,6 @@ class Meta:
             "document_type",
             "document_type_display",
             "visited",
-            "inferenced_by",
-            "is_pdf",
         )
 
 

From d4f2c4adaadc3f4bee741b98e2d2a1be1f7263d5 Mon Sep 17 00:00:00 2001
From: Ashish Acharya <ashish.acharya14@gmail.com>
Date: Tue, 17 Oct 2023 12:11:15 -0500
Subject: [PATCH 09/10] Revert all changes from recent merges in views

---
 sde_collections/urls.py  |   1 -
 sde_collections/views.py | 116 +++++----------------------------------
 2 files changed, 13 insertions(+), 104 deletions(-)

diff --git a/sde_collections/urls.py b/sde_collections/urls.py
index 81f9da48..831ad842 100644
--- a/sde_collections/urls.py
+++ b/sde_collections/urls.py
@@ -13,7 +13,6 @@
     PushToGithubView,
     RequiredUrlsDeleteView,
     TitlePatternViewSet,
-    HealthCheckView,
 )
 
 router = routers.DefaultRouter()
diff --git a/sde_collections/views.py b/sde_collections/views.py
index a1d69ac9..c38b81cb 100644
--- a/sde_collections/views.py
+++ b/sde_collections/views.py
@@ -1,15 +1,11 @@
-import csv
 import re
-from io import StringIO
 
 from django.contrib.auth import get_user_model
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.db import models
-from django.http import HttpResponse
 from django.shortcuts import redirect
 from django.urls import reverse
 from django.utils import timezone
-from django.views import View
 from django.views.generic.detail import DetailView
 from django.views.generic.edit import DeleteView
 from django.views.generic.list import ListView
@@ -17,15 +13,10 @@
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from Document_Classifier_inference.main import batch_predicts
-
 from .forms import CollectionGithubIssueForm, RequiredUrlForm
 from .models.candidate_url import CandidateURL
 from .models.collection import Collection, RequiredUrls
-from .models.collection_choice_fields import (
-    CurationStatusChoices,
-    WorkflowStatusChoices,
-)
+from .models.collection_choice_fields import CurationStatusChoices
 from .models.pattern import DocumentTypePattern, ExcludePattern, TitlePattern
 from .serializers import (
     CandidateURLBulkCreateSerializer,
@@ -36,46 +27,10 @@
     TitlePatternSerializer,
 )
 from .tasks import push_to_github_task
-from .utils.health_check import health_check
 
 User = get_user_model()
 
 
-def model_inference(request):
-    if request.method == "POST":
-        collection_id = request.POST.get("collection_id")
-        candidate_urls = CandidateURL.objects.filter(
-            collection_id=Collection.objects.get(pk=collection_id),
-        ).exclude(document_type__in=[1, 2, 3, 4, 5, 6])
-        # These list of urls are to be inferred
-        to_infer_url_list = [candidate_url.url for candidate_url in candidate_urls]
-        if to_infer_url_list:
-            collection_id = candidate_urls[0].collection_id
-            prediction, pdf_lists = batch_predicts(
-                "Document_Classifier_inference/config.json", to_infer_url_list
-            )
-            # Update document_type for corresponding URLs
-            for candidate_url in candidate_urls:
-                new_document_type = prediction.get(candidate_url.url)
-                if new_document_type is not None:
-                    candidate_url.document_type = new_document_type
-                    candidate_url.inferenced_by = "model"
-                    candidate_url.save()  # Updating the changes in candidateurl table
-                    # Create a new DocumentTypePattern entry for each URL and its document_type
-                    DocumentTypePattern.objects.create(
-                        collection_id=candidate_url.collection_id,
-                        match_pattern=candidate_url.url.replace("https://", ""),
-                        match_pattern_type=DocumentTypePattern.MatchPatternTypeChoices.INDIVIDUAL_URL,
-                        document_type=new_document_type,
-                    )  # Adding the new record in documenttypepattern table
-                if (
-                    candidate_url.url in pdf_lists
-                ):  # flagging created for url with pdf response
-                    candidate_url.is_pdf = True
-                    candidate_url.save()
-        return HttpResponse(status=204)
-
-
 class CollectionListView(LoginRequiredMixin, ListView):
     """
     Display a list of collections in the system
@@ -99,7 +54,6 @@ def get_context_data(self, **kwargs):
         context["segment"] = "collections"
         context["curators"] = User.objects.filter(groups__name="Curators")
         context["curation_status_choices"] = CurationStatusChoices
-        context["workflow_status_choices"] = WorkflowStatusChoices
 
         return context
 
@@ -140,7 +94,6 @@ def post(self, request, *args, **kwargs):
             if "claim_button" in request.POST:
                 user = self.request.user
                 collection.curation_status = CurationStatusChoices.BEING_CURATED
-                collection.workflow_status = WorkflowStatusChoices.CURATION_IN_PROGRESS
                 collection.curated_by = user
                 collection.curation_started = timezone.now()
                 collection.save()
@@ -310,28 +263,20 @@ def get_queryset(self):
 
     def create(self, request, *args, **kwargs):
         document_type = request.POST.get("document_type")
-        inferencer = request.POST.get("inferencer")
-        collection_id = request.POST.get("collection")
-        match_pattern = request.POST.get("match_pattern")
-        candidate_url = CandidateURL.objects.get(
-            collection_id=Collection.objects.get(id=collection_id),
-            url="https://" + match_pattern,
-        )
         if not int(document_type) == 0:  # 0=none
-            candidate_url.inferenced_by = inferencer
-            candidate_url.save()
             return super().create(request, *args, **kwargs)
-        try:
-            candidate_url.inferenced_by = ""
-            candidate_url.save()
-            DocumentTypePattern.objects.get(
-                collection_id=Collection.objects.get(id=collection_id),
-                match_pattern=match_pattern,
-                match_pattern_type=DocumentTypePattern.MatchPatternTypeChoices.INDIVIDUAL_URL,
-            ).delete()
-            return Response(status=status.HTTP_204_NO_CONTENT)
-        except DocumentTypePattern.DoesNotExist:
-            return Response(status=status.HTTP_204_NO_CONTENT)
+        else:
+            collection_id = request.POST.get("collection")
+            match_pattern = request.POST.get("match_pattern")
+            try:
+                DocumentTypePattern.objects.get(
+                    collection_id=Collection.objects.get(id=collection_id),
+                    match_pattern=match_pattern,
+                    match_pattern_type=DocumentTypePattern.MatchPatternTypeChoices.INDIVIDUAL_URL,
+                ).delete()
+                return Response(status=status.HTTP_200_OK)
+            except DocumentTypePattern.DoesNotExist:
+                return Response(status=status.HTTP_204_NO_CONTENT)
 
 
 class CollectionViewSet(viewsets.ModelViewSet):
@@ -353,38 +298,3 @@ def post(self, request):
             {"Success": "Started pushing collections to github"},
             status=status.HTTP_200_OK,
         )
-
-
-class HealthCheckView(View):
-    """
-    This view checks whether the rules in indexer db has been correctly reflected
-    in our prod/test sinequa instances or not and at the end generates a report.
-    """
-
-    def get(self, *args, **kwargs):
-        collection = Collection.objects.get(pk=kwargs.get("pk"))
-        sync_check_report = health_check(collection, server_name="production")
-        field_names = [
-            "id",
-            "collection_name",
-            "config_folder",
-            "curation_status",
-            "workflow_status",
-            "pattern_name",
-            "pattern",
-            "scraped_title",
-            "non_compliant_url",
-        ]
-
-        # download the report in CSV format
-        csv_data = StringIO()
-        writer = csv.DictWriter(csv_data, fieldnames=field_names)
-        writer.writeheader()
-        for item in sync_check_report:
-            writer.writerow(item)
-
-        http_response = HttpResponse(content_type="text/csv")
-        http_response["Content-Disposition"] = 'attachment; filename="report.csv"'
-        http_response.write(csv_data.getvalue())
-
-        return http_response

From cfa988b708b48363f0a7e98c2b5ee37c8ef033e4 Mon Sep 17 00:00:00 2001
From: Ashish Acharya <ashish.acharya14@gmail.com>
Date: Tue, 17 Oct 2023 12:17:33 -0500
Subject: [PATCH 10/10] Update views to include recent code

---
 sde_collections/views.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sde_collections/views.py b/sde_collections/views.py
index c38b81cb..de68c7eb 100644
--- a/sde_collections/views.py
+++ b/sde_collections/views.py
@@ -16,7 +16,7 @@
 from .forms import CollectionGithubIssueForm, RequiredUrlForm
 from .models.candidate_url import CandidateURL
 from .models.collection import Collection, RequiredUrls
-from .models.collection_choice_fields import CurationStatusChoices
+from .models.collection_choice_fields import CurationStatusChoices, WorkflowStatusChoices
 from .models.pattern import DocumentTypePattern, ExcludePattern, TitlePattern
 from .serializers import (
     CandidateURLBulkCreateSerializer,
@@ -54,6 +54,7 @@ def get_context_data(self, **kwargs):
         context["segment"] = "collections"
         context["curators"] = User.objects.filter(groups__name="Curators")
         context["curation_status_choices"] = CurationStatusChoices
+        context["workflow_status_choices"] = WorkflowStatusChoices
 
         return context
 
@@ -93,7 +94,7 @@ def post(self, request, *args, **kwargs):
         else:
             if "claim_button" in request.POST:
                 user = self.request.user
-                collection.curation_status = CurationStatusChoices.BEING_CURATED
+                collection.curation_status = WorkflowStatusChoices.CURATION_IN_PROGRESS
                 collection.curated_by = user
                 collection.curation_started = timezone.now()
                 collection.save()