Rename ext/elasticsearch to opensearch

Rename ext/elasticsearch to ext/opensearch and replace code imports and references to ext.elasticsearch.

fixes/add_histfactory_analyses.py

@@ -9,7 +9,7 @@
 from hepdata.celery import dynamic_tasks
 from hepdata.config import HISTFACTORY_FILE_TYPE
 from hepdata.cli import fix
-from hepdata.ext.elasticsearch.api import reindex_batch
+from hepdata.ext.opensearch.api import reindex_batch
 from hepdata.modules.records.utils.common import is_histfactory
 from hepdata.modules.submission.api import get_latest_hepsubmission
 from hepdata.modules.submission.models import HEPSubmission

fixes/cleanup_index.py

@@ -9,7 +9,7 @@
 from sqlalchemy.orm import aliased
 
 from hepdata.cli import fix
-from hepdata.ext.elasticsearch.api import default_index
+from hepdata.ext.opensearch.api import default_index
 from hepdata.modules.submission.models import HEPSubmission, DataSubmission
 
 logging.basicConfig()

fixes/data_submission_recids.py

@@ -31,7 +31,7 @@
 from sqlalchemy import func
 
 from hepdata.cli import fix
-from hepdata.ext.elasticsearch.api import index_record_ids, push_data_keywords
+from hepdata.ext.opensearch.api import index_record_ids, push_data_keywords
 from hepdata.modules.records.utils.common import get_record_by_id
 from hepdata.modules.records.utils.doi_minter import generate_dois_for_submission
 from hepdata.modules.records.utils.submission import finalise_datasubmission

fixes/missing_record_ids.py

@@ -5,7 +5,7 @@
 from invenio_db import db
 
 from hepdata.cli import fix
-from hepdata.ext.elasticsearch.api import index_record_ids, push_data_keywords
+from hepdata.ext.opensearch.api import index_record_ids, push_data_keywords
 from hepdata.modules.submission.models import HEPSubmission, DataSubmission
 from hepdata.modules.records.utils.common import get_record_by_id
 from hepdata.modules.records.utils.doi_minter import generate_doi_for_table

hepdata/cli.py

@@ -29,14 +29,14 @@
 from flask import current_app
 from flask.cli import with_appcontext
 from invenio_base.app import create_cli
-from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer
+from hepdata.ext.opensearch.admin_view.api import AdminIndexer
 from hepdata.modules.converter.tasks import convert_and_store
 from hepdata.modules.records.utils.common import record_exists, get_record_by_id
 from hepdata.modules.submission.models import HEPSubmission
 from hepdata.modules.submission.api import get_latest_hepsubmission
 from .factory import create_app
 from hepdata.config import CFG_PUB_TYPE
-from hepdata.ext.elasticsearch.api import reindex_all, get_records_matching_field
+from hepdata.ext.opensearch.api import reindex_all, get_records_matching_field
 from hepdata.modules.records.importer import api as importer_api
 from hepdata.modules.records.utils import data_files
 from hepdata.modules.records.utils.analyses import update_analyses
@@ -84,7 +84,7 @@ def import_records(inspireids, recreate_index, base_url, update_existing, send_e
 
     Usage: ``hepdata importer import-records -i 'ins1262703' -rc False``
     """
-    from hepdata.ext.elasticsearch.api import recreate_index as reindex
+    from hepdata.ext.opensearch.api import recreate_index as reindex
     if current_app.config.get('ENV') == 'production':
         click.confirm('You are currently running in production mode on'
                       ' %s. Are you sure you want to import records from %s?'

hepdata/ext/elasticsearch/api.py → hepdata/ext/opensearch/api.py

@@ -31,7 +31,7 @@
 from sqlalchemy.orm import aliased
 
 
-from hepdata.ext.elasticsearch.document_enhancers import enhance_data_document, enhance_publication_document
+from hepdata.ext.opensearch.document_enhancers import enhance_data_document, enhance_publication_document
 from .config.es_config import sort_fields_mapping, add_default_aggregations
 from .utils import calculate_sort_order, prepare_author_for_indexing
 from hepdata.config import CFG_PUB_TYPE, CFG_DATA_TYPE

hepdata/ext/elasticsearch/document_enhancers.py → hepdata/ext/opensearch/document_enhancers.py

@@ -30,7 +30,7 @@
 from flask import current_app
 
 from hepdata.config import CFG_PUB_TYPE, CFG_DATA_TYPE, HISTFACTORY_FILE_TYPE
-from hepdata.ext.elasticsearch.config.record_mapping import mapping as es_mapping
+from hepdata.ext.opensearch.config.record_mapping import mapping as es_mapping
 from hepdata.modules.permissions.models import SubmissionParticipant
 from hepdata.modules.submission.api import get_latest_hepsubmission
 

hepdata/ext/elasticsearch/process_results.py → hepdata/ext/opensearch/process_results.py

@@ -111,7 +111,7 @@ def get_basic_record_information(record):
 
 
 def fetch_remaining_papers(tables, papers):
-    from hepdata.ext.elasticsearch.api import fetch_record
+    from hepdata.ext.opensearch.api import fetch_record
     hit_papers = list(map(lambda x: int(x['_id']), papers))
     for table in tables:
         paper_id = table['_source'].get('related_publication')

hepdata/modules/dashboard/api.py

@@ -32,7 +32,7 @@
 from sqlalchemy import or_, func
 from werkzeug.exceptions import Forbidden as ForbiddenError
 
-from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer
+from hepdata.ext.opensearch.admin_view.api import AdminIndexer
 from hepdata.modules.permissions.models import SubmissionParticipant
 from hepdata.modules.records.utils.common import get_record_by_id, decode_string
 from hepdata.modules.submission.api import get_latest_hepsubmission, get_submission_participants_for_record

hepdata/modules/dashboard/views.py

@@ -27,9 +27,9 @@
 from flask_login import login_required, current_user
 from invenio_accounts.models import User, Role
 
-from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer
-from hepdata.ext.elasticsearch.api import reindex_all
-from hepdata.ext.elasticsearch.api import push_data_keywords
+from hepdata.ext.opensearch.admin_view.api import AdminIndexer
+from hepdata.ext.opensearch.api import reindex_all
+from hepdata.ext.opensearch.api import push_data_keywords
 from hepdata.modules.dashboard.api import prepare_submissions, get_pending_invitations_for_user, get_submission_count, \
     list_submission_titles, get_dashboard_current_user, set_dashboard_current_user, get_submissions_summary, \
     get_submissions_csv

hepdata/modules/doi_banner/views.py

@@ -1,7 +1,7 @@
 """HEPData DOI Banner Views."""
 
 from flask import Blueprint, redirect, abort, send_file, url_for
-from hepdata.ext.elasticsearch.api import get_records_matching_field
+from hepdata.ext.opensearch.api import get_records_matching_field
 import logging
 import os
 

hepdata/modules/permissions/views.py

@@ -34,7 +34,7 @@
 
 from flask import Blueprint, jsonify, url_for, redirect, request, abort, render_template
 
-from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer
+from hepdata.ext.opensearch.admin_view.api import AdminIndexer
 from hepdata.modules.dashboard.api import get_dashboard_current_user
 from hepdata.modules.email.api import send_coordinator_request_mail, send_coordinator_approved_email, \
     send_cookie_email, send_reserve_email

hepdata/modules/records/utils/analyses.py

@@ -29,7 +29,7 @@
 from invenio_db import db
 import requests
 
-from hepdata.ext.elasticsearch.api import index_record_ids
+from hepdata.ext.opensearch.api import index_record_ids
 from hepdata.modules.submission.api import get_latest_hepsubmission, is_resource_added_to_submission
 from hepdata.modules.submission.models import DataResource, HEPSubmission, data_reference_link
 

hepdata/modules/records/utils/common.py

@@ -29,7 +29,7 @@
 from sqlalchemy.orm.exc import NoResultFound
 
 from hepdata.config import CFG_PUB_TYPE, HISTFACTORY_FILE_TYPE
-from hepdata.ext.elasticsearch.api import get_record
+from hepdata.ext.opensearch.api import get_record
 from hepdata.modules.submission.models import HEPSubmission, License
 
 FILE_TYPES = {

hepdata/modules/records/utils/records_update_utils.py

@@ -9,7 +9,7 @@
 from hepdata.modules.records.utils.common import get_record_by_id
 from hepdata.modules.records.utils.workflow import update_record
 from hepdata.modules.inspire_api.views import get_inspire_record_information
-from hepdata.ext.elasticsearch.api import index_record_ids, push_data_keywords
+from hepdata.ext.opensearch.api import index_record_ids, push_data_keywords
 from hepdata.modules.email.api import notify_publication_update
 from hepdata.resilient_requests import resilient_requests
 from hepdata.config import TESTING

hepdata/modules/records/utils/submission.py

@@ -34,8 +34,8 @@
 from flask_login import current_user
 from hepdata_converter_ws_client import get_data_size
 from hepdata.config import CFG_DATA_TYPE, CFG_PUB_TYPE
-from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer
-from hepdata.ext.elasticsearch.api import get_records_matching_field, \
+from hepdata.ext.opensearch.admin_view.api import AdminIndexer
+from hepdata.ext.opensearch.api import get_records_matching_field, \
     delete_item_from_index, index_record_ids, push_data_keywords
 from hepdata.modules.converter import prepare_data_folder
 from hepdata.modules.converter.tasks import convert_and_store

hepdata/modules/records/views.py

@@ -39,7 +39,7 @@
 from yaml import CBaseLoader as Loader
 
 from hepdata.config import CFG_DATA_TYPE, CFG_PUB_TYPE, SITE_URL
-from hepdata.ext.elasticsearch.api import get_records_matching_field, get_count_for_collection, get_n_latest_records, \
+from hepdata.ext.opensearch.api import get_records_matching_field, get_count_for_collection, get_n_latest_records, \
     index_record_ids
 from hepdata.modules.email.api import send_notification_email, send_new_review_message_email, NoParticipantsException, \
     send_question_email, send_coordinator_notification_email

hepdata/modules/search/views.py

@@ -23,7 +23,7 @@
 
 from flask import Blueprint, request, render_template, jsonify
 from hepdata.config import CFG_DATA_KEYWORDS
-from hepdata.ext.elasticsearch.api import search as es_search, \
+from hepdata.ext.opensearch.api import search as es_search, \
     search_authors as es_search_authors, get_all_ids as es_get_all_ids
 from hepdata.modules.records.utils.common import decode_string
 from hepdata.modules.records.api import get_all_ids as db_get_all_ids

hepdata/modules/submission/views.py

@@ -21,7 +21,7 @@
 from flask_login import login_required, current_user
 from invenio_db import db
 
-from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer
+from hepdata.ext.opensearch.admin_view.api import AdminIndexer
 from hepdata.modules.email.api import send_cookie_email, notify_submission_created
 from hepdata.modules.inspire_api.views import get_inspire_record_information
 from hepdata.modules.permissions.models import SubmissionParticipant

setup.py

@@ -148,7 +148,7 @@ def run_tests(self):
             'hepdata_doi = hepdata.modules.records.utils.doi_minter',
             'hepdata_mail = hepdata.modules.email.utils',
             'hepdata_conversion = hepdata.modules.converter.tasks',
-            'hepdata_elasticsearch = hepdata.ext.elasticsearch.api',
+            'hepdata_opensearch = hepdata.ext.opensearch.api',
             'hepdata_inspireupdate = hepdata.modules.records.utils.records_update_utils',
         ],
         'invenio_i18n.translations': [

tests/aggregations_test.py

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with HEPData; if not, write to the Free Software Foundation, Inc.,
 # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-from hepdata.ext.elasticsearch.aggregations import parse_author_aggregations, \
+from hepdata.ext.opensearch.aggregations import parse_author_aggregations, \
     parse_date_aggregations, parse_collaboration_aggregations, \
     parse_other_facets, parse_aggregations, parse_cmenergies_aggregations, \
     create_dummy_cmenergies_facets

tests/conftest.py

@@ -31,8 +31,8 @@
 from invenio_db import db
 import pytest
 
-from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer
-from hepdata.ext.elasticsearch.api import reindex_all
+from hepdata.ext.opensearch.admin_view.api import AdminIndexer
+from hepdata.ext.opensearch.api import reindex_all
 from hepdata.factory import create_app
 from hepdata.modules.records.importer.api import import_records, _download_file
 

tests/e2e/conftest.py

@@ -44,7 +44,7 @@
 from time import sleep
 
 from hepdata.config import CFG_TMPDIR, RUN_SELENIUM_LOCALLY
-from hepdata.ext.elasticsearch.api import reindex_all
+from hepdata.ext.opensearch.api import reindex_all
 from hepdata.factory import create_app
 from tests.conftest import get_identifiers, import_default_data
 

tests/e2e/test_general.py

@@ -36,7 +36,7 @@
 from functools import reduce
 from tests.conftest import import_default_data
 
-from hepdata.ext.elasticsearch.api import reindex_all
+from hepdata.ext.opensearch.api import reindex_all
 from hepdata.modules.records.importer.api import import_records
 from hepdata.modules.submission.api import get_latest_hepsubmission
 from hepdata.modules.records.utils.submission import unload_submission, get_or_create_hepsubmission

tests/search_test.py

@@ -22,15 +22,15 @@
 from invenio_db import db
 from unittest.mock import call
 
-from hepdata.ext.elasticsearch.config.es_config import \
+from hepdata.ext.opensearch.config.es_config import \
     add_default_aggregations, sort_fields_mapping
-from hepdata.ext.elasticsearch import api as es_api
-from hepdata.ext.elasticsearch.config.es_config import get_filter_field
-from hepdata.ext.elasticsearch.document_enhancers import add_data_keywords, process_cmenergies
-from hepdata.ext.elasticsearch.process_results import merge_results, match_tables_to_papers, \
+from hepdata.ext.opensearch import api as es_api
+from hepdata.ext.opensearch.config.es_config import get_filter_field
+from hepdata.ext.opensearch.document_enhancers import add_data_keywords, process_cmenergies
+from hepdata.ext.opensearch.process_results import merge_results, match_tables_to_papers, \
     get_basic_record_information, is_datatable
-from hepdata.ext.elasticsearch.query_builder import QueryBuilder, HEPDataQueryParser
-from hepdata.ext.elasticsearch.utils import flip_sort_order, parse_and_format_date, prepare_author_for_indexing, \
+from hepdata.ext.opensearch.query_builder import QueryBuilder, HEPDataQueryParser
+from hepdata.ext.opensearch.utils import flip_sort_order, parse_and_format_date, prepare_author_for_indexing, \
     calculate_sort_order, push_keywords
 from hepdata.modules.records.importer.api import import_records
 from hepdata.modules.submission.models import HEPSubmission
@@ -477,7 +477,7 @@ def test_reindex_all(app, load_default_data, identifiers, mocker):
     es_api.reindex_all(index=index, recreate=False, update_mapping=True, synchronous=True)
 
     # Test other params using mocking
-    m = mocker.patch('hepdata.ext.elasticsearch.api.reindex_batch')
+    m = mocker.patch('hepdata.ext.opensearch.api.reindex_batch')
 
     # Start and end at publication_recid 1, batch size 2:
     # should call reindex_batch twice with submission ids [1] then [2]
@@ -550,8 +550,8 @@ def test_reindex_batch(app, load_default_data, mocker):
     index = app.config.get('OPENSEARCH_INDEX')
 
     # Mock methods called so we can check they're called with correct parameters
-    mock_index_record_ids = mocker.patch('hepdata.ext.elasticsearch.api.index_record_ids')
-    mock_push_data_keywords = mocker.patch('hepdata.ext.elasticsearch.api.push_data_keywords')
+    mock_index_record_ids = mocker.patch('hepdata.ext.opensearch.api.index_record_ids')
+    mock_push_data_keywords = mocker.patch('hepdata.ext.opensearch.api.push_data_keywords')
 
     # Reindex submission id 1 (pub_recid=1, with data submissions 2-15)
     mock_index_record_ids.return_value = {'publication': [1], 'datatable': list(range(2,16))}
@@ -582,7 +582,7 @@ def test_update_record_mapping(app, mocker):
 
     # mapping should be as defined in record_mapping
     mapping = index.get_mapping(using=es)
-    from hepdata.ext.elasticsearch.config.record_mapping import mapping as real_mapping
+    from hepdata.ext.opensearch.config.record_mapping import mapping as real_mapping
     assert 'properties' in mapping['mock_index']['mappings']
     for k in real_mapping.keys():
         assert k in mapping['mock_index']['mappings']['properties']

tests/submission_test.py

@@ -31,7 +31,7 @@
 from invenio_db import db
 import pytest
 
-from hepdata.ext.elasticsearch.api import get_records_matching_field
+from hepdata.ext.opensearch.api import get_records_matching_field
 from hepdata.modules.permissions.models import SubmissionParticipant
 from hepdata.modules.records.api import format_submission, process_saved_file, create_new_version
 from hepdata.modules.records.utils.common import infer_file_type, contains_accepted_url, allowed_file, record_exists, \