diff --git a/fixes/add_histfactory_analyses.py b/fixes/add_histfactory_analyses.py index 72ceb0bd..06c88b4e 100644 --- a/fixes/add_histfactory_analyses.py +++ b/fixes/add_histfactory_analyses.py @@ -9,7 +9,7 @@ from hepdata.celery import dynamic_tasks from hepdata.config import HISTFACTORY_FILE_TYPE from hepdata.cli import fix -from hepdata.ext.elasticsearch.api import reindex_batch +from hepdata.ext.opensearch.api import reindex_batch from hepdata.modules.records.utils.common import is_histfactory from hepdata.modules.submission.api import get_latest_hepsubmission from hepdata.modules.submission.models import HEPSubmission diff --git a/fixes/cleanup_index.py b/fixes/cleanup_index.py index afd7ca79..4860c564 100644 --- a/fixes/cleanup_index.py +++ b/fixes/cleanup_index.py @@ -9,7 +9,7 @@ from sqlalchemy.orm import aliased from hepdata.cli import fix -from hepdata.ext.elasticsearch.api import default_index +from hepdata.ext.opensearch.api import default_index from hepdata.modules.submission.models import HEPSubmission, DataSubmission logging.basicConfig() diff --git a/fixes/data_submission_recids.py b/fixes/data_submission_recids.py index 3435770a..2b7ac0ef 100644 --- a/fixes/data_submission_recids.py +++ b/fixes/data_submission_recids.py @@ -31,7 +31,7 @@ from sqlalchemy import func from hepdata.cli import fix -from hepdata.ext.elasticsearch.api import index_record_ids, push_data_keywords +from hepdata.ext.opensearch.api import index_record_ids, push_data_keywords from hepdata.modules.records.utils.common import get_record_by_id from hepdata.modules.records.utils.doi_minter import generate_dois_for_submission from hepdata.modules.records.utils.submission import finalise_datasubmission diff --git a/fixes/missing_record_ids.py b/fixes/missing_record_ids.py index 14a7dbe6..83cdbb0b 100644 --- a/fixes/missing_record_ids.py +++ b/fixes/missing_record_ids.py @@ -5,7 +5,7 @@ from invenio_db import db from hepdata.cli import fix -from hepdata.ext.elasticsearch.api import index_record_ids, push_data_keywords +from hepdata.ext.opensearch.api import index_record_ids, push_data_keywords from hepdata.modules.submission.models import HEPSubmission, DataSubmission from hepdata.modules.records.utils.common import get_record_by_id from hepdata.modules.records.utils.doi_minter import generate_doi_for_table diff --git a/hepdata/cli.py b/hepdata/cli.py index c4f25bbf..d85e0eff 100644 --- a/hepdata/cli.py +++ b/hepdata/cli.py @@ -29,14 +29,14 @@ from flask import current_app from flask.cli import with_appcontext from invenio_base.app import create_cli -from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer +from hepdata.ext.opensearch.admin_view.api import AdminIndexer from hepdata.modules.converter.tasks import convert_and_store from hepdata.modules.records.utils.common import record_exists, get_record_by_id from hepdata.modules.submission.models import HEPSubmission from hepdata.modules.submission.api import get_latest_hepsubmission from .factory import create_app from hepdata.config import CFG_PUB_TYPE -from hepdata.ext.elasticsearch.api import reindex_all, get_records_matching_field +from hepdata.ext.opensearch.api import reindex_all, get_records_matching_field from hepdata.modules.records.importer import api as importer_api from hepdata.modules.records.utils import data_files from hepdata.modules.records.utils.analyses import update_analyses @@ -84,7 +84,7 @@ def import_records(inspireids, recreate_index, base_url, update_existing, send_e Usage: ``hepdata importer import-records -i 'ins1262703' -rc False`` """ - from hepdata.ext.elasticsearch.api import recreate_index as reindex + from hepdata.ext.opensearch.api import recreate_index as reindex if current_app.config.get('ENV') == 'production': click.confirm('You are currently running in production mode on' ' %s. Are you sure you want to import records from %s?' diff --git a/hepdata/ext/elasticsearch/__init__.py b/hepdata/ext/opensearch/__init__.py similarity index 100% rename from hepdata/ext/elasticsearch/__init__.py rename to hepdata/ext/opensearch/__init__.py diff --git a/hepdata/ext/elasticsearch/admin_view/__init__.py b/hepdata/ext/opensearch/admin_view/__init__.py similarity index 100% rename from hepdata/ext/elasticsearch/admin_view/__init__.py rename to hepdata/ext/opensearch/admin_view/__init__.py diff --git a/hepdata/ext/elasticsearch/admin_view/api.py b/hepdata/ext/opensearch/admin_view/api.py similarity index 100% rename from hepdata/ext/elasticsearch/admin_view/api.py rename to hepdata/ext/opensearch/admin_view/api.py diff --git a/hepdata/ext/elasticsearch/aggregations.py b/hepdata/ext/opensearch/aggregations.py similarity index 100% rename from hepdata/ext/elasticsearch/aggregations.py rename to hepdata/ext/opensearch/aggregations.py diff --git a/hepdata/ext/elasticsearch/api.py b/hepdata/ext/opensearch/api.py similarity index 99% rename from hepdata/ext/elasticsearch/api.py rename to hepdata/ext/opensearch/api.py index bce870ab..dd068d5a 100644 --- a/hepdata/ext/elasticsearch/api.py +++ b/hepdata/ext/opensearch/api.py @@ -31,7 +31,7 @@ from sqlalchemy.orm import aliased -from hepdata.ext.elasticsearch.document_enhancers import enhance_data_document, enhance_publication_document +from hepdata.ext.opensearch.document_enhancers import enhance_data_document, enhance_publication_document from .config.es_config import sort_fields_mapping, add_default_aggregations from .utils import calculate_sort_order, prepare_author_for_indexing from hepdata.config import CFG_PUB_TYPE, CFG_DATA_TYPE diff --git a/hepdata/ext/elasticsearch/config/__init__.py b/hepdata/ext/opensearch/config/__init__.py similarity index 100% rename from hepdata/ext/elasticsearch/config/__init__.py rename to hepdata/ext/opensearch/config/__init__.py diff --git a/hepdata/ext/elasticsearch/config/es_config.py b/hepdata/ext/opensearch/config/es_config.py similarity index 100% rename from hepdata/ext/elasticsearch/config/es_config.py rename to hepdata/ext/opensearch/config/es_config.py diff --git a/hepdata/ext/elasticsearch/config/record_mapping.py b/hepdata/ext/opensearch/config/record_mapping.py similarity index 100% rename from hepdata/ext/elasticsearch/config/record_mapping.py rename to hepdata/ext/opensearch/config/record_mapping.py diff --git a/hepdata/ext/elasticsearch/document_enhancers.py b/hepdata/ext/opensearch/document_enhancers.py similarity index 98% rename from hepdata/ext/elasticsearch/document_enhancers.py rename to hepdata/ext/opensearch/document_enhancers.py index a2d1b8fd..500f3aee 100644 --- a/hepdata/ext/elasticsearch/document_enhancers.py +++ b/hepdata/ext/opensearch/document_enhancers.py @@ -30,7 +30,7 @@ from flask import current_app from hepdata.config import CFG_PUB_TYPE, CFG_DATA_TYPE, HISTFACTORY_FILE_TYPE -from hepdata.ext.elasticsearch.config.record_mapping import mapping as es_mapping +from hepdata.ext.opensearch.config.record_mapping import mapping as es_mapping from hepdata.modules.permissions.models import SubmissionParticipant from hepdata.modules.submission.api import get_latest_hepsubmission diff --git a/hepdata/ext/elasticsearch/process_results.py b/hepdata/ext/opensearch/process_results.py similarity index 98% rename from hepdata/ext/elasticsearch/process_results.py rename to hepdata/ext/opensearch/process_results.py index 16ea3c2b..e5be6b5d 100644 --- a/hepdata/ext/elasticsearch/process_results.py +++ b/hepdata/ext/opensearch/process_results.py @@ -111,7 +111,7 @@ def get_basic_record_information(record): def fetch_remaining_papers(tables, papers): - from hepdata.ext.elasticsearch.api import fetch_record + from hepdata.ext.opensearch.api import fetch_record hit_papers = list(map(lambda x: int(x['_id']), papers)) for table in tables: paper_id = table['_source'].get('related_publication') diff --git a/hepdata/ext/elasticsearch/query_builder.py b/hepdata/ext/opensearch/query_builder.py similarity index 100% rename from hepdata/ext/elasticsearch/query_builder.py rename to hepdata/ext/opensearch/query_builder.py diff --git a/hepdata/ext/elasticsearch/utils.py b/hepdata/ext/opensearch/utils.py similarity index 100% rename from hepdata/ext/elasticsearch/utils.py rename to hepdata/ext/opensearch/utils.py diff --git a/hepdata/modules/dashboard/api.py b/hepdata/modules/dashboard/api.py index 9be5441d..e74cf27c 100644 --- a/hepdata/modules/dashboard/api.py +++ b/hepdata/modules/dashboard/api.py @@ -32,7 +32,7 @@ from sqlalchemy import or_, func from werkzeug.exceptions import Forbidden as ForbiddenError -from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer +from hepdata.ext.opensearch.admin_view.api import AdminIndexer from hepdata.modules.permissions.models import SubmissionParticipant from hepdata.modules.records.utils.common import get_record_by_id, decode_string from hepdata.modules.submission.api import get_latest_hepsubmission, get_submission_participants_for_record diff --git a/hepdata/modules/dashboard/views.py b/hepdata/modules/dashboard/views.py index b2af5b43..9cf3b0b3 100644 --- a/hepdata/modules/dashboard/views.py +++ b/hepdata/modules/dashboard/views.py @@ -27,9 +27,9 @@ from flask_login import login_required, current_user from invenio_accounts.models import User, Role -from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer -from hepdata.ext.elasticsearch.api import reindex_all -from hepdata.ext.elasticsearch.api import push_data_keywords +from hepdata.ext.opensearch.admin_view.api import AdminIndexer +from hepdata.ext.opensearch.api import reindex_all +from hepdata.ext.opensearch.api import push_data_keywords from hepdata.modules.dashboard.api import prepare_submissions, get_pending_invitations_for_user, get_submission_count, \ list_submission_titles, get_dashboard_current_user, set_dashboard_current_user, get_submissions_summary, \ get_submissions_csv diff --git a/hepdata/modules/doi_banner/views.py b/hepdata/modules/doi_banner/views.py index a5caebbc..41821611 100644 --- a/hepdata/modules/doi_banner/views.py +++ b/hepdata/modules/doi_banner/views.py @@ -1,7 +1,7 @@ """HEPData DOI Banner Views.""" from flask import Blueprint, redirect, abort, send_file, url_for -from hepdata.ext.elasticsearch.api import get_records_matching_field +from hepdata.ext.opensearch.api import get_records_matching_field import logging import os diff --git a/hepdata/modules/permissions/views.py b/hepdata/modules/permissions/views.py index 56aa267b..d23c9644 100644 --- a/hepdata/modules/permissions/views.py +++ b/hepdata/modules/permissions/views.py @@ -34,7 +34,7 @@ from flask import Blueprint, jsonify, url_for, redirect, request, abort, render_template -from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer +from hepdata.ext.opensearch.admin_view.api import AdminIndexer from hepdata.modules.dashboard.api import get_dashboard_current_user from hepdata.modules.email.api import send_coordinator_request_mail, send_coordinator_approved_email, \ send_cookie_email, send_reserve_email diff --git a/hepdata/modules/records/utils/analyses.py b/hepdata/modules/records/utils/analyses.py index b6c2f5f6..949b5351 100644 --- a/hepdata/modules/records/utils/analyses.py +++ b/hepdata/modules/records/utils/analyses.py @@ -29,7 +29,7 @@ from invenio_db import db import requests -from hepdata.ext.elasticsearch.api import index_record_ids +from hepdata.ext.opensearch.api import index_record_ids from hepdata.modules.submission.api import get_latest_hepsubmission, is_resource_added_to_submission from hepdata.modules.submission.models import DataResource, HEPSubmission, data_reference_link diff --git a/hepdata/modules/records/utils/common.py b/hepdata/modules/records/utils/common.py index e79970c5..2046caca 100644 --- a/hepdata/modules/records/utils/common.py +++ b/hepdata/modules/records/utils/common.py @@ -29,7 +29,7 @@ from sqlalchemy.orm.exc import NoResultFound from hepdata.config import CFG_PUB_TYPE, HISTFACTORY_FILE_TYPE -from hepdata.ext.elasticsearch.api import get_record +from hepdata.ext.opensearch.api import get_record from hepdata.modules.submission.models import HEPSubmission, License FILE_TYPES = { diff --git a/hepdata/modules/records/utils/records_update_utils.py b/hepdata/modules/records/utils/records_update_utils.py index 4ad8110a..dd498fea 100644 --- a/hepdata/modules/records/utils/records_update_utils.py +++ b/hepdata/modules/records/utils/records_update_utils.py @@ -9,7 +9,7 @@ from hepdata.modules.records.utils.common import get_record_by_id from hepdata.modules.records.utils.workflow import update_record from hepdata.modules.inspire_api.views import get_inspire_record_information -from hepdata.ext.elasticsearch.api import index_record_ids, push_data_keywords +from hepdata.ext.opensearch.api import index_record_ids, push_data_keywords from hepdata.modules.email.api import notify_publication_update from hepdata.resilient_requests import resilient_requests from hepdata.config import TESTING diff --git a/hepdata/modules/records/utils/submission.py b/hepdata/modules/records/utils/submission.py index 93e0a0b8..a81f3fd2 100644 --- a/hepdata/modules/records/utils/submission.py +++ b/hepdata/modules/records/utils/submission.py @@ -34,8 +34,8 @@ from flask_login import current_user from hepdata_converter_ws_client import get_data_size from hepdata.config import CFG_DATA_TYPE, CFG_PUB_TYPE -from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer -from hepdata.ext.elasticsearch.api import get_records_matching_field, \ +from hepdata.ext.opensearch.admin_view.api import AdminIndexer +from hepdata.ext.opensearch.api import get_records_matching_field, \ delete_item_from_index, index_record_ids, push_data_keywords from hepdata.modules.converter import prepare_data_folder from hepdata.modules.converter.tasks import convert_and_store diff --git a/hepdata/modules/records/views.py b/hepdata/modules/records/views.py index 87429de2..134d39e3 100644 --- a/hepdata/modules/records/views.py +++ b/hepdata/modules/records/views.py @@ -39,7 +39,7 @@ from yaml import CBaseLoader as Loader from hepdata.config import CFG_DATA_TYPE, CFG_PUB_TYPE, SITE_URL -from hepdata.ext.elasticsearch.api import get_records_matching_field, get_count_for_collection, get_n_latest_records, \ +from hepdata.ext.opensearch.api import get_records_matching_field, get_count_for_collection, get_n_latest_records, \ index_record_ids from hepdata.modules.email.api import send_notification_email, send_new_review_message_email, NoParticipantsException, \ send_question_email, send_coordinator_notification_email diff --git a/hepdata/modules/search/views.py b/hepdata/modules/search/views.py index f9f8ccac..f05fb5ca 100644 --- a/hepdata/modules/search/views.py +++ b/hepdata/modules/search/views.py @@ -23,7 +23,7 @@ from flask import Blueprint, request, render_template, jsonify from hepdata.config import CFG_DATA_KEYWORDS -from hepdata.ext.elasticsearch.api import search as es_search, \ +from hepdata.ext.opensearch.api import search as es_search, \ search_authors as es_search_authors, get_all_ids as es_get_all_ids from hepdata.modules.records.utils.common import decode_string from hepdata.modules.records.api import get_all_ids as db_get_all_ids diff --git a/hepdata/modules/submission/views.py b/hepdata/modules/submission/views.py index 86888f6c..18da1c61 100644 --- a/hepdata/modules/submission/views.py +++ b/hepdata/modules/submission/views.py @@ -21,7 +21,7 @@ from flask_login import login_required, current_user from invenio_db import db -from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer +from hepdata.ext.opensearch.admin_view.api import AdminIndexer from hepdata.modules.email.api import send_cookie_email, notify_submission_created from hepdata.modules.inspire_api.views import get_inspire_record_information from hepdata.modules.permissions.models import SubmissionParticipant diff --git a/setup.py b/setup.py index b262898f..18196a6c 100644 --- a/setup.py +++ b/setup.py @@ -148,7 +148,7 @@ def run_tests(self): 'hepdata_doi = hepdata.modules.records.utils.doi_minter', 'hepdata_mail = hepdata.modules.email.utils', 'hepdata_conversion = hepdata.modules.converter.tasks', - 'hepdata_elasticsearch = hepdata.ext.elasticsearch.api', + 'hepdata_opensearch = hepdata.ext.opensearch.api', 'hepdata_inspireupdate = hepdata.modules.records.utils.records_update_utils', ], 'invenio_i18n.translations': [ diff --git a/tests/aggregations_test.py b/tests/aggregations_test.py index 77450342..45bc0924 100644 --- a/tests/aggregations_test.py +++ b/tests/aggregations_test.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with HEPData; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -from hepdata.ext.elasticsearch.aggregations import parse_author_aggregations, \ +from hepdata.ext.opensearch.aggregations import parse_author_aggregations, \ parse_date_aggregations, parse_collaboration_aggregations, \ parse_other_facets, parse_aggregations, parse_cmenergies_aggregations, \ create_dummy_cmenergies_facets diff --git a/tests/conftest.py b/tests/conftest.py index 7d0a8397..1fcb378f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,8 +31,8 @@ from invenio_db import db import pytest -from hepdata.ext.elasticsearch.admin_view.api import AdminIndexer -from hepdata.ext.elasticsearch.api import reindex_all +from hepdata.ext.opensearch.admin_view.api import AdminIndexer +from hepdata.ext.opensearch.api import reindex_all from hepdata.factory import create_app from hepdata.modules.records.importer.api import import_records, _download_file diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 9faeb934..108810ab 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -44,7 +44,7 @@ from time import sleep from hepdata.config import CFG_TMPDIR, RUN_SELENIUM_LOCALLY -from hepdata.ext.elasticsearch.api import reindex_all +from hepdata.ext.opensearch.api import reindex_all from hepdata.factory import create_app from tests.conftest import get_identifiers, import_default_data diff --git a/tests/e2e/test_general.py b/tests/e2e/test_general.py index 6ced0704..d1cdbd27 100644 --- a/tests/e2e/test_general.py +++ b/tests/e2e/test_general.py @@ -36,7 +36,7 @@ from functools import reduce from tests.conftest import import_default_data -from hepdata.ext.elasticsearch.api import reindex_all +from hepdata.ext.opensearch.api import reindex_all from hepdata.modules.records.importer.api import import_records from hepdata.modules.submission.api import get_latest_hepsubmission from hepdata.modules.records.utils.submission import unload_submission, get_or_create_hepsubmission diff --git a/tests/search_test.py b/tests/search_test.py index a81cfd26..f37a9c3f 100644 --- a/tests/search_test.py +++ b/tests/search_test.py @@ -22,15 +22,15 @@ from invenio_db import db from unittest.mock import call -from hepdata.ext.elasticsearch.config.es_config import \ +from hepdata.ext.opensearch.config.es_config import \ add_default_aggregations, sort_fields_mapping -from hepdata.ext.elasticsearch import api as es_api -from hepdata.ext.elasticsearch.config.es_config import get_filter_field -from hepdata.ext.elasticsearch.document_enhancers import add_data_keywords, process_cmenergies -from hepdata.ext.elasticsearch.process_results import merge_results, match_tables_to_papers, \ +from hepdata.ext.opensearch import api as es_api +from hepdata.ext.opensearch.config.es_config import get_filter_field +from hepdata.ext.opensearch.document_enhancers import add_data_keywords, process_cmenergies +from hepdata.ext.opensearch.process_results import merge_results, match_tables_to_papers, \ get_basic_record_information, is_datatable -from hepdata.ext.elasticsearch.query_builder import QueryBuilder, HEPDataQueryParser -from hepdata.ext.elasticsearch.utils import flip_sort_order, parse_and_format_date, prepare_author_for_indexing, \ +from hepdata.ext.opensearch.query_builder import QueryBuilder, HEPDataQueryParser +from hepdata.ext.opensearch.utils import flip_sort_order, parse_and_format_date, prepare_author_for_indexing, \ calculate_sort_order, push_keywords from hepdata.modules.records.importer.api import import_records from hepdata.modules.submission.models import HEPSubmission @@ -477,7 +477,7 @@ def test_reindex_all(app, load_default_data, identifiers, mocker): es_api.reindex_all(index=index, recreate=False, update_mapping=True, synchronous=True) # Test other params using mocking - m = mocker.patch('hepdata.ext.elasticsearch.api.reindex_batch') + m = mocker.patch('hepdata.ext.opensearch.api.reindex_batch') # Start and end at publication_recid 1, batch size 2: # should call reindex_batch twice with submission ids [1] then [2] @@ -550,8 +550,8 @@ def test_reindex_batch(app, load_default_data, mocker): index = app.config.get('OPENSEARCH_INDEX') # Mock methods called so we can check they're called with correct parameters - mock_index_record_ids = mocker.patch('hepdata.ext.elasticsearch.api.index_record_ids') - mock_push_data_keywords = mocker.patch('hepdata.ext.elasticsearch.api.push_data_keywords') + mock_index_record_ids = mocker.patch('hepdata.ext.opensearch.api.index_record_ids') + mock_push_data_keywords = mocker.patch('hepdata.ext.opensearch.api.push_data_keywords') # Reindex submission id 1 (pub_recid=1, with data submissions 2-15) mock_index_record_ids.return_value = {'publication': [1], 'datatable': list(range(2,16))} @@ -582,7 +582,7 @@ def test_update_record_mapping(app, mocker): # mapping should be as defined in record_mapping mapping = index.get_mapping(using=es) - from hepdata.ext.elasticsearch.config.record_mapping import mapping as real_mapping + from hepdata.ext.opensearch.config.record_mapping import mapping as real_mapping assert 'properties' in mapping['mock_index']['mappings'] for k in real_mapping.keys(): assert k in mapping['mock_index']['mappings']['properties'] diff --git a/tests/submission_test.py b/tests/submission_test.py index 1dfa2e4c..fb7c22d2 100644 --- a/tests/submission_test.py +++ b/tests/submission_test.py @@ -31,7 +31,7 @@ from invenio_db import db import pytest -from hepdata.ext.elasticsearch.api import get_records_matching_field +from hepdata.ext.opensearch.api import get_records_matching_field from hepdata.modules.permissions.models import SubmissionParticipant from hepdata.modules.records.api import format_submission, process_saved_file, create_new_version from hepdata.modules.records.utils.common import infer_file_type, contains_accepted_url, allowed_file, record_exists, \