From e32e7076bfeb5f2416431eec147d86b8f3b751f9 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 9 Oct 2023 15:44:39 -0500 Subject: [PATCH] Upgrade ES to 8.x (#10771) * Upgrade ES to 8.x We need to wait for https://github.com/django-es/django-elasticsearch-dsl/pull/458 or https://github.com/readthedocs/readthedocs.org/issues/10730. There are no breaking changes in the API, so this can be deployed without any downtime. The only change is that it now has authentication enabled by default, so our docker image now uses a password. How to deploy: - Deploy web extra - The new packages will continue working with the old ES, but we can make sure by searching from web-extra. - Finish the deploy - Upgrade the ES cluster from cloud.elastic.co * Forgot about this * Fix tests * Upgrade dependencies --- .circleci/config.yml | 5 ++++- .../search/tests/test_faceted_search.py | 4 ++-- readthedocs/search/tests/test_views.py | 4 ++-- readthedocs/settings/base.py | 3 +-- requirements/deploy.txt | 21 +++++++++++-------- requirements/docker.txt | 21 +++++++++++-------- requirements/pip.in | 7 ++++--- requirements/pip.txt | 19 +++++++++-------- requirements/testing.txt | 21 +++++++++++-------- 9 files changed, 59 insertions(+), 46 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a66ccaad3e..a6387736db0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,11 +11,14 @@ jobs: environment: TOX_POSARGS: '' PYTEST_COVERAGE: --cov-report=xml --cov-config .coveragerc --cov=. --cov-append - - image: 'docker.elastic.co/elasticsearch/elasticsearch:7.17.13' + - image: 'docker.elastic.co/elasticsearch/elasticsearch:8.10.2' name: search environment: discovery.type: single-node ES_JAVA_OPTS: -Xms750m -Xmx750m + ELASTIC_PASSWORD: password + # Disabled SSL for testing. + xpack.security.transport.ssl.enabled: 'false' steps: - checkout - run: git submodule sync diff --git a/readthedocs/search/tests/test_faceted_search.py b/readthedocs/search/tests/test_faceted_search.py index b4b4d9c59e8..54406d44f9c 100644 --- a/readthedocs/search/tests/test_faceted_search.py +++ b/readthedocs/search/tests/test_faceted_search.py @@ -26,11 +26,11 @@ def test_search_exact_match(self, client, project, case): assert len(results) == 2 assert results[0]["project"] == "kuma" assert results[0]["path"] == "testdocumentation" - assert results[0]["version"] == "stable" + assert results[0]["version"] == "latest" assert results[1]["project"] == "kuma" assert results[1]["path"] == "testdocumentation" - assert results[1]["version"] == "latest" + assert results[1]["version"] == "stable" def test_search_combined_result(self, client, project): """Check search result are combined of both `AND` and `OR` operator diff --git a/readthedocs/search/tests/test_views.py b/readthedocs/search/tests/test_views.py index 213467aaa19..3ad1c708c9b 100644 --- a/readthedocs/search/tests/test_views.py +++ b/readthedocs/search/tests/test_views.py @@ -260,8 +260,8 @@ def test_file_search_exact_match(self, client, project): # There are two results, # one from each version of the "kuma" project. assert len(results) == 2 - assert results[0]["version"] == {"slug": "stable"} - assert results[1]["version"] == {"slug": "latest"} + assert results[0]["version"] == {"slug": "latest"} + assert results[1]["version"] == {"slug": "stable"} for result in results: assert result["project"] == {"alias": None, "slug": "kuma"} assert result["domain"] == "http://readthedocs.org" diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 51082e05b7d..5519b18f995 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -795,10 +795,9 @@ def DOCKER_LIMITS(self): RTD_ORG_TRIAL_PERIOD_DAYS = 30 # Elasticsearch settings. - ES_HOSTS = ['search:9200'] ELASTICSEARCH_DSL = { 'default': { - 'hosts': 'search:9200' + 'hosts': 'http://elastic:password@search:9200', }, } # Chunk size for elasticsearch reindex celery tasks diff --git a/requirements/deploy.txt b/requirements/deploy.txt index 7916d309031..9c0ed9c59b6 100644 --- a/requirements/deploy.txt +++ b/requirements/deploy.txt @@ -40,7 +40,7 @@ celery==5.2.7 certifi==2023.7.22 # via # -r requirements/pip.txt - # elasticsearch + # elastic-transport # requests # sentry-sdk cffi==1.16.0 @@ -135,7 +135,7 @@ django-csp==3.7 # via -r requirements/pip.txt django-debug-toolbar==4.2.0 # via -r requirements/pip.txt -django-elasticsearch-dsl==7.4 +django-elasticsearch-dsl==8.0 # via -r requirements/pip.txt django-extensions==3.2.3 # via -r requirements/pip.txt @@ -155,7 +155,7 @@ django-polymorphic==3.1.0 # via -r requirements/pip.txt django-simple-history==3.0.0 # via -r requirements/pip.txt -django-storages[boto3]==1.14.1 +django-storages[boto3]==1.14.2 # via -r requirements/pip.txt django-structlog==2.2.0 # via -r requirements/pip.txt @@ -185,11 +185,15 @@ drf-extensions==0.7.1 # via -r requirements/pip.txt drf-flex-fields==1.0.2 # via -r requirements/pip.txt -elasticsearch==7.17.9 +elastic-transport==8.4.1 + # via + # -r requirements/pip.txt + # elasticsearch +elasticsearch==8.10.0 # via # -r requirements/pip.txt # elasticsearch-dsl -elasticsearch-dsl==7.4.1 +elasticsearch-dsl==8.9.0 # via # -r requirements/pip.txt # django-elasticsearch-dsl @@ -341,7 +345,6 @@ six==1.16.0 # asttokens # django-annoying # django-elasticsearch-dsl - # elasticsearch-dsl # python-dateutil # unicode-slugify slumber==0.7.1 @@ -357,7 +360,7 @@ stripe==4.2.0 # via # -r requirements/pip.txt # dj-stripe -structlog==23.1.0 +structlog==23.2.0 # via # -r requirements/pip.txt # django-structlog @@ -395,7 +398,7 @@ urllib3==1.26.17 # -r requirements/pip.txt # botocore # docker - # elasticsearch + # elastic-transport # requests # sentry-sdk user-agents==2.2.0 @@ -412,7 +415,7 @@ wcwidth==0.2.8 # via # -r requirements/pip.txt # prompt-toolkit -websocket-client==1.6.3 +websocket-client==1.6.4 # via # -r requirements/pip.txt # docker diff --git a/requirements/docker.txt b/requirements/docker.txt index 1424771845f..b62d4a4383f 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -44,7 +44,7 @@ celery==5.2.7 certifi==2023.7.22 # via # -r requirements/pip.txt - # elasticsearch + # elastic-transport # requests cffi==1.16.0 # via @@ -146,7 +146,7 @@ django-csp==3.7 # via -r requirements/pip.txt django-debug-toolbar==4.2.0 # via -r requirements/pip.txt -django-elasticsearch-dsl==7.4 +django-elasticsearch-dsl==8.0 # via -r requirements/pip.txt django-extensions==3.2.3 # via -r requirements/pip.txt @@ -166,7 +166,7 @@ django-polymorphic==3.1.0 # via -r requirements/pip.txt django-simple-history==3.0.0 # via -r requirements/pip.txt -django-storages[boto3]==1.14.1 +django-storages[boto3]==1.14.2 # via -r requirements/pip.txt django-structlog==2.2.0 # via -r requirements/pip.txt @@ -196,11 +196,15 @@ drf-extensions==0.7.1 # via -r requirements/pip.txt drf-flex-fields==1.0.2 # via -r requirements/pip.txt -elasticsearch==7.17.9 +elastic-transport==8.4.1 + # via + # -r requirements/pip.txt + # elasticsearch +elasticsearch==8.10.0 # via # -r requirements/pip.txt # elasticsearch-dsl -elasticsearch-dsl==7.4.1 +elasticsearch-dsl==8.9.0 # via # -r requirements/pip.txt # django-elasticsearch-dsl @@ -374,7 +378,6 @@ six==1.16.0 # asttokens # django-annoying # django-elasticsearch-dsl - # elasticsearch-dsl # python-dateutil # unicode-slugify slumber==0.7.1 @@ -390,7 +393,7 @@ stripe==4.2.0 # via # -r requirements/pip.txt # dj-stripe -structlog==23.1.0 +structlog==23.2.0 # via # -r requirements/pip.txt # django-structlog @@ -430,7 +433,7 @@ urllib3==1.26.17 # -r requirements/pip.txt # botocore # docker - # elasticsearch + # elastic-transport # requests user-agents==2.2.0 # via -r requirements/pip.txt @@ -448,7 +451,7 @@ wcwidth==0.2.8 # via # -r requirements/pip.txt # prompt-toolkit -websocket-client==1.6.3 +websocket-client==1.6.4 # via # -r requirements/pip.txt # docker diff --git a/requirements/pip.in b/requirements/pip.in index 6c59f225e64..fb7dc05cf6b 100644 --- a/requirements/pip.in +++ b/requirements/pip.in @@ -87,9 +87,10 @@ django-allauth==0.51.0 requests-oauthlib # Search -elasticsearch<8.0 -elasticsearch-dsl<8.0 -django-elasticsearch-dsl<8.0 +elasticsearch~=8.0 +elasticsearch-dsl~=8.0 +django-elasticsearch-dsl~=8.0 + selectolax # embed diff --git a/requirements/pip.txt b/requirements/pip.txt index 9ae9c4fb1d4..332bc225a92 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -24,7 +24,7 @@ celery==5.2.7 # django-celery-beat certifi==2023.7.22 # via - # elasticsearch + # elastic-transport # requests cffi==1.16.0 # via cryptography @@ -95,7 +95,7 @@ django-csp==3.7 # via -r requirements/pip.in django-debug-toolbar==4.2.0 # via -r requirements/pip.in -django-elasticsearch-dsl==7.4 +django-elasticsearch-dsl==8.0 # via -r requirements/pip.in django-extensions==3.2.3 # via -r requirements/pip.in @@ -113,7 +113,7 @@ django-polymorphic==3.1.0 # via -r requirements/pip.in django-simple-history==3.0.0 # via -r requirements/pip.in -django-storages[boto3]==1.14.1 +django-storages[boto3]==1.14.2 # via -r requirements/pip.in django-structlog==2.2.0 # via -r requirements/pip.in @@ -141,11 +141,13 @@ drf-extensions==0.7.1 # via -r requirements/pip.in drf-flex-fields==1.0.2 # via -r requirements/pip.in -elasticsearch==7.17.9 +elastic-transport==8.4.1 + # via elasticsearch +elasticsearch==8.10.0 # via # -r requirements/pip.in # elasticsearch-dsl -elasticsearch-dsl==7.4.1 +elasticsearch-dsl==8.9.0 # via # -r requirements/pip.in # django-elasticsearch-dsl @@ -241,7 +243,6 @@ six==1.16.0 # via # django-annoying # django-elasticsearch-dsl - # elasticsearch-dsl # python-dateutil # unicode-slugify slumber==0.7.1 @@ -254,7 +255,7 @@ stripe==4.2.0 # via # -r requirements/pip.in # dj-stripe -structlog==23.1.0 +structlog==23.2.0 # via # -r requirements/pip.in # django-structlog @@ -276,7 +277,7 @@ urllib3==1.26.17 # via # botocore # docker - # elasticsearch + # elastic-transport # requests user-agents==2.2.0 # via -r requirements/pip.in @@ -289,7 +290,7 @@ virtualenv==20.24.5 # via -r requirements/pip.in wcwidth==0.2.8 # via prompt-toolkit -websocket-client==1.6.3 +websocket-client==1.6.4 # via docker # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/testing.txt b/requirements/testing.txt index cc738aaa27c..84196625fa5 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -40,7 +40,7 @@ celery==5.2.7 certifi==2023.7.22 # via # -r requirements/pip.txt - # elasticsearch + # elastic-transport # requests cffi==1.16.0 # via @@ -136,7 +136,7 @@ django-debug-toolbar==4.2.0 # via -r requirements/pip.txt django-dynamic-fixture==4.0.1 # via -r requirements/testing.in -django-elasticsearch-dsl==7.4 +django-elasticsearch-dsl==8.0 # via -r requirements/pip.txt django-extensions==3.2.3 # via -r requirements/pip.txt @@ -156,7 +156,7 @@ django-polymorphic==3.1.0 # via -r requirements/pip.txt django-simple-history==3.0.0 # via -r requirements/pip.txt -django-storages[boto3]==1.14.1 +django-storages[boto3]==1.14.2 # via -r requirements/pip.txt django-structlog==2.2.0 # via -r requirements/pip.txt @@ -188,11 +188,15 @@ drf-extensions==0.7.1 # via -r requirements/pip.txt drf-flex-fields==1.0.2 # via -r requirements/pip.txt -elasticsearch==7.17.9 +elastic-transport==8.4.1 + # via + # -r requirements/pip.txt + # elasticsearch +elasticsearch==8.10.0 # via # -r requirements/pip.txt # elasticsearch-dsl -elasticsearch-dsl==7.4.1 +elasticsearch-dsl==8.9.0 # via # -r requirements/pip.txt # django-elasticsearch-dsl @@ -355,7 +359,6 @@ six==1.16.0 # -r requirements/pip.txt # django-annoying # django-elasticsearch-dsl - # elasticsearch-dsl # python-dateutil # requests-mock # unicode-slugify @@ -392,7 +395,7 @@ stripe==4.2.0 # via # -r requirements/pip.txt # dj-stripe -structlog==23.1.0 +structlog==23.2.0 # via # -r requirements/pip.txt # django-structlog @@ -425,7 +428,7 @@ urllib3==1.26.17 # -r requirements/pip.txt # botocore # docker - # elasticsearch + # elastic-transport # requests user-agents==2.2.0 # via -r requirements/pip.txt @@ -441,7 +444,7 @@ wcwidth==0.2.8 # via # -r requirements/pip.txt # prompt-toolkit -websocket-client==1.6.3 +websocket-client==1.6.4 # via # -r requirements/pip.txt # docker