Check if alt_hosts are reachable & online instead of reachable

Definitions
Reachable: Successful socket connection
Online: Successful HTTP GET request to `/_nodes`

Fixes uncaught OpenSearchHttpError (status code 503): OpenSearch Security not initialized. (Fixes transient integration test failure—test 1 from #211 (comment))

lib/charms/opensearch/v0/helper_http.py

@@ -4,7 +4,6 @@
 """File containing http related helpers."""
 from typing import Any, Dict, List, Optional
 
-from charms.opensearch.v0.helper_networking import reachable_hosts
 from tenacity import RetryCallState
 
 # The unique Charmhub library identifier, never change it
@@ -31,20 +30,3 @@ def log_error(retry_state: RetryCallState):
         )
 
     return log_error
-
-
-def full_urls(
-    primary_host: str, port: int, path: str, alt_hosts: List[str], check_hosts_reach: bool = False
-) -> List[str]:
-    """Returns a list of well formatted and potentially reachable hosts."""
-    target_hosts = [primary_host]
-    if alt_hosts:
-        target_hosts.extend([alt_host for alt_host in alt_hosts if alt_host != primary_host])
-
-    if not check_hosts_reach:
-        return target_hosts
-
-    return [
-        f"https://{host_candidate}:{port}/{path}"
-        for host_candidate in reachable_hosts(target_hosts)
-    ]

lib/charms/opensearch/v0/helper_networking.py

@@ -94,13 +94,3 @@ def is_reachable(host: str, port: int) -> bool:
         return False
     finally:
         s.close()
-
-
-def reachable_hosts(hosts: List[str]) -> List[str]:
-    """Returns a list of reachable hosts."""
-    reachable: List[str] = []
-    for host_candidate in hosts:
-        if is_reachable(host_candidate, 9200):
-            reachable.append(host_candidate)
-
-    return reachable

lib/charms/opensearch/v0/opensearch_base_charm.py

@@ -41,7 +41,6 @@
 from charms.opensearch.v0.helper_networking import (
     get_host_ip,
     is_reachable,
-    reachable_hosts,
     unit_ip,
     units_ips,
 )
@@ -1314,4 +1313,6 @@ def alt_hosts(self) -> Optional[List[str]]:
         if not all_hosts:
             return None
 
-        return reachable_hosts([host for host in all_hosts if host != self.unit_ip])
+        return [
+            host for host in all_hosts if host != self.unit_ip and self.opensearch.is_node_up(host)
+        ]

lib/charms/opensearch/v0/opensearch_distro.py

@@ -153,13 +153,17 @@ def is_failed(self) -> bool:
         """Check if OpenSearch daemon has failed."""
         pass
 
-    def is_node_up(self) -> bool:
-        """Get status of current node. This assumes OpenSearch is Running."""
-        if not self.is_started():
+    def is_node_up(self, host: str = None) -> bool:
+        """Get status of node. This assumes OpenSearch is Running.
+
+        Defaults to this unit"""
+        if host is None:
+            host = self.host
+        if not is_reachable(host, self.port):
             return False
 
         try:
-            resp_code = self.request("GET", "/_nodes", resp_status_code=True)
+            resp_code = self.request("GET", "/_nodes", host=host, resp_status_code=True)
             return resp_code < 400
         except (OpenSearchHttpError, Exception):
             return False
@@ -190,7 +194,6 @@ def request(  # noqa
         payload: Optional[Union[str, Dict[str, any], List[Dict[str, any]]]] = None,
         host: Optional[str] = None,
         alt_hosts: Optional[List[str]] = None,
-        check_hosts_reach: bool = True,
         resp_status_code: bool = False,
         retries: int = 0,
         timeout: int = 5,
@@ -203,7 +206,6 @@ def request(  # noqa
             payload: str, JSON obj or array body payload.
             host: host of the node we wish to make a request on, by default current host.
             alt_hosts: in case the default host is unreachable, fallback/alternative hosts.
-            check_hosts_reach: if true, performs a ping for each host
             resp_status_code: whether to only return the HTTP code from the response.
             retries: number of retries
             timeout: number of seconds before a timeout happens
@@ -228,7 +230,7 @@ def call(url: str) -> requests.Response:
 
                     request_kwargs = {
                         "method": method.upper(),
-                        "url": urls[0],
+                        "url": url,
                         "verify": f"{self.paths.certs}/chain.pem",
                         "headers": {
                             "Accept": "application/json",
@@ -251,7 +253,11 @@ def call(url: str) -> requests.Response:
         if endpoint.startswith("/"):
             endpoint = endpoint[1:]
 
-        urls = full_urls(host or self.host, self.port, endpoint, alt_hosts, check_hosts_reach)
+        urls = [
+            f"https://{host_candidate}:{self.port}/{endpoint}"
+            for host_candidate in (host or self.host, *alt_hosts)
+            if self.is_node_up(host_candidate)
+        ]
         if not urls:
             raise OpenSearchHttpError(
                 f"Host {host or self.host}:{self.port} and alternative_hosts: {alt_hosts or []} not reachable."