[DPE-1032] Added Rollout service start stop (#19) (#20)

Node by node Rollout of start / stop events on the opensearch service

.github/workflows/ci.yaml

@@ -83,21 +83,21 @@ jobs:
           
           tox -e ha-integration
 
-  integration-test-lxd-ha-service:
-    name: Integration tests for HA - Service (lxd)
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: lxd
-      - name: Run integration tests
-        run: |
-          # set sysctl values in case the cloudinit-userdata not applied
-          sudo sysctl -w vm.max_map_count=262144
-          sudo sysctl -w vm.swappiness=0
-          sudo sysctl -w net.ipv4.tcp_retries2=5
-          
-          tox -e ha-service-integration
+#  integration-test-lxd-ha-service:
+#    name: Integration tests for HA - Service (lxd)
+#    runs-on: ubuntu-22.04
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v3
+#      - name: Setup operator environment
+#        uses: charmed-kubernetes/actions-operator@main
+#        with:
+#          provider: lxd
+#      - name: Run integration tests
+#        run: |
+#          # set sysctl values in case the cloudinit-userdata not applied
+#          sudo sysctl -w vm.max_map_count=262144
+#          sudo sysctl -w vm.swappiness=0
+#          sudo sysctl -w net.ipv4.tcp_retries2=5
+#
+#          tox -e ha-service-integration

actions.yaml

@@ -22,10 +22,3 @@ set-tls-private-key:
 
 get-admin-secrets:
   description: Get the admin user password and cert chain used by charm.
-
-start-service:
-  description: Start the OpenSearch daemon in a node.
-stop-service:
-  description: Safely stop the OpenSearch daemon running in a node.
-restart-service:
-  description: Safely restart the OpenSearch daemon running in a node.

lib/charms/opensearch/v0/constants_charm.py

@@ -20,6 +20,7 @@
 CertsExpirationError = "The certificates: {} need to be refreshed."
 WaitingForBusyShards = "The shards: {} need to complete building."
 AllocationExclusionFailed = "The exclusion of this node from the allocations failed."
+ServiceStartError = "An error occurred during the start of the OpenSearch service."
 ServiceStopped = "The OpenSearch service stopped."
 ServiceStopFailed = "An error occurred while attempting to stop the OpenSearch service."
 ServiceIsStopping = "The OpenSearch service is stopping."
@@ -28,8 +29,12 @@
     "Relation broken with the TLS Operator while TLS not fully configured. Stopping OpenSearch."
 )
 
+# Wait status
+RequestUnitServiceOps = "Requesting lock on operation: {}"
+
 # Maintenance statuses
 InstallProgress = "Installing OpenSearch..."
 SecurityIndexInitProgress = "Initializing the security index..."
 AdminUserInitProgress = "Configuring admin user..."
 HorizontalScaleUpSuggest = "Horizontal scale up advised: {} shards unassigned."
+WaitingForOtherUnitServiceOps = "Waiting for other units to complete the ops on their service."

lib/charms/opensearch/v0/helper_databag.py

@@ -26,8 +26,8 @@ class Scope(BaseStrEnum):
     UNIT = "unit"
 
 
-class SecretStore:
-    """Class representing a secret store for a charm.
+class Store:
+    """Class representing a relation data store for a charm.
 
     Requires the following 2 properties on the charm:
       - app_peers_data
@@ -38,20 +38,20 @@ def __init__(self, charm):
         self._charm = charm
 
     def put(self, scope: Scope, key: str, value: Optional[str]) -> None:
-        """Put string secret into the secret storage."""
+        """Put object into the relation data store."""
         if scope is None:
             raise ValueError("Scope undefined.")
 
         data = self._charm.unit_peers_data
         if scope == Scope.APP:
             data = self._charm.app_peers_data
 
-        self._put_or_delete(data, key, value)
+        self.put_or_delete(data, key, value)
 
     def put_object(
         self, scope: Scope, key: str, value: Dict[str, any], merge: bool = False
     ) -> None:
-        """Put dict / json object secret into the secret storage."""
+        """Put dict / json object into relation data store."""
         if merge:
             stored = self.get_object(scope, key)
 
@@ -66,7 +66,7 @@ def put_object(
         self.put(scope, key, payload_str)
 
     def get(self, scope: Scope, key: str) -> Optional[str]:
-        """Get string secret from the secret storage."""
+        """Get string from the relation data store."""
         if scope is None:
             raise ValueError("Scope undefined.")
 
@@ -77,22 +77,31 @@ def get(self, scope: Scope, key: str) -> Optional[str]:
         return data.get(key, None)
 
     def get_object(self, scope: Scope, key: str) -> Optional[Dict[str, any]]:
-        """Get dict / json object secret from the secret storage."""
+        """Get dict / json object from the relation data store."""
         data = self.get(scope, key)
         if data is None:
             return None
 
         return json.loads(data)
 
     def delete(self, scope: Scope, key: str):
-        """Delete secret from the secret storage."""
+        """Delete object from the relation data store."""
         self.put(scope, key, None)
 
     @staticmethod
-    def _put_or_delete(peers_data: Dict[str, str], key: str, value: Optional[str]):
-        """Put data into the secret storage or delete if value is None."""
+    def put_or_delete(peers_data: Dict[str, str], key: str, value: Optional[str]):
+        """Put data into the relation data store or delete if value is None."""
         if value is None:
             del peers_data[key]
             return
 
         peers_data.update({key: value})
+
+
+class SecretStore(Store):
+    """Class representing a secret store for a charm.
+
+    For now, it is simply a base class for regular Relation data store
+    """
+
+    pass

lib/charms/opensearch/v0/opensearch_base_charm.py

@@ -12,7 +12,7 @@
     HorizontalScaleUpSuggest,
 )
 from charms.opensearch.v0.constants_tls import TLS_RELATION, CertType
-from charms.opensearch.v0.helper_databag import Scope, SecretStore
+from charms.opensearch.v0.helper_databag import Scope, SecretStore, Store
 from charms.opensearch.v0.helper_enums import BaseStrEnum
 from charms.opensearch.v0.helper_networking import get_host_ip, units_ips
 from charms.opensearch.v0.opensearch_config import OpenSearchConfig
@@ -36,6 +36,7 @@
 
 
 PEER = "opensearch-peers"
+SERVICE_MANAGER = "service"
 
 
 logger = logging.getLogger(__name__)
@@ -63,6 +64,7 @@ def __init__(self, *args, distro: Type[OpenSearchDistribution] = None):
         self.opensearch = distro(self, PEER)
         self.opensearch_config = OpenSearchConfig(self.opensearch)
         self.secrets = SecretStore(self)
+        self.relation_store = Store(self)
         self.tls = OpenSearchTLS(self, TLS_RELATION)
 
     def on_allocation_exclusion_add_failed(self):

lib/charms/opensearch/v0/opensearch_distro.py

@@ -269,6 +269,7 @@ def request(
         endpoint: str,
         payload: Optional[Dict[str, any]] = None,
         host: Optional[str] = None,
+        alt_hosts: Optional[List[str]] = None,
     ) -> Union[Dict[str, any], List[any]]:
         """Make an HTTP request.
 
@@ -277,16 +278,27 @@ def request(
             endpoint: relative to the base uri.
             payload: JSON / map body payload.
             host: host of the node we wish to make a request on, by default current host.
+            alt_hosts: in case the default host is unreachable, fallback hosts
         """
         if None in [endpoint, method]:
             raise ValueError("endpoint or method missing")
 
         if endpoint.startswith("/"):
             endpoint = endpoint[1:]
 
-        target_host = host if host else self.host
-        if not is_reachable(target_host, self.port):
-            logger.error(f"Host {target_host}:{self.port} not reachable.")
+        primary_host = host or self.host
+        target_hosts = [primary_host]
+        if alt_hosts:
+            target_hosts.extend([alt_host for alt_host in alt_hosts if alt_host != primary_host])
+
+        target_host: Optional[str] = None
+        for host_candidate in target_hosts:
+            if is_reachable(host_candidate, self.port):
+                target_host = host_candidate
+                break
+
+        if not target_host:
+            logger.error(f"Host {primary_host}:{self.port} and alternative_hosts not reachable.")
             raise OpenSearchHttpError()
 
         full_url = f"https://{target_host}:{self.port}/{endpoint}"

lib/charms/operator_libs_linux/v1/snap.py

@@ -82,7 +82,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 4
+LIBPATCH = 5
 
 
 def _cache_init(func):
@@ -885,7 +885,7 @@ def _wrap_snap_operations(
 
 
 def install_local(
-    self, filename: str, classic: Optional[bool] = False, dangerous: Optional[bool] = False
+    filename: str, classic: Optional[bool] = False, dangerous: Optional[bool] = False
 ) -> Snap:
     """Perform a snap operation.
 
@@ -901,9 +901,11 @@ def install_local(
         "snap",
         "install",
         filename,
-        "--classic" if classic else "",
-        "--dangerous" if dangerous else "",
     ]
+    if classic:
+        _cmd.append("--classic")
+    if dangerous:
+        _cmd.append("--dangerous")
     try:
         result = subprocess.check_output(_cmd, universal_newlines=True).splitlines()[0]
         snap_name, _ = result.split(" ", 1)