merge with main + opensearch-keystore using run_bin only + plugin_man…

…ager does not depend on cluster ready

.github/workflows/ci.yaml

@@ -1,9 +1,9 @@
-# Copyright 2022 Canonical Ltd.
+# Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 name: Tests
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: "${{ github.workflow }}-${{ github.ref }}"
   cancel-in-progress: true
 
 on:
@@ -16,7 +16,7 @@ on:
 jobs:
   lint:
     name: Lint
-    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v13.2.0
 
   unit-test:
     name: Unit test charm
@@ -61,22 +61,22 @@ jobs:
         path:
         - .
         - ./tests/integration/relations/opensearch_provider/application-charm/
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v13.2.0
     with:
       path-to-charm-directory: ${{ matrix.path }}
       cache: true
 
   integration-test:
-    name: Integration test charm
+    name: Integration test charm | 3.4.2
     needs:
       - lint
       - unit-test
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v13.2.0
     with:
       artifact-prefix: packed-charm-cache-true
       cloud: lxd
-      juju-snap-channel: 3.3/stable
+      juju-agent-version: 3.4.2
     secrets:
       # GitHub appears to redact each line of a multi-line secret
       # Avoid putting `{` or `}` on a line by itself so that it doesn't get redacted in logs

.github/workflows/release.yaml

@@ -1,4 +1,4 @@
-# Copyright 2023 Canonical Ltd.
+# Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 name: Release to latest/edge
 
@@ -32,13 +32,13 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v13.2.0
 
   release:
     name: Release charm
     needs:
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v13.1.1
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v13.2.0
     with:
       channel: 2/edge
       artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}

.github/workflows/sync_issue_to_jira.yaml

@@ -1,4 +1,4 @@
-# Copyright 2023 Canonical Ltd.
+# Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 name: Sync issue to Jira
 

lib/charms/data_platform_libs/v0/data_interfaces.py

@@ -331,7 +331,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 34
+LIBPATCH = 36
 
 PYDEPS = ["ops>=2.0.0"]
 
@@ -642,16 +642,16 @@ def _move_to_new_label_if_needed(self):
             return
 
         # Create a new secret with the new label
-        old_meta = self._secret_meta
         content = self._secret_meta.get_content()
+        self._secret_uri = None
 
         # I wish we could just check if we are the owners of the secret...
         try:
             self._secret_meta = self.add_secret(content, label=self.label)
         except ModelError as err:
             if "this unit is not the leader" not in str(err):
                 raise
-        old_meta.remove_all_revisions()
+        self.current_label = None
 
     def set_content(self, content: Dict[str, str]) -> None:
         """Setting cached secret content."""
@@ -1586,7 +1586,7 @@ def _register_secret_to_relation(
         """
         label = self._generate_secret_label(relation_name, relation_id, group)
 
-        # Fetchin the Secret's meta information ensuring that it's locally getting registered with
+        # Fetching the Secret's meta information ensuring that it's locally getting registered with
         CachedSecret(self._model, self.component, label, secret_id).meta
 
     def _register_secrets_to_relation(self, relation: Relation, params_name_list: List[str]):
@@ -2309,7 +2309,7 @@ def _secrets(self) -> dict:
         return self._cached_secrets
 
     def _get_secret(self, group) -> Optional[Dict[str, str]]:
-        """Retrieveing secrets."""
+        """Retrieving secrets."""
         if not self.app:
             return
         if not self._secrets.get(group):
@@ -3016,7 +3016,7 @@ class KafkaRequiresEvents(CharmEvents):
 # Kafka Provides and Requires
 
 
-class KafkaProvidesData(ProviderData):
+class KafkaProviderData(ProviderData):
     """Provider-side of the Kafka relation."""
 
     def __init__(self, model: Model, relation_name: str) -> None:
@@ -3059,12 +3059,12 @@ def set_zookeeper_uris(self, relation_id: int, zookeeper_uris: str) -> None:
         self.update_relation_data(relation_id, {"zookeeper-uris": zookeeper_uris})
 
 
-class KafkaProvidesEventHandlers(EventHandlers):
+class KafkaProviderEventHandlers(EventHandlers):
     """Provider-side of the Kafka relation."""
 
     on = KafkaProvidesEvents()  # pyright: ignore [reportAssignmentType]
 
-    def __init__(self, charm: CharmBase, relation_data: KafkaProvidesData) -> None:
+    def __init__(self, charm: CharmBase, relation_data: KafkaProviderData) -> None:
         super().__init__(charm, relation_data)
         # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above
         self.relation_data = relation_data
@@ -3086,15 +3086,15 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None:
             )
 
 
-class KafkaProvides(KafkaProvidesData, KafkaProvidesEventHandlers):
+class KafkaProvides(KafkaProviderData, KafkaProviderEventHandlers):
     """Provider-side of the Kafka relation."""
 
     def __init__(self, charm: CharmBase, relation_name: str) -> None:
-        KafkaProvidesData.__init__(self, charm.model, relation_name)
-        KafkaProvidesEventHandlers.__init__(self, charm, self)
+        KafkaProviderData.__init__(self, charm.model, relation_name)
+        KafkaProviderEventHandlers.__init__(self, charm, self)
 
 
-class KafkaRequiresData(RequirerData):
+class KafkaRequirerData(RequirerData):
     """Requirer-side of the Kafka relation."""
 
     def __init__(
@@ -3124,12 +3124,12 @@ def topic(self, value):
         self._topic = value
 
 
-class KafkaRequiresEventHandlers(RequirerEventHandlers):
+class KafkaRequirerEventHandlers(RequirerEventHandlers):
     """Requires-side of the Kafka relation."""
 
     on = KafkaRequiresEvents()  # pyright: ignore [reportAssignmentType]
 
-    def __init__(self, charm: CharmBase, relation_data: KafkaRequiresData) -> None:
+    def __init__(self, charm: CharmBase, relation_data: KafkaRequirerData) -> None:
         super().__init__(charm, relation_data)
         # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above
         self.relation_data = relation_data
@@ -3142,10 +3142,13 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None:
             return
 
         # Sets topic, extra user roles, and "consumer-group-prefix" in the relation
-        relation_data = {
-            f: getattr(self, f.replace("-", "_"), "")
-            for f in ["consumer-group-prefix", "extra-user-roles", "topic"]
-        }
+        relation_data = {"topic": self.relation_data.topic}
+
+        if self.relation_data.extra_user_roles:
+            relation_data["extra-user-roles"] = self.relation_data.extra_user_roles
+
+        if self.relation_data.consumer_group_prefix:
+            relation_data["consumer-group-prefix"] = self.relation_data.consumer_group_prefix
 
         self.relation_data.update_relation_data(event.relation.id, relation_data)
 
@@ -3188,7 +3191,7 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None:
             return
 
 
-class KafkaRequires(KafkaRequiresData, KafkaRequiresEventHandlers):
+class KafkaRequires(KafkaRequirerData, KafkaRequirerEventHandlers):
     """Provider-side of the Kafka relation."""
 
     def __init__(
@@ -3200,7 +3203,7 @@ def __init__(
         consumer_group_prefix: Optional[str] = None,
         additional_secret_fields: Optional[List[str]] = [],
     ) -> None:
-        KafkaRequiresData.__init__(
+        KafkaRequirerData.__init__(
             self,
             charm.model,
             relation_name,
@@ -3209,7 +3212,7 @@ def __init__(
             consumer_group_prefix,
             additional_secret_fields,
         )
-        KafkaRequiresEventHandlers.__init__(self, charm, self)
+        KafkaRequirerEventHandlers.__init__(self, charm, self)
 
 
 # Opensearch related events

lib/charms/data_platform_libs/v0/s3.py

@@ -137,7 +137,7 @@ def _on_credential_gone(self, event: CredentialsGoneEvent):
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 4
+LIBPATCH = 5
 
 logger = logging.getLogger(__name__)
 
@@ -212,7 +212,7 @@ class S3CredentialEvents(CharmEvents):
 class S3Provider(Object):
     """A provider handler for communicating S3 credentials to consumers."""
 
-    on = S3CredentialEvents()  # pyright: ignore [reportGeneralTypeIssues]
+    on = S3CredentialEvents()  # pyright: ignore [reportAssignmentType]
 
     def __init__(
         self,
@@ -481,6 +481,18 @@ def set_s3_api_version(self, relation_id: int, s3_api_version: str) -> None:
         """
         self.update_connection_info(relation_id, {"s3-api-version": s3_api_version})
 
+    def set_delete_older_than_days(self, relation_id: int, days: int) -> None:
+        """Sets the retention days for full backups in application databag.
+
+        This function writes in the application data bag, therefore,
+        only the leader unit can call it.
+
+        Args:
+            relation_id: the identifier for a particular relation.
+            days: the value.
+        """
+        self.update_connection_info(relation_id, {"delete-older-than-days": str(days)})
+
     def set_attributes(self, relation_id: int, attributes: List[str]) -> None:
         """Sets the connection attributes in application databag.
 
@@ -580,6 +592,17 @@ def s3_api_version(self) -> Optional[str]:
 
         return self.relation.data[self.relation.app].get("s3-api-version")
 
+    @property
+    def delete_older_than_days(self) -> Optional[int]:
+        """Returns the retention days for full backups."""
+        if not self.relation.app:
+            return None
+
+        days = self.relation.data[self.relation.app].get("delete-older-than-days")
+        if days is None:
+            return None
+        return int(days)
+
     @property
     def attributes(self) -> Optional[List[str]]:
         """Returns the attributes."""
@@ -613,7 +636,7 @@ class S3CredentialRequiresEvents(ObjectEvents):
 class S3Requirer(Object):
     """Requires-side of the s3 relation."""
 
-    on = S3CredentialRequiresEvents()  # pyright: ignore[reportGeneralTypeIssues]
+    on = S3CredentialRequiresEvents()  # pyright: ignore[reportAssignmentType]
 
     def __init__(
         self, charm: ops.charm.CharmBase, relation_name: str, bucket_name: Optional[str] = None

lib/charms/opensearch/v0/constants_charm.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Canonical Ltd.
+# Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 """In this file we declare the constants and enums used by the charm."""
@@ -42,6 +42,9 @@
 ClusterHealthYellow = (
     "1 or more 'replica' shards are not assigned, please scale your application up."
 )
+ClusterHealthRedUpgrade = (
+    "1 or more 'primary' shards are not assigned in the cluster. Fix unhealthy units"
+)
 IndexCreationFailed = "failed to create {index} index - deferring index-requested event..."
 UserCreationFailed = "failed to create users for {rel_name} relation {id}"
 PluginConfigChangeError = "Failed to apply config changes on the plugin."
@@ -61,6 +64,10 @@
 )
 PluginConfigError = "Unexpected error during plugin configuration, check the logs"
 BackupSetupFailed = "Backup setup failed, check logs for details"
+S3RelMissing = "Backup failover cluster missing S3 relation."
+S3RelShouldNotExist = "This unit should not be related to S3"
+S3RelDataIncomplete = "S3 relation data missing or incomplete."
+S3RelUneligible = "Only orchestrator clusters should relate to S3."
 
 # Wait status
 RequestUnitServiceOps = "Requesting lock on operation: {}"
@@ -101,7 +108,7 @@
 KibanaserverRole = "kibana_server"
 
 # Opensearch Snap revision
-OPENSEARCH_SNAP_REVISION = 40  # Keep in sync with `workload_version` file
+OPENSEARCH_SNAP_REVISION = 50  # Keep in sync with `workload_version` file
 
 # User-face Backup ID format
 OPENSEARCH_BACKUP_ID_FORMAT = "%Y-%m-%dT%H:%M:%SZ"

lib/charms/opensearch/v0/constants_secrets.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Canonical Ltd.
+# Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 """In this file we declare the constants and enums used by Juju secrets in Opensearch."""

lib/charms/opensearch/v0/constants_tls.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Canonical Ltd.
+# Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 """In this file we declare the constants and enums used by TLS related components."""

lib/charms/opensearch/v0/helper_charm.py

@@ -1,15 +1,18 @@
-# Copyright 2023 Canonical Ltd.
+# Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 """Utility functions for charms related operations."""
 import re
-import typing
+from time import time_ns
+from typing import TYPE_CHECKING
 
+from charms.data_platform_libs.v0.data_interfaces import Scope
+from charms.opensearch.v0.constants_charm import PeerRelationName
 from charms.opensearch.v0.helper_enums import BaseStrEnum
 from ops import CharmBase
 from ops.model import ActiveStatus, StatusBase
 
-if typing.TYPE_CHECKING:
+if TYPE_CHECKING:
     from charms.opensearch.v0.opensearch_base_charm import OpenSearchBaseCharm
 
 # The unique Charmhub library identifier, never change it
@@ -115,3 +118,22 @@ def relation_departure_reason(charm: CharmBase, relation_name: str) -> RelDepart
         return RelDepartureReason.SCALE_DOWN
 
     return RelDepartureReason.REL_BROKEN
+
+
+def trigger_peer_rel_changed(
+    charm: "OpenSearchBaseCharm",
+    only_by_leader: bool = False,
+    on_other_units: bool = True,
+    on_current_unit: bool = False,
+) -> None:
+    """Force trigger a peer rel changed event."""
+    if only_by_leader and not charm.unit.is_leader():
+        return
+
+    if on_other_units or not on_current_unit:
+        charm.peers_data.put(Scope.APP if only_by_leader else Scope.UNIT, "update-ts", time_ns())
+
+    if on_current_unit:
+        charm.on[PeerRelationName].relation_changed.emit(
+            charm.model.get_relation(PeerRelationName)
+        )