Merge branch 'master' into lb-checks-1

Kafka/legos/kafka_check_lag_change/README.md

@@ -0,0 +1,25 @@
+[<img align="left" src="https://unskript.com/assets/favicon.png" width="100" height="100" style="padding-right: 5px">]
+(https://unskript.com/assets/favicon.png)
+<h1>Kafka check lag change</h1>
+
+## Description
+This action checks if the lag for consumer groups is not changing for a threshold number of hours.
+
+
+## Lego Details
+	kafka_check_lag_change(handle, group_id: str= "", threshold: int=1)
+		handle: Object of type unSkript KAFKA Connector.
+		group_id: Consumer group ID.
+		threshold: The number of hours to check if the lag hasn't changed.
+
+
+## Lego Input
+This Lego takes inputs handle, group_id, threshold.
+
+## Lego Output
+Here is a sample output.
+<img src="./1.png">
+
+## See it in Action
+
+You can see this Lego in action following this link [unSkript Live](https://us.app.unskript.io)

Kafka/legos/kafka_check_lag_change/kafka_check_lag_change.json

@@ -0,0 +1,15 @@
+{
+  "action_title": "Kafka check lag change",
+  "action_description": "This action checks if the lag for consumer groups is not changing for a threshold number of hours.\n",
+  "action_type": "LEGO_TYPE_KAFKA",
+  "action_entry_function": "kafka_check_lag_change",
+  "action_needs_credential": true,
+  "action_output_type": "ACTION_OUTPUT_TYPE_LIST",
+  "action_is_check": true,
+  "action_next_hop": [
+    ""
+  ],
+  "action_next_hop_parameter_mapping": {},
+  "action_supports_iteration": true,
+  "action_supports_poll": true
+}

Kafka/legos/kafka_check_lag_change/kafka_check_lag_change.py

@@ -0,0 +1,104 @@
+##
+# Copyright (c) 2023 unSkript, Inc
+# All rights reserved.
+##
+from typing import Tuple, Optional
+from kafka.admin import KafkaAdminClient
+from kafka import KafkaConsumer, TopicPartition
+from pydantic import BaseModel, Field
+from tabulate import tabulate
+import time
+
+
+
+class InputSchema(BaseModel):
+    group_id: Optional[str] = Field(
+        '',
+        description='Consumer group ID to which this consumer belongs',
+        title='Consumer group ID',
+    )
+    threshold: Optional[int] = Field(
+        3,
+        description="The number of hours to check if the lag hasn't changed.",
+        title='Threshold (in hours)',
+    )
+
+# This would be a global or persisted store of previous lags at the last check.
+# Format: { "topic-partition": [timestamp, lag] }
+prev_lags = {}
+
+
+def kafka_check_lag_change_printer(output):
+    status, issues = output
+
+    if status:
+        print("All consumer groups are maintaining their lags!")
+    else:
+        print("Lag issues detected:")
+        headers = ['Consumer Group', 'Topic', 'Partition', 'Description']
+        table_data = [(issue['consumer_group'], issue['topic'], issue['partition'], issue['description']) for issue in issues]
+        print(tabulate(table_data, headers=headers, tablefmt='grid'))
+
+
+def kafka_check_lag_change(handle, group_id: str= "", threshold: int=1) -> Tuple:
+    """
+    kafka_check_lag_change checks if the lag for consumer groups is not changing for X hours.
+
+    :param handle: Object of type unSkript KAFKA Connector.
+
+    :param group_id: Consumer group ID.
+
+    :param threshold: The number of hours to check if the lag hasn't changed.
+
+    :return: Tuple containing a status and an optional list of issues with lag.
+    """
+
+    issues = []
+    current_time = time.time()
+    admin_client = KafkaAdminClient(bootstrap_servers=handle.config['bootstrap_servers'])
+
+    if group_id:
+        consumer_groups = [group_id]
+    else:
+        consumer_groups = [group[0] for group in admin_client.list_consumer_groups()]
+
+    for group in consumer_groups:
+        consumer = KafkaConsumer(bootstrap_servers=handle.config['bootstrap_servers'], group_id=group)
+
+        for topic in consumer.topics():
+            partitions = consumer.partitions_for_topic(topic)
+            for partition in partitions:
+                tp = TopicPartition(topic, partition)
+                end_offset = consumer.end_offsets([tp])[tp]
+                committed = consumer.committed(tp) or 0
+                lag = end_offset - committed
+
+                if lag == 0:
+                    continue
+
+                key = f"{group}-{topic}-{partition}"
+                if key in prev_lags:
+                    prev_timestamp, prev_lag = prev_lags[key]
+
+                    # Only update timestamp in prev_lags if there's a change in the lag
+                    if prev_lag != lag:
+                        prev_lags[key] = (current_time, lag)
+                    elif (current_time - prev_timestamp) >= threshold * 3600:
+                        print(f"Issue detected with {key}. Adding to issues list.")
+                        issues.append({
+                            'consumer_group': group,
+                            'topic': topic,
+                            'partition': partition,
+                            'description': f"Lag hasn't changed for {threshold} hours. Current Lag: {lag}"
+                        })
+                else:
+                    prev_lags[key] = (current_time, lag)
+
+        consumer.close()
+
+    if issues:
+        return (False, issues)
+    return (True, None)
+
+
+

Keycloak/legos/keycloak_get_audit_report/README.md

@@ -0,0 +1,22 @@
+[<img align="left" src="https://unskript.com/assets/favicon.png" width="100" height="100" style="padding-right: 5px">]
+(https://unskript.com/assets/favicon.png)
+<h1>Get Keycloak audit report</h1>
+
+## Description
+Fetches the audit events from Keycloak.
+
+## Lego Details
+	keycloak_get_audit_report(handle):
+    handle: Handle object containing the KeycloakAdmin instance.
+
+
+## Lego Input
+This Lego takes inputs handle.
+
+## Lego Output
+Here is a sample output.
+<img src="./1.png">
+
+## See it in Action
+
+You can see this Lego in action following this link [unSkript Live](https://us.app.unskript.io)

Keycloak/legos/keycloak_get_audit_report/keycloak_get_audit_report.json

@@ -0,0 +1,12 @@
+{
+    "action_title": "Get Keycloak audit report",
+    "action_description": "Fetches the audit events from Keycloak.",
+    "action_type": "LEGO_TYPE_KEYCLOAK",
+    "action_entry_function": "keycloak_get_audit_report",
+    "action_needs_credential": true,
+    "action_output_type": "ACTION_OUTPUT_TYPE_LIST",
+    "action_is_check": false,
+    "action_supports_iteration": true,
+    "action_supports_poll": true,
+    "action_categories":["CATEGORY_TYPE_SRE","CATEGORY_TYPE_KEYCLOAK"]
+  }

Keycloak/legos/keycloak_get_audit_report/keycloak_get_audit_report.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2023 unSkript.com
+# All rights reserved.
+#
+from typing import List
+from tabulate import tabulate
+from pydantic import BaseModel
+
+
+class InputSchema(BaseModel):
+    pass
+
+
+def keycloak_get_audit_report_printer(output):
+    if not output:
+        print("No audit events found.")
+        return
+
+    # Extract relevant event data for tabulation
+    table_data = []
+    for event in output:
+        table_data.append({
+            "Time": event['time'],
+            "Type": event['type'],
+            "User ID": event['userId'],
+            "Client ID": event['clientId'],
+            "IP Address": event['ipAddress'],
+            "Error": event.get('error', '')
+        })
+
+    # Convert list of dictionaries to tabulated format
+    headers = ["Time", "Type", "User ID", "Client ID", "IP Address", "Error"]
+    print(tabulate(table_data, headers=headers, tablefmt="grid"))
+
+
+def keycloak_get_audit_report(handle) -> List:
+    """
+    keycloak_get_audit_report fetches the audit events from Keycloak.
+
+    :type handle: KeycloakAdmin
+    :param handle: Handle containing the KeycloakAdmin instance.
+
+    :rtype: List of dictionaries representing the audit events.
+    """
+    try:
+        # Fetch the events
+        events = handle.get_events()
+        return events if events else []
+
+    except Exception as e:
+        raise e
+
+

Keycloak/legos/keycloak_get_handle/README.md

@@ -0,0 +1,25 @@
+[<img align="left" src="https://unskript.com/assets/favicon.png" width="100" height="100" style="padding-right: 5px">](https://unskript.com/assets/favicon.png) 
+<h2>Get Keycloak Handle</h2>
+
+<br>
+
+## Description
+This Lego Get Keycloak Handle.
+
+
+## Lego Details
+
+    keycloak_get_handle(handle: object)
+
+        handle: Object of type unSkript Keycloak Connector
+
+## Lego Input
+This Lego take one input handle.
+
+## Lego Output
+Here is a sample output.
+
+
+## See it in Action
+
+You can see this Lego in action following this link [unSkript Live](https://us.app.unskript.io)

Keycloak/legos/keycloak_get_handle/keycloak_get_handle.json

@@ -0,0 +1,11 @@
+{
+    "action_title": "Keycloak Get Handle",
+    "action_description": "Get Keycloak Handle",
+    "action_type": "LEGO_TYPE_KEYCLOAK",
+    "action_entry_function": "keycloak_get_handle",
+    "action_needs_credential": true,
+    "action_supports_poll": false,
+    "action_supports_iteration": false,
+    "action_output_type": "ACTION_OUTPUT_TYPE_LIST"
+}
+

Keycloak/legos/keycloak_get_handle/keycloak_get_handle.py

@@ -0,0 +1,17 @@
+##
+##  Copyright (c) 2023 unSkript, Inc
+##  All rights reserved.
+##
+from pydantic import BaseModel
+
+
+class InputSchema(BaseModel):
+    pass
+
+
+def keycloak_get_handle(handle):
+    """keycloak_get_handle returns the Keycloak handle.
+
+          :rtype: Keycloak handle.
+    """
+    return handle

Keycloak/legos/keycloak_get_service_health/README.md

@@ -0,0 +1,22 @@
+[<img align="left" src="https://unskript.com/assets/favicon.png" width="100" height="100" style="padding-right: 5px">]
+(https://unskript.com/assets/favicon.png)
+<h1>Get Keycloak service health</h1>
+
+## Description
+Fetches the health of the Keycloak service by trying to list available realms.
+
+## Lego Details
+	keycloak_get_service_health fetches the health of the Keycloak service by trying to list available realms.
+    handle: Handle containing the KeycloakAdmin instance.
+
+
+## Lego Input
+This Lego takes inputs handle.
+
+## Lego Output
+Here is a sample output.
+<img src="./1.png">
+
+## See it in Action
+
+You can see this Lego in action following this link [unSkript Live](https://us.app.unskript.io)

Keycloak/legos/keycloak_get_service_health/keycloak_get_service_health.json

@@ -0,0 +1,16 @@
+{
+    "action_title": "Get Keycloak service health",
+    "action_description": "Fetches the health of the Keycloak service by trying to list available realms.",
+    "action_type": "LEGO_TYPE_KEYCLOAK",
+    "action_entry_function": "keycloak_get_service_health",
+    "action_needs_credential": true,
+    "action_output_type": "ACTION_OUTPUT_TYPE_LIST",
+    "action_is_check": true,
+    "action_next_hop": [
+      ""
+    ],
+    "action_next_hop_parameter_mapping": {},
+    "action_supports_iteration": true,
+    "action_supports_poll": true,
+    "action_categories":["CATEGORY_TYPE_SRE","CATEGORY_TYPE_KEYCLOAK"]
+  }

Keycloak/legos/keycloak_get_service_health/keycloak_get_service_health.py

@@ -0,0 +1,44 @@
+#
+# Copyright (c) 2023 unSkript.com
+# All rights reserved.
+#
+
+from typing import Tuple
+from pydantic import BaseModel
+
+
+class InputSchema(BaseModel):
+    pass
+
+def keycloak_get_service_health(handle) -> Tuple:
+    """
+    keycloak_get_service_health fetches the health of the Keycloak service by trying to list available realms.
+
+    :type handle: object
+    :param handle: Handle containing the KeycloakAdmin instance.
+
+    :rtype: Tuple indicating if the service is healthy and a list of available realms (or None if healthy).
+    """
+    try:
+        realms = handle.get_realms()
+        available_realms = [realm["realm"] for realm in realms]
+
+        if not available_realms:
+            return (False, available_realms)
+
+        return (True, None)
+
+    except Exception as e:
+        raise e
+
+def keycloak_get_service_health_printer(output):
+    is_healthy, realms = output
+
+    if is_healthy:
+        print("Keycloak Service is Healthy.")
+    else:
+        print("Keycloak Service is Unhealthy.")
+        if realms:
+            print("\nUnavailable Realms:")
+            for realm in realms:
+                print(f"  - {realm}")