🔨 adding helper release monitoring script (#4504)

.codeclimate.yml

@@ -64,7 +64,7 @@ exclude_patterns:
   - "dist/"
   - "features/"
   - "**/node_modules/"
-  - "script/"
+  - "script"
   - "**/spec/"
   - "**/test/"
   - "**/tests/"
@@ -83,4 +83,5 @@ exclude_patterns:
   - packages/models-library/src/models_library/utils/_original_fastapi_encoders.py
   - services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py
   - services/web/server/src/simcore_service_webserver/exporter/formatters/sds/xlsx/templates/code_description.py
-  - services/web/server/src/simcore_service_webserver/projects/db.py  # NOTE: refactor will be done in upcomming PRs
+  - services/web/server/src/simcore_service_webserver/projects/db.py
+  - "scripts/"

scripts/release/monitor/Makefile

@@ -0,0 +1,3 @@
+.PHONY: install-dev
+install-dev:
+	pip install .

scripts/release/monitor/README.md

@@ -0,0 +1,35 @@
+# Helper for monitoring of release
+`pip install .`
+`monitor-release --help`
+
+Check current status of containers
+`monitor-release master containers`
+Check running sidecars:
+`monitor-release master sidecars`
+
+# Create .env file
+```
+MASTER_PORTAINER_URL=
+MASTER_PORTAINER_USERNAME=
+MASTER_PORTAINER_PASSWORD=
+
+DALCO_STAGING_PORTAINER_URL=
+DALCO_STAGING_PORTAINER_USERNAME=
+DALCO_STAGING_PORTAINER_PASSWORD=
+
+DALCO_PRODUCTION_PORTAINER_URL=
+DALCO_PRODUCTION_PORTAINER_USERNAME=
+DALCO_PRODUCTION_PORTAINER_PASSWORD=
+
+TIP_PRODUCTION_PORTAINER_URL=
+TIP_PRODUCTION_PORTAINER_USERNAME=
+TIP_PRODUCTION_PORTAINER_PASSWORD=
+
+AWS_STAGING_PORTAINER_URL=
+AWS_STAGING_PORTAINER_USERNAME=
+AWS_STAGING_PORTAINER_PASSWORD=
+
+AWS_PRODUCTION_PORTAINER_URL=
+AWS_PRODUCTION_PORTAINER_USERNAME=
+AWS_PRODUCTION_PORTAINER_PASSWORD=
+```

scripts/release/monitor/monitor_release/cli.py

@@ -0,0 +1,27 @@
+from enum import Enum
+
+import typer
+from monitor_release.models import Deployment
+from monitor_release.portainer import check_containers_deploys, check_running_sidecars
+from monitor_release.settings import get_settings
+from rich.console import Console
+
+app = typer.Typer()
+console = Console()
+
+
+class Action(str, Enum):
+    containers = "containers"
+    sidecars = "sidecars"
+
+
+@app.command()
+def main(deployment: Deployment, action: Action):
+    settings = get_settings(deployment)
+    console.print(f"Deployment: {deployment}")
+    console.print(f"Action: {action}")
+
+    if action == Action.containers:
+        check_containers_deploys(settings, deployment)
+    if action == Action.sidecars:
+        check_running_sidecars(settings, deployment)

scripts/release/monitor/monitor_release/gitlab.py

@@ -0,0 +1,10 @@
+# GET https://git.speag.com/api/v4/projects/300/pipeline_schedules
+
+# POST https://git.speag.com/api/v4/projects/300/pipeline_schedules/43/take_ownership
+
+# PUT https://git.speag.com/api/v4/projects/300/pipeline_schedules/74
+# {
+#     "active": true
+# }
+
+# HEADERS:      PRIVATE-TOKEN:

scripts/release/monitor/monitor_release/models.py

@@ -0,0 +1,22 @@
+from datetime import datetime
+from enum import Enum
+
+from pydantic import BaseModel
+
+
+class Deployment(str, Enum):
+    master = "master"
+    aws_staging = "aws-staging"
+    dalco_staging = "dalco-staging"
+    aws_production = "aws-production"
+    dalco_production = "dalco-production"
+    tip_production = "tip-production"
+
+
+class RunningSidecar(BaseModel):
+    name: str
+    created_at: datetime
+    user_id: str
+    project_id: str
+    service_key: str
+    service_version: str

scripts/release/monitor/monitor_release/portainer.py

@@ -0,0 +1,85 @@
+from monitor_release.models import RunningSidecar
+from monitor_release.portainer_utils import (
+    check_simcore_deployed_services,
+    check_simcore_running_sidecars,
+    get_bearer_token,
+    get_containers,
+    get_services,
+    get_tasks,
+)
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+
+def check_containers_deploys(settings, deployment):
+    token = get_bearer_token(settings)
+    services = get_services(settings, token)
+    tasks = get_tasks(settings, token)
+    containers = get_containers(settings, token)
+
+    output = check_simcore_deployed_services(settings, services, tasks, containers)
+
+    table = Table(
+        "Service",
+        "Status",
+        "Last Updated",
+        "Git SHA",
+        title=f"[bold yellow]{deployment.upper()}[/bold yellow]",
+    )
+    for item in output.values():
+        service_name = item["service_name"]
+        container_status = "[bold red]Not running[/bold red]"
+        container_timestamp = None
+        container_git_sha = None
+        for task in item["tasks"]:
+            oldest_running_task_timestamp = None
+            if task["status"] == "running":
+                if (
+                    oldest_running_task_timestamp is None
+                    or oldest_running_task_timestamp > task["timestamp"]
+                ):
+                    container_status = f"[green]{task['status']}[/green]"
+                    container_timestamp = f"{task['timestamp']}"
+                    container_git_sha = task["git_sha"]
+
+                    oldest_running_task_timestamp = task["timestamp"]
+            if task["status"] == "starting":
+                container_status = f"[blue]{task['status']}[/blue]"
+                container_timestamp = f"{task['timestamp']}"
+                container_git_sha = task["git_sha"]
+                break
+
+        table.add_row(
+            service_name, container_status, container_timestamp, container_git_sha
+        )
+
+    console.print(table)
+
+
+def check_running_sidecars(settings, deployment):
+    token = get_bearer_token(settings)
+    services = get_services(settings, token)
+
+    sidecars: list[RunningSidecar] = check_simcore_running_sidecars(settings, services)
+    table = Table(
+        "Sidecar name",
+        "Created at",
+        "User ID",
+        "Project ID",
+        "Service Key",
+        "Service Version",
+        title=f"[bold yellow]{deployment.upper()}[/bold yellow]",
+    )
+    for sidecar in sidecars:
+        table.add_row(
+            sidecar.name,
+            f"{sidecar.created_at}",
+            sidecar.user_id,
+            sidecar.project_id,
+            sidecar.service_key,
+            sidecar.service_version,
+        )
+
+    console.print(table)

scripts/release/monitor/monitor_release/portainer_utils.py

@@ -0,0 +1,134 @@
+import json
+
+import arrow
+import requests
+from monitor_release.models import RunningSidecar
+from monitor_release.settings import Settings
+
+
+def get_bearer_token(settings: Settings):
+    headers = {"accept": "application/json", "Content-Type": "application/json"}
+    payload = json.dumps(
+        {
+            "Username": settings.portainer_username,
+            "Password": settings.portainer_password,
+        }
+    )
+    response = requests.post(
+        f"{settings.portainer_url}/portainer/api/auth",
+        headers=headers,
+        data=payload,
+    )
+    bearer_token = response.json()["jwt"]
+    return bearer_token
+
+
+def get_services(settings: Settings, bearer_token):
+    services_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/services"
+    response = requests.get(
+        services_url,
+        headers={
+            "Authorization": "Bearer " + bearer_token,
+            "Content-Type": "application/json",
+        },
+    )
+    services = response.json()
+    return services
+
+
+def get_tasks(settings: Settings, bearer_token):
+    tasks_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/tasks"
+    response = requests.get(
+        tasks_url,
+        headers={
+            "Authorization": "Bearer " + bearer_token,
+            "Content-Type": "application/json",
+        },
+    )
+    tasks = response.json()
+    return tasks
+
+
+def get_containers(settings: Settings, bearer_token):
+    bearer_token = get_bearer_token(settings)
+
+    containers_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/containers/json?all=true"
+    response = requests.get(
+        containers_url,
+        headers={
+            "Authorization": "Bearer " + bearer_token,
+            "Content-Type": "application/json",
+        },
+    )
+    containers = response.json()
+    return containers
+
+
+def check_simcore_running_sidecars(settings: Settings, services):
+    running_sidecars: list[RunningSidecar] = []
+    for service in services:
+        if (
+            service["Spec"]["Name"].startswith("dy-sidecar")
+            and service["Spec"]["Labels"]["io.simcore.runtime.swarm-stack-name"]
+            == settings.swarm_stack_name
+        ):
+            running_sidecars.append(
+                RunningSidecar(
+                    name=service["Spec"]["Name"],
+                    created_at=arrow.get(service["CreatedAt"]).datetime,
+                    user_id=service["Spec"]["Labels"]["io.simcore.runtime.user-id"],
+                    project_id=service["Spec"]["Labels"][
+                        "io.simcore.runtime.project-id"
+                    ],
+                    service_key=service["Spec"]["Labels"][
+                        "io.simcore.runtime.service-key"
+                    ],
+                    service_version=service["Spec"]["Labels"][
+                        "io.simcore.runtime.service-version"
+                    ],
+                )
+            )
+    return running_sidecars
+
+
+def _generate_containers_map(containers):
+    container_map = {}
+    for container in containers:
+        git_sha = (
+            container.get("Labels").get("org.opencontainers.image.revision")
+            if container.get("Labels").get(
+                "org.opencontainers.image.revision"
+            )  # container.get("Labels").get("org.label-schema.vcs-ref")
+            else container.get("Labels").get("org.label-schema.vcs-ref")
+        )
+
+        container_map[container["Id"]] = {"git_sha": git_sha}
+    return container_map
+
+
+def check_simcore_deployed_services(settings: Settings, services, tasks, containers):
+    container_map = _generate_containers_map(containers)
+    service_task_map = {}
+    for service in services:
+        if service["Spec"]["Name"].startswith(settings.starts_with):
+            service_task_map[service["ID"]] = {
+                "service_name": service["Spec"]["Name"],
+                "tasks": [],
+            }
+
+    for task in tasks:
+        if task["ServiceID"] in service_task_map:
+            if task["Status"].get("ContainerStatus") is None:
+                continue
+            container_id = task["Status"]["ContainerStatus"]["ContainerID"]
+
+            service_task_map[task["ServiceID"]]["tasks"].append(
+                {
+                    "created_at": arrow.get(task["CreatedAt"]).datetime,
+                    "status": task["Status"]["State"],
+                    "timestamp": arrow.get(task["Status"]["Timestamp"]).datetime,
+                    "git_sha": container_map.get(container_id, {}).get("git_sha"),
+                }
+            )
+
+    return service_task_map

scripts/release/monitor/monitor_release/postgres.py

@@ -0,0 +1 @@
+# placeholder