diff --git a/.codeclimate.yml b/.codeclimate.yml index 02e8782e65d..a0936fee04a 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -64,7 +64,7 @@ exclude_patterns: - "dist/" - "features/" - "**/node_modules/" - - "script/" + - "script" - "**/spec/" - "**/test/" - "**/tests/" @@ -83,4 +83,5 @@ exclude_patterns: - packages/models-library/src/models_library/utils/_original_fastapi_encoders.py - services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py - services/web/server/src/simcore_service_webserver/exporter/formatters/sds/xlsx/templates/code_description.py - - services/web/server/src/simcore_service_webserver/projects/db.py # NOTE: refactor will be done in upcomming PRs + - services/web/server/src/simcore_service_webserver/projects/db.py + - "scripts/" diff --git a/scripts/release/monitor/Makefile b/scripts/release/monitor/Makefile new file mode 100644 index 00000000000..25427db8b97 --- /dev/null +++ b/scripts/release/monitor/Makefile @@ -0,0 +1,3 @@ +.PHONY: install-dev +install-dev: + pip install . diff --git a/scripts/release/monitor/README.md b/scripts/release/monitor/README.md new file mode 100644 index 00000000000..beb4342e3c1 --- /dev/null +++ b/scripts/release/monitor/README.md @@ -0,0 +1,35 @@ +# Helper for monitoring of release +`pip install .` +`monitor-release --help` + +Check current status of containers +`monitor-release master containers` +Check running sidecars: +`monitor-release master sidecars` + +# Create .env file +``` +MASTER_PORTAINER_URL= +MASTER_PORTAINER_USERNAME= +MASTER_PORTAINER_PASSWORD= + +DALCO_STAGING_PORTAINER_URL= +DALCO_STAGING_PORTAINER_USERNAME= +DALCO_STAGING_PORTAINER_PASSWORD= + +DALCO_PRODUCTION_PORTAINER_URL= +DALCO_PRODUCTION_PORTAINER_USERNAME= +DALCO_PRODUCTION_PORTAINER_PASSWORD= + +TIP_PRODUCTION_PORTAINER_URL= +TIP_PRODUCTION_PORTAINER_USERNAME= +TIP_PRODUCTION_PORTAINER_PASSWORD= + +AWS_STAGING_PORTAINER_URL= +AWS_STAGING_PORTAINER_USERNAME= +AWS_STAGING_PORTAINER_PASSWORD= + +AWS_PRODUCTION_PORTAINER_URL= +AWS_PRODUCTION_PORTAINER_USERNAME= +AWS_PRODUCTION_PORTAINER_PASSWORD= +``` diff --git a/scripts/release/monitor/monitor_release/__init__.py b/scripts/release/monitor/monitor_release/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/release/monitor/monitor_release/cli.py b/scripts/release/monitor/monitor_release/cli.py new file mode 100644 index 00000000000..d29e10fc16f --- /dev/null +++ b/scripts/release/monitor/monitor_release/cli.py @@ -0,0 +1,27 @@ +from enum import Enum + +import typer +from monitor_release.models import Deployment +from monitor_release.portainer import check_containers_deploys, check_running_sidecars +from monitor_release.settings import get_settings +from rich.console import Console + +app = typer.Typer() +console = Console() + + +class Action(str, Enum): + containers = "containers" + sidecars = "sidecars" + + +@app.command() +def main(deployment: Deployment, action: Action): + settings = get_settings(deployment) + console.print(f"Deployment: {deployment}") + console.print(f"Action: {action}") + + if action == Action.containers: + check_containers_deploys(settings, deployment) + if action == Action.sidecars: + check_running_sidecars(settings, deployment) diff --git a/scripts/release/monitor/monitor_release/gitlab.py b/scripts/release/monitor/monitor_release/gitlab.py new file mode 100644 index 00000000000..3115195b565 --- /dev/null +++ b/scripts/release/monitor/monitor_release/gitlab.py @@ -0,0 +1,10 @@ +# GET https://git.speag.com/api/v4/projects/300/pipeline_schedules + +# POST https://git.speag.com/api/v4/projects/300/pipeline_schedules/43/take_ownership + +# PUT https://git.speag.com/api/v4/projects/300/pipeline_schedules/74 +# { +# "active": true +# } + +# HEADERS: PRIVATE-TOKEN: diff --git a/scripts/release/monitor/monitor_release/models.py b/scripts/release/monitor/monitor_release/models.py new file mode 100644 index 00000000000..f23aca48bf3 --- /dev/null +++ b/scripts/release/monitor/monitor_release/models.py @@ -0,0 +1,22 @@ +from datetime import datetime +from enum import Enum + +from pydantic import BaseModel + + +class Deployment(str, Enum): + master = "master" + aws_staging = "aws-staging" + dalco_staging = "dalco-staging" + aws_production = "aws-production" + dalco_production = "dalco-production" + tip_production = "tip-production" + + +class RunningSidecar(BaseModel): + name: str + created_at: datetime + user_id: str + project_id: str + service_key: str + service_version: str diff --git a/scripts/release/monitor/monitor_release/portainer.py b/scripts/release/monitor/monitor_release/portainer.py new file mode 100644 index 00000000000..08864eda7a6 --- /dev/null +++ b/scripts/release/monitor/monitor_release/portainer.py @@ -0,0 +1,85 @@ +from monitor_release.models import RunningSidecar +from monitor_release.portainer_utils import ( + check_simcore_deployed_services, + check_simcore_running_sidecars, + get_bearer_token, + get_containers, + get_services, + get_tasks, +) +from rich.console import Console +from rich.table import Table + +console = Console() + + +def check_containers_deploys(settings, deployment): + token = get_bearer_token(settings) + services = get_services(settings, token) + tasks = get_tasks(settings, token) + containers = get_containers(settings, token) + + output = check_simcore_deployed_services(settings, services, tasks, containers) + + table = Table( + "Service", + "Status", + "Last Updated", + "Git SHA", + title=f"[bold yellow]{deployment.upper()}[/bold yellow]", + ) + for item in output.values(): + service_name = item["service_name"] + container_status = "[bold red]Not running[/bold red]" + container_timestamp = None + container_git_sha = None + for task in item["tasks"]: + oldest_running_task_timestamp = None + if task["status"] == "running": + if ( + oldest_running_task_timestamp is None + or oldest_running_task_timestamp > task["timestamp"] + ): + container_status = f"[green]{task['status']}[/green]" + container_timestamp = f"{task['timestamp']}" + container_git_sha = task["git_sha"] + + oldest_running_task_timestamp = task["timestamp"] + if task["status"] == "starting": + container_status = f"[blue]{task['status']}[/blue]" + container_timestamp = f"{task['timestamp']}" + container_git_sha = task["git_sha"] + break + + table.add_row( + service_name, container_status, container_timestamp, container_git_sha + ) + + console.print(table) + + +def check_running_sidecars(settings, deployment): + token = get_bearer_token(settings) + services = get_services(settings, token) + + sidecars: list[RunningSidecar] = check_simcore_running_sidecars(settings, services) + table = Table( + "Sidecar name", + "Created at", + "User ID", + "Project ID", + "Service Key", + "Service Version", + title=f"[bold yellow]{deployment.upper()}[/bold yellow]", + ) + for sidecar in sidecars: + table.add_row( + sidecar.name, + f"{sidecar.created_at}", + sidecar.user_id, + sidecar.project_id, + sidecar.service_key, + sidecar.service_version, + ) + + console.print(table) diff --git a/scripts/release/monitor/monitor_release/portainer_utils.py b/scripts/release/monitor/monitor_release/portainer_utils.py new file mode 100644 index 00000000000..23b99883326 --- /dev/null +++ b/scripts/release/monitor/monitor_release/portainer_utils.py @@ -0,0 +1,134 @@ +import json + +import arrow +import requests +from monitor_release.models import RunningSidecar +from monitor_release.settings import Settings + + +def get_bearer_token(settings: Settings): + headers = {"accept": "application/json", "Content-Type": "application/json"} + payload = json.dumps( + { + "Username": settings.portainer_username, + "Password": settings.portainer_password, + } + ) + response = requests.post( + f"{settings.portainer_url}/portainer/api/auth", + headers=headers, + data=payload, + ) + bearer_token = response.json()["jwt"] + return bearer_token + + +def get_services(settings: Settings, bearer_token): + services_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/services" + response = requests.get( + services_url, + headers={ + "Authorization": "Bearer " + bearer_token, + "Content-Type": "application/json", + }, + ) + services = response.json() + return services + + +def get_tasks(settings: Settings, bearer_token): + tasks_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/tasks" + response = requests.get( + tasks_url, + headers={ + "Authorization": "Bearer " + bearer_token, + "Content-Type": "application/json", + }, + ) + tasks = response.json() + return tasks + + +def get_containers(settings: Settings, bearer_token): + bearer_token = get_bearer_token(settings) + + containers_url = f"{settings.portainer_url}/portainer/api/endpoints/{settings.portainer_endpoint_version}/docker/containers/json?all=true" + response = requests.get( + containers_url, + headers={ + "Authorization": "Bearer " + bearer_token, + "Content-Type": "application/json", + }, + ) + containers = response.json() + return containers + + +def check_simcore_running_sidecars(settings: Settings, services): + running_sidecars: list[RunningSidecar] = [] + for service in services: + if ( + service["Spec"]["Name"].startswith("dy-sidecar") + and service["Spec"]["Labels"]["io.simcore.runtime.swarm-stack-name"] + == settings.swarm_stack_name + ): + running_sidecars.append( + RunningSidecar( + name=service["Spec"]["Name"], + created_at=arrow.get(service["CreatedAt"]).datetime, + user_id=service["Spec"]["Labels"]["io.simcore.runtime.user-id"], + project_id=service["Spec"]["Labels"][ + "io.simcore.runtime.project-id" + ], + service_key=service["Spec"]["Labels"][ + "io.simcore.runtime.service-key" + ], + service_version=service["Spec"]["Labels"][ + "io.simcore.runtime.service-version" + ], + ) + ) + return running_sidecars + + +def _generate_containers_map(containers): + container_map = {} + for container in containers: + git_sha = ( + container.get("Labels").get("org.opencontainers.image.revision") + if container.get("Labels").get( + "org.opencontainers.image.revision" + ) # container.get("Labels").get("org.label-schema.vcs-ref") + else container.get("Labels").get("org.label-schema.vcs-ref") + ) + + container_map[container["Id"]] = {"git_sha": git_sha} + return container_map + + +def check_simcore_deployed_services(settings: Settings, services, tasks, containers): + container_map = _generate_containers_map(containers) + service_task_map = {} + for service in services: + if service["Spec"]["Name"].startswith(settings.starts_with): + service_task_map[service["ID"]] = { + "service_name": service["Spec"]["Name"], + "tasks": [], + } + + for task in tasks: + if task["ServiceID"] in service_task_map: + if task["Status"].get("ContainerStatus") is None: + continue + container_id = task["Status"]["ContainerStatus"]["ContainerID"] + + service_task_map[task["ServiceID"]]["tasks"].append( + { + "created_at": arrow.get(task["CreatedAt"]).datetime, + "status": task["Status"]["State"], + "timestamp": arrow.get(task["Status"]["Timestamp"]).datetime, + "git_sha": container_map.get(container_id, {}).get("git_sha"), + } + ) + + return service_task_map diff --git a/scripts/release/monitor/monitor_release/postgres.py b/scripts/release/monitor/monitor_release/postgres.py new file mode 100644 index 00000000000..fdffa2a0fd7 --- /dev/null +++ b/scripts/release/monitor/monitor_release/postgres.py @@ -0,0 +1 @@ +# placeholder diff --git a/scripts/release/monitor/monitor_release/settings.py b/scripts/release/monitor/monitor_release/settings.py new file mode 100644 index 00000000000..4d74c2905d9 --- /dev/null +++ b/scripts/release/monitor/monitor_release/settings.py @@ -0,0 +1,99 @@ +import os + +from dotenv import load_dotenv +from pydantic import BaseModel + + +class Settings(BaseModel): + portainer_url: str + portainer_username: str + portainer_password: str + starts_with: str + swarm_stack_name: str + portainer_endpoint_version: int + + +def get_settings(deployment): + # pylint: disable=too-many-return-statements + load_dotenv("/home/matus/Projects/osparc-simcore/scripts/release/monitor/.env") + + if deployment == "master": + portainer_url = os.getenv("MASTER_PORTAINER_URL") + portainer_username = os.getenv("MASTER_PORTAINER_USERNAME") + portainer_password = os.getenv("MASTER_PORTAINER_PASSWORD") + + return Settings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="master-simcore_master", + swarm_stack_name="master-simcore", + portainer_endpoint_version=1, + ) + if deployment == "dalco-staging": + portainer_url = os.getenv("DALCO_STAGING_PORTAINER_URL") + portainer_username = os.getenv("DALCO_STAGING_PORTAINER_USERNAME") + portainer_password = os.getenv("DALCO_STAGING_PORTAINER_PASSWORD") + + return Settings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="staging-simcore_staging", + swarm_stack_name="staging-simcore", + portainer_endpoint_version=1, + ) + if deployment == "dalco-production": + portainer_url = os.getenv("DALCO_PRODUCTION_PORTAINER_URL") + portainer_username = os.getenv("DALCO_PRODUCTION_PORTAINER_USERNAME") + portainer_password = os.getenv("DALCO_PRODUCTION_PORTAINER_PASSWORD") + + return Settings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="production-simcore_production", + swarm_stack_name="production-simcore", + portainer_endpoint_version=1, + ) + if deployment == "tip-production": + portainer_url = os.getenv("TIP_PRODUCTION_PORTAINER_URL") + portainer_username = os.getenv("TIP_PRODUCTION_PORTAINER_USERNAME") + portainer_password = os.getenv("TIP_PRODUCTION_PORTAINER_PASSWORD") + + return Settings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="production-simcore_production", + swarm_stack_name="production-simcore", + portainer_endpoint_version=1, + ) + if deployment == "aws-staging": + portainer_url = os.getenv("AWS_STAGING_PORTAINER_URL") + portainer_username = os.getenv("AWS_STAGING_PORTAINER_USERNAME") + portainer_password = os.getenv("AWS_STAGING_PORTAINER_PASSWORD") + + return Settings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="staging-simcore_staging", + swarm_stack_name="staging-simcore", + portainer_endpoint_version=2, + ) + if deployment == "aws-production": + portainer_url = os.getenv("AWS_PRODUCTION_PORTAINER_URL") + portainer_username = os.getenv("AWS_PRODUCTION_PORTAINER_USERNAME") + portainer_password = os.getenv("AWS_PRODUCTION_PORTAINER_PASSWORD") + + return Settings( + portainer_url=portainer_url, + portainer_username=portainer_username, + portainer_password=portainer_password, + starts_with="production-simcore_production", + swarm_stack_name="production-simcore", + portainer_endpoint_version=2, + ) + else: + raise ValueError("Invalid environment type provided.") diff --git a/scripts/release/monitor/pyproject.toml b/scripts/release/monitor/pyproject.toml new file mode 100644 index 00000000000..b60275b6eaf --- /dev/null +++ b/scripts/release/monitor/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "monitor_release" +version = "1.2.3" +authors = [{name="Matus Drobuliak", email="drobuliak@itis.swiss" }] +description = "Helper script for monitoring releases" +readme = "README.md" +dependencies = ["python-dotenv","pydantic", "typer[all]", "rich", "requests", "arrow"] +requires-python = ">=3.10" + +[project.scripts] +monitor-release = "monitor_release.cli:app"