✨ Add: Manage grafana content/assets via terraform

scripts/common.Makefile

@@ -22,6 +22,13 @@ $(if $(REPO_CONFIG_LOCATION),,$(error The location of the repo.config file given
 $(if $(shell cat $(REPO_CONFIG_LOCATION)),,$(error The location of the repo.config file given in .config.location is invalid. Aborting))
 $(if $(shell wc -l $(REPO_BASE_DIR)/.config.location | grep 1),,$(error The .config.location file has more than one path specified. Only one path is allowed. Aborting))
 
+# Extract DEPLOYMENT_FQDN using Make functions
+DEPLOYMENT_FQDN := $(notdir $(patsubst %/,%, $(dir $(REPO_CONFIG_LOCATION))))
+# Construct CI_ENV_FILE using Make functions
+CI_ENV_FILE := $(realpath $(dir $(REPO_CONFIG_LOCATION))/../../.gitlab/pipelines/1_configurations/$(DEPLOYMENT_FQDN)/ci.env)
+$(if $(CI_ENV_FILE),,$(error The location of the repo.config file given in .config.location is invalid. Cannot find the ci.env file. Aborting))
+$(if $(shell cat $(CI_ENV_FILE)),,$(error The location of the repo.config file given in .config.location is invalid. Cannot find the ci.env file. Aborting))
+
 ifeq ($(_yq),)
 _yq = docker run --rm -i -v $${PWD}:/workdir mikefarah/yq:4.30.4
 endif
@@ -253,13 +260,13 @@ venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environmen
 ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2) --customize j2cli_customization.py
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) --customize j2cli_customization.py
 endef
 
 else
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2)
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3)
 endef
 
 endif

scripts/tf_helper_list_json_files_in_folder.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+DIRECTORY=$1
+
+# Find all JSON files within the directory
+FILES=$(find "$DIRECTORY" -mindepth 1 -maxdepth 1 -type f -name '*.json')
+
+# Create a JSON object where each file's basename is the key, with full paths as values
+JSON_OBJECT=$(echo "$FILES" | while read -r FILE; do
+    BASENAME=$(basename "$FILE" .json)
+    echo "{\"$BASENAME\": \"$FILE\"}"
+done | jq -s 'add')
+
+# Output the JSON map
+jq -n --argjson files "$JSON_OBJECT" '$files'

scripts/tf_helper_list_subfolders.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+DIRECTORY=$1
+
+# Use `find` to get the directories' base names
+SUBFOLDERS=$(find "$DIRECTORY" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)
+
+# Convert the subfolder names into a JSON object with jq, where each is paired with itself
+JSON_OBJECT=$(echo "$SUBFOLDERS" | tr ' ' '\n' | jq -Rn '
+  [inputs] |
+  map(select(. != "")) |
+  map({key: ., value: .}) |
+  from_entries')
+# Output the JSON map
+jq -n --argjson subfolders "$JSON_OBJECT" '$subfolders'

services/maintenance-page/Makefile

@@ -33,7 +33,7 @@ up-master: up
 
 .PHONY: docker-compose.yml
 docker-compose.yml: .venv .env
-	@$(call jinja, docker-compose.yml.j2, docker-compose.yml.unlinted) && \
+	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;
 

services/monitoring/Makefile

@@ -74,7 +74,7 @@ ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml con
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
 
 docker-compose.yml: docker-compose.yml.j2 .env .venv pgsql_query_exporter_config.yaml
-	$(call jinja,$<,$@)
+	$(call jinja,$<,.env,$@)
 
 .PHONY: update.grafana.pwd
 update.grafana.pwd: .env ## Change grafana pwd
@@ -84,24 +84,17 @@ update.grafana.pwd: .env ## Change grafana pwd
 	grafanacontainerid=$$(docker ps | grep grafana | awk '{print $$1;}');\
 	docker exec -ti $$grafanacontainerid grafana-cli admin reset-admin-password $$TRAEFIK_PASSWORD
 
-
-.PHONY: grafana-export
-grafana-export: .venv## Export the remote grafana dashboards and datasources TO YOUR LOCAL MACHINE
-	@cd grafana/scripts;\
-	source ${REPO_BASE_DIR}/.venv/bin/activate;\
-	pip install -r requirements.txt > /dev/null 2>&1;\
-	python3 export.py;
-
 .PHONY: grafana-import
-grafana-import: grafana/assets .venv ## Imports AND OVERWRITES the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE
-	@cd grafana/scripts;\
-	source ${REPO_BASE_DIR}/.venv/bin/activate;\
-	pip install -r requirements.txt > /dev/null 2>&1;\
-	python3 import.py
+grafana-import: grafana/assets ## Imports the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE
+	@pushd ${REPO_BASE_DIR}/services/monitoring/grafana && \
+	$(MAKE) terraform-plan && \
+	$(MAKE) terraform-apply; \
+	popd > /dev/null
+
 
 .PHONY: config.grafana.dashboards
 config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters
-	$(call jinja, $<, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
+	$(call jinja, $<, .env, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
 
 .PHONY: config.monitoring
 config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCATION}
@@ -133,7 +126,7 @@ config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .venv
 	@set -o allexport; \
 	source $<; \
 	set +o allexport; \
-	$(call jinja, prometheus/prometheus-ceph.yml.j2, prometheus/prometheus-ceph.yml); \
+	$(call jinja, prometheus/prometheus-ceph.yml.j2, .env, prometheus/prometheus-ceph.yml); \
 	cat prometheus/prometheus-base.yml | $(_yq) '. *+ load("prometheus/prometheus-simcore.yml")' | cat | \
 	$(_yq) '. *+ load("prometheus/prometheus-ceph.yml")' > prometheus/prometheus.yml; \
 	envsubst < prometheus/prometheus.yml > prometheus/prometheus.temp.yml; \
@@ -148,10 +141,10 @@ config.prometheus: ${REPO_CONFIG_LOCATION} .venv
 	mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
 
 pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
-	$(call jinja, $<, $@);
+	$(call jinja, $<, .env, $@);
 
 smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
-	$(call jinja, $<, $@);
+	$(call jinja, $<, .env, $@);
 
 .PHONY: grafana/assets
 grafana/assets: ${REPO_CONFIG_LOCATION}

services/monitoring/docker-compose.yml.j2

@@ -25,8 +25,6 @@ configs:
     file: ./prometheus/prometheus-federation.yml
   prometheus_rules:
     file: ./prometheus/prometheus.rules.yml
-  grafana_image_renderer_config:
-    file: ./grafana-image-renderer/config.json
   pgsql_query_exporter_config:
     file: ./pgsql_query_exporter_config.yaml
   smokeping_prober_config:

services/monitoring/grafana/Makefile

@@ -0,0 +1,58 @@
+.DEFAULT_GOAL := help
+REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
+include ${REPO_BASE_DIR}/scripts/common.Makefile
+
+
+
+# Internal VARIABLES ------------------------------------------------
+TF_STATE_FILE := terraform/.terraform/terraform.tfstate
+
+terraform/main.tf: terraform/main.tf.j2 .venv $(CI_ENV_FILE)
+	# generate $@
+	@$(call jinja, $<, $(CI_ENV_FILE), $@)
+	# validate and format $@
+
+terraform-init: $(TF_STATE_FILE)  ## init terraform
+
+$(TF_STATE_FILE): $(CI_ENV_FILE) terraform/main.tf
+	# terraform init
+	@set -a; source $<; set +a; \
+	if [ "$${TF_STATE_BACKEND_TYPE}" = "local" ]; then \
+	  terraform -chdir=./terraform init; \
+	else \
+	  terraform -chdir=./terraform init -backend-config="access_key=$${TF_BACKEND_ACCESS_KEY}" -backend-config="secret_key=$${TF_BACKEND_SECRET_KEY}"; \
+	fi
+
+terraform/plan.cache:
+	@echo "$@ file not found. Run 'make terraform-plan' to generate it."
+	@exit 1
+
+.PHONY: terraform-plan
+terraform-plan: $(CI_ENV_FILE) $(TF_STATE_FILE) ensure-grafana-online ## terraform plan
+	# terraform plan
+	@set -a; source $<; source $${REPO_CONFIG_LOCATION}; set +a; \
+    TF_VAR_aws_region=$${TF_AWS_REGION} TF_VAR_aws_access_key=$${TF_AWS_ACCESS_KEY_ID} TF_VAR_aws_secret_key=$${TF_AWS_SECRET_ACCESS_KEY} terraform -chdir=./terraform plan -out=plan.cache
+
+.PHONY: terraform-apply
+terraform-apply: $(CI_ENV_FILE) terraform/plan.cache $(TF_STATE_FILE) ensure-grafana-online ## terraform apply
+	# terraform apply
+	@set -a; source $<; set +a; \
+	TF_VAR_aws_region=$${TF_AWS_REGION} TF_VAR_aws_access_key=$${TF_AWS_ACCESS_KEY_ID} TF_VAR_aws_secret_key=$${TF_AWS_SECRET_ACCESS_KEY} terraform -chdir=./terraform apply plan.cache
+
+.PHONY: ensure-grafana-online
+ensure-grafana-online:
+	@set -o allexport; \
+	source $(REPO_CONFIG_LOCATION); \
+	set +o allexport; \
+	url=$${TF_VAR_GRAFANA_URL}; \
+	echo "Waiting for grafana at $$url to become reachable..."; \
+	while true; do \
+	    status_code=$$(curl -k -o /dev/null -s -w "%{http_code}" $$url); \
+	    if [ "$$status_code" -ge 200 ] && [ "$$status_code" -lt 400 ]; then \
+	        echo "Grafana is online"; \
+	        break; \
+	    else \
+	        echo "Grafana still unreachable, waiting 5s for grafana to become reachable..."; \
+	        sleep 5; \
+	    fi; \
+done;