-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
118 lines (99 loc) · 4.9 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
# FIXME : verifier que fichier avec variables d'env respecte les regles pour Makefile car sourcé par lui
SHELL := /bin/bash
ENV_FILE = .env
include_env = $(wildcard $(ENV_FILE))
ifneq ($(include_env),)
include .env
endif
up: setup
$(MAKE) data_copy
$(MAKE) dashboard
@touch .env_started
setup:
@mkdir -p volumes/elastic/data volumes/elastic/logs volumes/shiny volumes/sqlite
# FIXME : version de chargement des donnes avec data.json envoye par email
data_copy: .elasticsearch_started
@echo "Mapping of cmdb index" && curl -XPUT "http://localhost:9200/cmdb" -k -u ${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD} -H "Content-Type: application/json" -d @./prod_to_dev/mapping_cmdb.json
@echo "\nPull elasticdump image" && docker pull elasticdump/elasticsearch-dump
@echo "Load cmdb index in elasticsearch" && docker run --net=host --rm -ti -v ./prod_to_dev/internal_data:/tmp elasticdump/elasticsearch-dump \
--input=/tmp/cmdb.json \
--output=http://${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD}@localhost:9200/cmdb \
--type=data
@echo "Load ssl index in elasticsearch" && docker run --net=host --rm -ti -v ./prod_to_dev/internal_data:/tmp elasticdump/elasticsearch-dump \
--input=/tmp/ssl.json \
--output=http://${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD}@localhost:9200/ssl \
--type=data
@curl -XPUT "http://localhost:9200/_settings" -k -u ${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD} -H "Content-Type: application/json" -d '{"index.max_result_window": 1000000}'
$(MAKE) nosql_into_sql
# TODO : version pour les deux methodes selon var d'env a tester
data: .elasticsearch_started
# FIXME : mapping necessaire ou non pour cmdb ?
@echo "Mapping of cmdb index" && curl -XPUT "http://localhost:9200/cmdb" -k -u ${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD} -H "Content-Type: application/json" -d @./prod_to_dev/mapping_cmdb.json
@echo "\nPull elasticdump image" && docker pull elasticdump/elasticsearch-dump
@echo "Load cmdb index in elasticsearch" && docker run --net=host --rm -ti -v ./prod_to_dev/internal_data:/tmp elasticdump/elasticsearch-dump \
--input=${SOURCE_CMDB} \
--output=http://${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD}@localhost:9200/cmdb \
--type=data
@echo "Load ssl index in elasticsearch" && docker run --net=host --rm -ti -v ./prod_to_dev/internal_data:/tmp elasticdump/elasticsearch-dump \
--input=${SOURCE_SSL} \
--output=http://${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD}@localhost:9200/ssl \
--type=data
@curl -XPUT "http://localhost:9200/_settings" -k -u ${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD} -H "Content-Type: application/json" -d '{"index.max_result_window": 1000000}'
$(MAKE) nosql_into_sql
nosql_into_sql:
cp cmdb_schema.sqlite ./volumes/sqlite/cmdb.sqlite
@ echo "Install 'here' R package" && R -e "install.packages(\"here\")"
@ echo "Load data from elasticsearch into sqlite" && Rscript add_cmdb_data.R
dashboard:
docker compose up -d cert_dashboard
#docker compose logs -f cert_dashboard
.elasticsearch_started:
$(MAKE) elasticsearch
echo "Waiting for elasticsearch to be ready"; \
while [ "$$(curl -s -o /dev/null -w '%{http_code}' -u ${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD} -XGET "localhost:9200/")" != "200" ]; do \
sleep 5; \
echo "."; \
done
@touch .elasticsearch_started
elasticsearch: vm-max_map_count
@docker compose up -d elasticsearch
#docker compose logs -f elasticsearch
vm-max_map_count:
@if [ "$$(uname)" = "Linux" ]; then \
sudo sysctl -w vm.max_map_count=262144 1>/dev/null && echo "vm.max_map_count changed"; \
fi
logs:
docker compose logs -f
# FIXME : utile ou non (plus si bug fixe) ?
elasticsearch_healthy:
watch -n 0.5 curl -u ${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD} -X GET "localhost:9200/_cluster/health?pretty"
kibana: .elasticsearch_started .kibana_token_available
docker compose up -d kibana
#docker compose logs -f kibana
.kibana_token_available:
$(MAKE) token
@touch .kibana_token_available
token:
echo "Waiting for elasticsearch to be ready"; \
while [ "$$(curl -s -o /dev/null -w '%{http_code}' -u ${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD} -XGET "localhost:9200/_security/_authenticate")" != "200" ]; do \
sleep 5; \
echo "..."; \
done
echo -e -n "ELASTICSEARCH_TOKEN=" >> .env && \
curl -X POST -u ${ELASTICSEARCH_USER}:${ELASTICSEARCH_PASSWORD} "localhost:9200/_security/service/elastic/kibana/credential/token/token1?pretty" | jq '.token'.'value' >> .env
# FIXME : .env ko si generation du token plusieurs fois...
clean:
docker compose stop
sed -i '/ELASTICSEARCH_TOKEN/d' .env
rm -rf ./volumes
rm -f .env_started .elasticsearch_started .kibana_token_available
# FIXME : toujours necessaire ?
reformat_ssl_json:
chmod +x ./prod_to_dev/reformat_json.bash
./prod_to_dev/reformat_json.bash
rm ./prod_to_dev/ssl.json
rm ./prod_to_dev/temp_ssl.json
mv ./prod_to_dev/formated_ssl.json ./prod_to_dev/ssl.json
# FIXME : besoin de tester le dashboard en dehors du docker pendant dev ?
test_dashboard:
Rscript wip_dashboard.R