Merge pull request #151 from devopshobbies/grafana/datasources

feat(grafana): Creation of Grafana data sources based on user inputs

app/main.py

@@ -5,4 +5,5 @@
 from app.routes.jcasc import *
 from app.routes.docker import *
 from app.routes.jenkins import *
-from app.routes.gitlab import *
+from app.routes.gitlab import *
+from app.routes.grafana_data_sources import *

app/media/MyGrafana/mimir.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+datasources:
+- name: Mimir
+  uid: mimir
+  type: prometheus
+  access: proxy
+  orgId: 1
+  url: http://mimir-nginx.mimir.svc.cluster.local/prometheus
+  editable: true
+  version: 1
+  jsonData:
+    alertmanagerUid: alertmanager

app/media/grafana_datasources/alertmanager.yml

@@ -0,0 +1,21 @@
+apiVersion: 1
+
+datasources:
+  - name: Alertmanager
+    uid: alertmanager
+    type: alertmanager
+    url: http://localhost:9093
+    access: proxy
+    orgId: 1
+    jsonData:
+      # Valid options for implementation include mimir, cortex and prometheus
+      implementation: [prometheus|cortex|mimir]
+      # Whether or not Grafana should send alert instances to this Alertmanager
+      handleGrafanaManagedAlerts: [false|true]
+
+    editable: [true|false]
+    # optionally
+    basicAuth: true
+    basicAuthUser: my_user
+    secureJsonData:
+      basicAuthPassword: test_password

app/media/grafana_datasources/elasticsearch.yml

@@ -0,0 +1,14 @@
+apiVersion: 1
+
+datasources:
+  - name: elasticsearch-v7-filebeat
+    type: elasticsearch
+    access: proxy
+    url: http://localhost:9200
+    editable: [true|false]
+    jsonData:
+      index: '[filebeat-]YYYY.MM.DD'
+      interval: Daily
+      timeField: '@timestamp'
+      logMessageField: message
+      logLevelField: fields.level

app/media/grafana_datasources/loki.yml

@@ -0,0 +1,19 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    uid: loki
+    type: loki
+    orgId: 1
+    access: proxy
+    editable: [true|false]
+    url: http://localhost:3100
+    jsonData:
+      timeout: 60
+      maxLines: 1000
+
+    # optionally
+    basicAuth: true
+    basicAuthUser: my_user
+    secureJsonData:
+      basicAuthPassword: test_password

app/media/grafana_datasources/mimir.yml

@@ -0,0 +1,16 @@
+apiVersion: 1
+
+datasources:
+  - name: Mimir
+    uid: mimir
+    type: prometheus
+    access: proxy
+    orgId: 1
+    url: http://mimir-nginx.mimir.svc.cluster.local/prometheus
+    editable: [true|false]
+    version: 1
+    jsonData:
+      httpHeaderName1: "X-Scope-OrgID"
+      alertmanagerUid: "alertmanager"
+    secureJsonData:
+      httpHeaderValue1: "pods"

app/media/grafana_datasources/mysql.yml

@@ -0,0 +1,20 @@
+apiVersion: 1
+
+datasources:
+  - name: MySQL
+    type: mysql
+    url: localhost:3306
+    user: grafana
+    editable: [true|false]
+    jsonData:
+      tlsAuth: true
+      tlsSkipVerify: true
+      database: grafana
+      maxOpenConns: 100 # Grafana v5.4+
+      maxIdleConns: 100 # Grafana v5.4+
+      maxIdleConnsAuto: true # Grafana v9.5.1+
+      connMaxLifetime: 14400 # Grafana v5.4+
+    secureJsonData:
+      password: ${GRAFANA_MYSQL_PASSWORD}
+      tlsClientCert: ${GRAFANA_TLS_CLIENT_CERT}
+      tlsCACert: ${GRAFANA_TLS_CA_CERT}

app/media/grafana_datasources/postgresql.yml

@@ -0,0 +1,19 @@
+apiVersion: 1
+
+datasources:
+  - name: Postgres
+    type: postgres
+    url: localhost:5432
+    user: grafana # Database user’s login/username
+    editable: [true|false]
+    secureJsonData:
+      password: 'Password!'
+    jsonData:
+      database: grafana
+      sslmode: 'disable' # disable/require/verify-ca/verify-full
+      maxOpenConns: 100 # Grafana v5.4+
+      maxIdleConns: 100 # Grafana v5.4+
+      maxIdleConnsAuto: true # Grafana v9.5.1+
+      connMaxLifetime: 14400 # Grafana v5.4+
+      postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10
+      timescaledb: false

app/media/grafana_datasources/prometheus.yml

@@ -0,0 +1,23 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    # Access mode - proxy (server in the UI) or direct (browser in the UI).
+    url: http://localhost:9090
+    editable: [true|false]
+    jsonData:
+      httpMethod: POST
+      manageAlerts: true
+      prometheusType: Prometheus
+      prometheusVersion: 2.44.0
+      cacheLevel: 'High'
+      disableRecordingRules: false
+      incrementalQueryOverlapWindow: 10m
+      exemplarTraceIdDestinations:
+        # Field with internal link pointing to data source in Grafana.
+        # datasourceUid value can be anything, but it should be unique across all defined data source uids.
+        - datasourceUid: my_jaeger_uid
+          name: traceID

app/media/grafana_datasources/tempo.yml

@@ -0,0 +1,25 @@
+apiVersion: 1
+
+datasources:
+  - name: Tempo
+    type: tempo
+    access: proxy
+    orgId: 1
+    url: http://tempo-query-frontend.tempo.svc.cluster.local:3100
+    basicAuth: false
+    version: 1
+    editable: true
+    apiVersion: 1
+    uid: tempo
+    jsonData:
+      httpMethod: GET
+      tracesToLogsV2: # If you are going to link your tracing data with logs, configure <<tracesToLogsV2>>
+        datasourceUid: 'loki'
+        spanStartTimeShift: '-2m'
+        spanEndTimeShift: '2m'
+        filterByTraceID: true
+        filterBySpanID: true
+      serviceMap: # If you are going to add serviceGraph feature to tempo, configure <<serviceMap.datasourceUid>>
+        datasourceUid: 'Mimir-OtelMetrics-Tenant'
+      nodeGraph: # If you are going to add nodeGraph feature to tempo, enable <<nodeGraph>>
+        enabled: true

app/models/__init__.py

@@ -6,4 +6,12 @@
 from .compose_models import *
 from .docker_installation_models import *
 from .jenkins import *
-from .gitlab_models import *
+from .gitlab_models import *
+from app.models.grafana.alert_managers_models import *
+from app.models.grafana.elastcsearch_models import *
+from app.models.grafana.loki_models import *
+from app.models.grafana.mimir_models import *
+from app.models.grafana.mysql_models import *
+from app.models.grafana.postgresql_models import *
+from app.models.grafana.prometheus_models import *
+from app.models.grafana.tempo_models import *

app/models/grafana/alert_managers_models.py

@@ -0,0 +1,28 @@
+from typing import List, Optional
+from pydantic import BaseModel, validator, ValidationError
+
+class BasicAuth(BaseModel):
+    basicAuthUser:str
+    basicAuthPassword:str
+
+
+class AlertManagerInput(BaseModel):
+    name:str =  "Alertmanager"
+    url:str = "http://localhost:9093"
+    uid:str = "alertmanager"
+    implementation:str 
+
+    handleGrafanaManagedAlerts:bool = True
+    editable: bool = True
+    basic_auth:Optional[BasicAuth]
+
+    @validator("implementation")
+    def validator_implementation(cls,value):
+        valid = ['prometheus','cortex','mimir']
+        if value not in valid:
+            raise ValueError(f"implementation must be in {valid}")
+        return value
+
+
+
+

app/models/grafana/elastcsearch_models.py

@@ -0,0 +1,21 @@
+from typing import List, Optional
+from pydantic import BaseModel, validator, ValidationError
+
+
+
+class ElasticSearchInput(BaseModel):
+    name:str = "elasticsearch-v7-filebeat"
+    url:str = "http://localhost:9200" 
+    editable: bool = True
+    index:str = "[filebeat-]YYYY.MM.DD"
+    interval:str = "Daily"
+    timeField:str = "@timestamp"
+    logMessageField:str = "message"
+    logLevelField:str = "fields.level"
+
+
+
+
+
+
+

app/models/grafana/loki_models.py

@@ -0,0 +1,20 @@
+from typing import List, Optional
+from pydantic import BaseModel, validator, ValidationError
+
+class BasicAuth(BaseModel):
+    basicAuthUser:str
+    basicAuthPassword:str
+
+class LokiInput(BaseModel):
+    name:str = "Loki"
+    uid:str = "loki"
+    url:str = "http://localhost:3100"
+    editable: bool = True
+    timeout:int = 60
+    maxLines:int = 1000
+    basic_auth:Optional[BasicAuth]
+
+
+
+
+

app/models/grafana/mimir_models.py

@@ -0,0 +1,17 @@
+from typing import List, Optional
+from pydantic import BaseModel, validator, ValidationError
+
+
+
+class MultiTenancy(BaseModel):
+    tenant_name:str = "pods"
+    httpHeaderName1:str = "X-Scope-OrgID"
+class MimirInput(BaseModel):
+    name:str = "Mimir"
+    uid:str = "mimir"
+    url:str = "http://mimir-nginx.mimir.svc.cluster.local/prometheus"
+    editable: bool = True
+    alertmanagerUid:str = "alertmanager"
+    multi_tenancy:Optional[MultiTenancy]
+
+

app/models/grafana/mysql_models.py

@@ -0,0 +1,25 @@
+from typing import List, Optional
+from pydantic import BaseModel, validator, ValidationError
+
+
+
+class TLS(BaseModel):
+        tlsClientCert:str = "${GRAFANA_TLS_CLIENT_CERT}"
+        tlsCACert:str =  "${GRAFANA_TLS_CA_CERT}"
+        tlsAuth:bool =  True
+        tlsSkipVerify:bool = True
+
+class MysqlInput(BaseModel):
+    name:str = "MySQL"
+    url:str = "localhost:3306"
+    user:str = "grafana"
+    editable: bool = True
+    database:str = "grafana"
+    maxOpenConns:int = 100
+    maxIdleConns:int = 100
+    maxIdleConnsAuto:bool = True 
+    connMaxLifetime:int = 14400 
+    password:str = "${GRAFANA_MYSQL_PASSWORD}"
+    tls :Optional[TLS]
+
+

app/models/grafana/postgresql_models.py

@@ -0,0 +1,22 @@
+from typing import List, Optional
+from pydantic import BaseModel, validator, ValidationError
+
+
+
+
+class PostgresInput(BaseModel):
+    name:str = "Postgres"
+    url:str = "localhost:5432"
+    user:str = "grafana"
+    editable: bool = True
+    database:str = "grafana"
+    sslmode:str = "'disable'"
+    password:str = "Password!"
+    maxOpenConns:int = 100 
+    maxIdleConns:int = 100
+    maxIdleConnsAuto:bool = True 
+    connMaxLifetime:int = 14400 
+    postgresVersion:int = 903
+    timescaledb:bool = False
+
+

app/models/grafana/prometheus_models.py

@@ -0,0 +1,17 @@
+from typing import List, Optional
+from pydantic import BaseModel, validator, ValidationError,field_validator
+
+
+
+class PrometheusInput(BaseModel):
+    name:str = "Prometheus"
+    url:str = "http://localhost:9090"
+    editable: bool = True
+    httpMethod:str = "POST"
+    manageAlerts:bool = True
+    prometheusType:str = "Prometheus"
+    prometheusVersion:str = "2.44.0"
+    cacheLevel:str = "High"
+    disableRecordingRules:bool = False
+    incrementalQueryOverlapWindow:str = "10m"
+