diff --git a/Core DW Infrastructure/app/streamlitdw_fe.py b/Core DW Infrastructure/app/streamlitdw_fe.py index 9fc4020..65dba76 100644 --- a/Core DW Infrastructure/app/streamlitdw_fe.py +++ b/Core DW Infrastructure/app/streamlitdw_fe.py @@ -112,7 +112,7 @@ def upload_to_minio(file, filename, bucket_name, project, preprocessing_option): def log_to_elasticsearch(log_data): try: - response = requests.post("http://logstash:5044", json=log_data) + response = requests.post("http://dp-logstash:5044", json=log_data) if response.status_code != 200: print(f"Failed to log to Logstash: {response.text}") except Exception as e: diff --git a/Core DW Infrastructure/docker-compose.yml b/Core DW Infrastructure/docker-compose.yml index c1eb9b1..cdef41d 100644 --- a/Core DW Infrastructure/docker-compose.yml +++ b/Core DW Infrastructure/docker-compose.yml @@ -80,9 +80,9 @@ services: ports: - 5432:5432 volumes: - - postgres-data:/var/lib/postgresql/data + - dp-postgres-data:/var/lib/postgresql/data - ./db-init:/docker-entrypoint-initdb.d - container_name: postgres + container_name: dp-postgres networks: - dw_network @@ -95,8 +95,8 @@ services: - 9200:9200 - 9300:9300 volumes: - - es-data:/usr/share/elasticsearch/data - container_name: elasticsearch + - dp-es-data:/usr/share/elasticsearch/data + container_name: dp-elasticsearch networks: - dw_network @@ -104,7 +104,7 @@ services: image: docker.elastic.co/kibana/kibana:7.10.1 ports: - 5601:5601 - container_name: kibana + container_name: dp-kibana volumes: - ./kibana_config/kibana.yml:/usr/share/kibana/config/kibana.yml networks: @@ -115,7 +115,7 @@ services: context: ./logstash volumes: - ./logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf - - logstash-data:/usr/share/logstash/data + - dp-logstash-data:/usr/share/logstash/data ports: - 5044:5044 environment: @@ -123,7 +123,7 @@ services: LOG_LEVEL: debug env_file: - ./logstash/.env - container_name: logstash + container_name: dp-logstash networks: - dw_network @@ -134,11 +134,11 @@ volumes: external: true fileuploadservice_dremio-data: external: true - postgres-data: + dp-postgres-data: external: true - es-data: + dp-es-data: external: true - logstash-data: + dp-logstash-data: external: true networks: diff --git a/Core DW Infrastructure/kibana_config/kibana.yml b/Core DW Infrastructure/kibana_config/kibana.yml index 977c38c..7aab444 100644 --- a/Core DW Infrastructure/kibana_config/kibana.yml +++ b/Core DW Infrastructure/kibana_config/kibana.yml @@ -1,3 +1,3 @@ server.name: kibana server.host: "0.0.0.0" -elasticsearch.hosts: ["http://elasticsearch:9200"] \ No newline at end of file +elasticsearch.hosts: ["http://dp-elasticsearch:9200"] \ No newline at end of file diff --git a/Core DW Infrastructure/logstash/logstash.conf b/Core DW Infrastructure/logstash/logstash.conf index a90af1a..5add73c 100644 --- a/Core DW Infrastructure/logstash/logstash.conf +++ b/Core DW Infrastructure/logstash/logstash.conf @@ -48,7 +48,7 @@ output { # Directly pass logs from file upload service to Elasticsearch if [source] == "upload_service" { elasticsearch { - hosts => ["http://elasticsearch:9200"] + hosts => ["http://dp-elasticsearch:9200"] index => "fileupload-%{+YYYY.MM.dd}" } } @@ -69,12 +69,12 @@ output { ] } elasticsearch { - hosts => ["http://elasticsearch:9200"] + hosts => ["http://dp-elasticsearch:9200"] index => "minio-%{+YYYY.MM.dd}" } } else if [event_type] =~ /^s3:ObjectAccessed:/ or [event_type] =~ /^s3:Error:/ or [event_type] =~ /^s3:Audit:/ or [event_type] =~ /^s3:BucketPolicy:/ or [event_type] =~ /^s3:BucketNotification:/ or [event_type] in ["s3:BucketCreated", "s3:BucketRemoved"]{ elasticsearch { - hosts => ["http://elasticsearch:9200"] + hosts => ["http://dp-elasticsearch:9200"] index => "minio-%{+YYYY.MM.dd}" } } else if [event_type] =~ /^s3:ObjectRestore:/ or [event_type] =~ /^s3:ObjectTransition:/ {