diff --git a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js index d7a2958353..fa1a23a3de 100644 --- a/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js +++ b/ansible/artifacts/sunbird/login/resources/js/telemetry_service.js @@ -2506,7 +2506,7 @@ if(client_id.toLowerCase() === 'android'){ "telemetry": { "pdata": { "id": pdataId, - "ver": "4.9.0", + "ver": "4.10.3", "pid": "sunbird-portal" } } diff --git a/ansible/es.yml b/ansible/es.yml index 3aa369f112..8042e5dbfa 100644 --- a/ansible/es.yml +++ b/ansible/es.yml @@ -24,11 +24,11 @@ tags: - log_es_backup -- hosts: log-es - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - roles: - - es2-data-purge - tags: - - purge_old_logs +# - hosts: log-es +# become: yes +# vars_files: +# - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] +# roles: +# - es2-data-purge +# tags: +# - purge_old_logs diff --git a/ansible/graylog-mongodb-import.yml b/ansible/graylog-mongodb-import.yml index fcdac42ca3..a0ffff1776 100644 --- a/ansible/graylog-mongodb-import.yml +++ b/ansible/graylog-mongodb-import.yml @@ -1,13 +1,10 @@ - hosts: graylog become: yes pre_tasks: - - name: Populate service facts - ansible.builtin.service_facts: - name: stop graylog systemd: name: graylog-server state: stopped - when: "'graylog-server' in ansible_facts.services" - name: wait for graylog to stop completely wait_for: @@ -26,7 +23,6 @@ systemd: name: graylog-server state: started - when: "'graylog-server' in ansible_facts.services" - name: Wait for graylog to startup uri: @@ -36,4 +32,3 @@ until: _result.status == 200 delay: 15 retries: 15 - when: "'graylog-server' in ansible_facts.services" diff --git a/ansible/graylog.yml b/ansible/graylog.yml index 17d5efc686..40b00964b6 100644 --- a/ansible/graylog.yml +++ b/ansible/graylog.yml @@ -3,11 +3,11 @@ vars_files: - "{{inventory_dir}}/secrets.yml" vars: - mongodb_version: "4.0" + mongodb_version: "4.4" mongodb_replication_replset: graylog mongodb_net_bindip: "127.0.0.1,{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}" graylog_version: "4.2" - graylog_full_version: "4.2.6-1" + graylog_full_version: "4.2.9-1" mongodb_login_host: "{{ groups['graylog'][0] }}" host_name: "{{ groups['graylog'][0] }}" graylog_install_elasticsearch: false diff --git a/ansible/roles/es-mapping/files/indices/orgv2.json b/ansible/roles/es-mapping/files/indices/orgv3.json similarity index 99% rename from ansible/roles/es-mapping/files/indices/orgv2.json rename to ansible/roles/es-mapping/files/indices/orgv3.json index 0a44cddb42..6f5dabc867 100644 --- a/ansible/roles/es-mapping/files/indices/orgv2.json +++ b/ansible/roles/es-mapping/files/indices/orgv3.json @@ -47,4 +47,4 @@ "aliases" : { "org_alias" : {} } -} +} \ No newline at end of file diff --git a/ansible/roles/es-mapping/files/mappings/orgv2-mapping.json b/ansible/roles/es-mapping/files/mappings/orgv3-mapping.json similarity index 99% rename from ansible/roles/es-mapping/files/mappings/orgv2-mapping.json rename to ansible/roles/es-mapping/files/mappings/orgv3-mapping.json index d4e1628919..5f952a7940 100644 --- a/ansible/roles/es-mapping/files/mappings/orgv2-mapping.json +++ b/ansible/roles/es-mapping/files/mappings/orgv3-mapping.json @@ -158,7 +158,7 @@ "analyzer": "keylower" } }, - "analyzer": "cs_search_analyzer", + "analyzer": "cs_index_analyzer", "search_analyzer": "cs_search_analyzer" }, "orgType": { diff --git a/ansible/roles/graylog-mongodb-import/defaults/main.yml b/ansible/roles/graylog-mongodb-import/defaults/main.yml index 32d6523690..2b144dd894 100644 --- a/ansible/roles/graylog-mongodb-import/defaults/main.yml +++ b/ansible/roles/graylog-mongodb-import/defaults/main.yml @@ -1,4 +1,4 @@ graylog_mongo_db: "graylog" mongo_master: "{{ groups['graylog'][0] }}" mongo_json_data: /tmp/mongo_json_data -graylog_mongo_collections: "cluster_config,content_packs,content_packs_installations,event_definitions,event_notifications,event_notification_status,grants,grok_patterns,index_sets,inputs,lut_caches,lut_data_adapters,lut_tables,pipeline_processor_pipelines,pipeline_processor_pipelines_streams,pipeline_processor_rules,roles,scheduler_job_definitions,scheduler_triggers,searches,streamrules,streams,users,views" +graylog_mongo_collections: "cluster_config,content_packs,content_packs_installations,event_definitions,event_notifications,grants,grok_patterns,index_sets,inputs,lut_caches,lut_data_adapters,lut_tables,pipeline_processor_pipelines,pipeline_processor_pipelines_streams,pipeline_processor_rules,roles,scheduler_job_definitions,scheduler_triggers,searches,streamrules,streams,users,views" diff --git a/ansible/roles/graylog-mongodb-import/templates/event_notifications.json b/ansible/roles/graylog-mongodb-import/templates/event_notifications.json index 796025906d..7aad9a2f10 100644 --- a/ansible/roles/graylog-mongodb-import/templates/event_notifications.json +++ b/ansible/roles/graylog-mongodb-import/templates/event_notifications.json @@ -1,3 +1,3 @@ -{"_id":{"$oid":"60af7dfaa650154320a12aa9"},"title":"Functional Alert","description":"Functional Alert","config":{"type":"email-notification-v1","sender":"{{ sunbird_mail_server_from_email }}","subject":"{{ env }} | ${event.fields.dimensions_id} | ${event.fields.dimensions_value}","body_template":"--- [Event] --------------------------------------\nTimestamp: ${event.timestamp}\nMessage: ${event.message}\nSource: ${event.source}\nKey: ${event.key}\nPriority: ${event.priority}\nAlert: ${event.alert}\nFields:\n${foreach event.fields field} ${field.key}: ${field.value}\n${end}\n${if backlog}\n--- [Backlog] ------------------------------------\nLast messages accounting for this alert:\n${foreach backlog message}\n${message}\n${end}\n${end}\n","html_body_template":"\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr style=\"line-height:1.5\"\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6\"\u003eEvent Definition\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTitle\u003c/td\u003e\u003ctd\u003e${event_definition_title}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDescription\u003c/td\u003e\u003ctd\u003e${event_definition_description}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eType\u003c/td\u003e\u003ctd\u003e${event_definition_type}\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6;line-height:1.5\"\u003eEvent\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTimestamp\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eMessage\u003c/td\u003e\u003ctd\u003e${event.message}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource\u003c/td\u003e\u003ctd\u003e${event.source}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eKey\u003c/td\u003e\u003ctd\u003e${event.key}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003ePriority\u003c/td\u003e\u003ctd\u003e${event.priority}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eAlert\u003c/td\u003e\u003ctd\u003e${event.alert}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimestamp Processing\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange Start\u003c/td\u003e\u003ctd\u003e${event.timerange_start}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange End\u003c/td\u003e\u003ctd\u003e${event.timerange_end}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource Streams\u003c/td\u003e\u003ctd\u003e${event.source_streams}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eFields\u003c/td\u003e\u003ctd\u003e\u003cul style=\"list-style-type:square;\"\u003e${foreach event.fields field}\u003cli\u003e${field.key}:${field.value}\u003c/li\u003e${end}\u003cul\u003e\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n${if backlog}\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth style=\"background-color:#e6e6e6;line-height:1.5\"\u003eBacklog (Last messages accounting for this alert)\u003c/th\u003e\u003c/tr\u003e\n${foreach backlog message}\n\u003ctr\u003e\u003ctd\u003e${message}\u003c/td\u003e\u003c/tr\u003e\n${end}\n\u003c/tbody\u003e\u003c/table\u003e\n${end}\n","email_recipients":{{ (graylog_alerts_mailing_list|default(alerts_mailing_list)).split(',') | map('trim') | list }},"user_recipients":[]}} -{"_id":{"$oid":"60e8311025aa97080bea8c8b"},"title":"DataProducts Notification","description":"DP Notification","config":{"type":"email-notification-v1","sender":"{{ sunbird_mail_server_from_email }}","subject":"{{ env }} | DataProduct notification: ${event.fields.DP_name} | ${event.fields.DP_status}","body_template":"--- [Event] --------------------------------------\nTimestamp: ${event.timestamp}\nMessage: ${event.message}\nSource: ${event.source}\nKey: ${event.key}\nPriority: ${event.priority}\nAlert: ${event.alert}\nFields:\n${foreach event.fields field} ${field.key}: ${field.value}\n${end}\n${if backlog}\n--- [Backlog] ------------------------------------\nLast messages accounting for this alert:\n${foreach backlog message}\n${message}\n${end}\n${end}\n","html_body_template":"\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr style=\"line-height:1.5\"\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6\"\u003eEvent Definition\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTitle\u003c/td\u003e\u003ctd\u003e${event_definition_title}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDescription\u003c/td\u003e\u003ctd\u003e${event_definition_description}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eType\u003c/td\u003e\u003ctd\u003e${event_definition_type}\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6;line-height:1.5\"\u003eEvent\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTimestamp\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eMessage\u003c/td\u003e\u003ctd\u003e${event.message}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource\u003c/td\u003e\u003ctd\u003e${event.source}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eKey\u003c/td\u003e\u003ctd\u003e${event.key}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003ePriority\u003c/td\u003e\u003ctd\u003e${event.priority}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eAlert\u003c/td\u003e\u003ctd\u003e${event.alert}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimestamp Processing\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange Start\u003c/td\u003e\u003ctd\u003e${event.timerange_start}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange End\u003c/td\u003e\u003ctd\u003e${event.timerange_end}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource Streams\u003c/td\u003e\u003ctd\u003e${event.source_streams}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eFields\u003c/td\u003e\u003ctd\u003e\u003cul style=\"list-style-type:square;\"\u003e${foreach event.fields field}\u003cli\u003e${field.key}:${field.value}\u003c/li\u003e${end}\u003cul\u003e\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n${if backlog}\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth style=\"background-color:#e6e6e6;line-height:1.5\"\u003eBacklog (Last messages accounting for this alert)\u003c/th\u003e\u003c/tr\u003e\n${foreach backlog message}\n\u003ctr\u003e\u003ctd\u003e${message}\u003c/td\u003e\u003c/tr\u003e\n${end}\n\u003c/tbody\u003e\u003c/table\u003e\n${end}\n","email_recipients":{{ (graylog_alerts_mailing_list|default(alerts_mailing_list)).split(',') | map('trim') | list }},"user_recipients":[]}} -{"_id":{"$oid":"6182707560a469693765f391"},"title":"DataProducts Output Size Notification","description":"DP Output File Size Notification","config":{"type":"email-notification-v1","sender":"{{ sunbird_mail_server_from_email }}","subject":"{{ env }} | DataProduct output file size notification: ${event.fields.DP_name} | ${event.fields.DP_output_file_size}","body_template":"--- [Event] --------------------------------------\nTimestamp: ${event.timestamp}\nMessage: ${event.message}\nSource: ${event.source}\nKey: ${event.key}\nPriority: ${event.priority}\nAlert: ${event.alert}\nFields:\n${foreach event.fields field} ${field.key}: ${field.value}\n${end}\n${if backlog}\n--- [Backlog] ------------------------------------\nLast messages accounting for this alert:\n${foreach backlog message}\n${message}\n${end}\n${end}\n","html_body_template":"\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr style=\"line-height:1.5\"\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6\"\u003eEvent Definition\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTitle\u003c/td\u003e\u003ctd\u003e${event_definition_title}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDescription\u003c/td\u003e\u003ctd\u003e${event_definition_description}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eType\u003c/td\u003e\u003ctd\u003e${event_definition_type}\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6;line-height:1.5\"\u003eEvent\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTimestamp\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eMessage\u003c/td\u003e\u003ctd\u003e${event.message}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource\u003c/td\u003e\u003ctd\u003e${event.source}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eKey\u003c/td\u003e\u003ctd\u003e${event.key}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003ePriority\u003c/td\u003e\u003ctd\u003e${event.priority}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eAlert\u003c/td\u003e\u003ctd\u003e${event.alert}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimestamp Processing\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange Start\u003c/td\u003e\u003ctd\u003e${event.timerange_start}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange End\u003c/td\u003e\u003ctd\u003e${event.timerange_end}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource Streams\u003c/td\u003e\u003ctd\u003e${event.source_streams}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eFields\u003c/td\u003e\u003ctd\u003e\u003cul style=\"list-style-type:square;\"\u003e${foreach event.fields field}\u003cli\u003e${field.key}:${field.value}\u003c/li\u003e${end}\u003cul\u003e\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n${if backlog}\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth style=\"background-color:#e6e6e6;line-height:1.5\"\u003eBacklog (Last messages accounting for this alert)\u003c/th\u003e\u003c/tr\u003e\n${foreach backlog message}\n\u003ctr\u003e\u003ctd\u003e${message}\u003c/td\u003e\u003c/tr\u003e\n${end}\n\u003c/tbody\u003e\u003c/table\u003e\n${end}\n","email_recipients":{{ (graylog_alerts_mailing_list|default(alerts_mailing_list)).split(',') | map('trim') | list }},"user_recipients":[]}} \ No newline at end of file +{"_id":{"$oid":"60af7dfaa650154320a12aa9"},"title":"Functional Alert","description":"Functional Alert","config":{"type":"email-notification-v1","sender":"{{ sunbird_mail_server_from_email }}","subject":"{{ env }} | ${event.fields.dimensions_id} | ${event.fields.dimensions_value}","body_template":"--- [Event] --------------------------------------\nTimestamp: ${event.timestamp}\nMessage: ${event.message}\nSource: ${event.source}\nKey: ${event.key}\nPriority: ${event.priority}\nAlert: ${event.alert}\nFields:\n${foreach event.fields field} ${field.key}: ${field.value}\n${end}\n${if backlog}\n--- [Backlog] ------------------------------------\nLast messages accounting for this alert:\n${foreach backlog message}\n${message}\n${end}\n${end}\n","html_body_template":"\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr style=\"line-height:1.5\"\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6\"\u003eEvent Definition\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTitle\u003c/td\u003e\u003ctd\u003e${event_definition_title}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDescription\u003c/td\u003e\u003ctd\u003e${event_definition_description}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eType\u003c/td\u003e\u003ctd\u003e${event_definition_type}\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6;line-height:1.5\"\u003eEvent\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTimestamp\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eMessage\u003c/td\u003e\u003ctd\u003e${event.message}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource\u003c/td\u003e\u003ctd\u003e${event.source}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eKey\u003c/td\u003e\u003ctd\u003e${event.key}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003ePriority\u003c/td\u003e\u003ctd\u003e${event.priority}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eAlert\u003c/td\u003e\u003ctd\u003e${event.alert}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimestamp Processing\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange Start\u003c/td\u003e\u003ctd\u003e${event.timerange_start}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange End\u003c/td\u003e\u003ctd\u003e${event.timerange_end}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource Streams\u003c/td\u003e\u003ctd\u003e${event.source_streams}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eFields\u003c/td\u003e\u003ctd\u003e\u003cul style=\"list-style-type:square;\"\u003e${foreach event.fields field}\u003cli\u003e${field.key}:${field.value}\u003c/li\u003e${end}\u003cul\u003e\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n${if backlog}\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth style=\"background-color:#e6e6e6;line-height:1.5\"\u003eBacklog (Last messages accounting for this alert)\u003c/th\u003e\u003c/tr\u003e\n${foreach backlog message}\n\u003ctr\u003e\u003ctd\u003e${message}\u003c/td\u003e\u003c/tr\u003e\n${end}\n\u003c/tbody\u003e\u003c/table\u003e\n${end}\n","email_recipients":{{ (graylog_alerts_mailing_list|default(alerts_mailing_list)).split(',') | to_json | replace(" ", "") }},"user_recipients":[]}} +{"_id":{"$oid":"60e8311025aa97080bea8c8b"},"title":"DataProducts Notification","description":"DP Notification","config":{"type":"email-notification-v1","sender":"{{ sunbird_mail_server_from_email }}","subject":"{{ env }} | DataProduct notification: ${event.fields.DP_name} | ${event.fields.DP_status}","body_template":"--- [Event] --------------------------------------\nTimestamp: ${event.timestamp}\nMessage: ${event.message}\nSource: ${event.source}\nKey: ${event.key}\nPriority: ${event.priority}\nAlert: ${event.alert}\nFields:\n${foreach event.fields field} ${field.key}: ${field.value}\n${end}\n${if backlog}\n--- [Backlog] ------------------------------------\nLast messages accounting for this alert:\n${foreach backlog message}\n${message}\n${end}\n${end}\n","html_body_template":"\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr style=\"line-height:1.5\"\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6\"\u003eEvent Definition\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTitle\u003c/td\u003e\u003ctd\u003e${event_definition_title}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDescription\u003c/td\u003e\u003ctd\u003e${event_definition_description}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eType\u003c/td\u003e\u003ctd\u003e${event_definition_type}\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6;line-height:1.5\"\u003eEvent\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTimestamp\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eMessage\u003c/td\u003e\u003ctd\u003e${event.message}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource\u003c/td\u003e\u003ctd\u003e${event.source}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eKey\u003c/td\u003e\u003ctd\u003e${event.key}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003ePriority\u003c/td\u003e\u003ctd\u003e${event.priority}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eAlert\u003c/td\u003e\u003ctd\u003e${event.alert}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimestamp Processing\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange Start\u003c/td\u003e\u003ctd\u003e${event.timerange_start}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange End\u003c/td\u003e\u003ctd\u003e${event.timerange_end}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource Streams\u003c/td\u003e\u003ctd\u003e${event.source_streams}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eFields\u003c/td\u003e\u003ctd\u003e\u003cul style=\"list-style-type:square;\"\u003e${foreach event.fields field}\u003cli\u003e${field.key}:${field.value}\u003c/li\u003e${end}\u003cul\u003e\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n${if backlog}\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth style=\"background-color:#e6e6e6;line-height:1.5\"\u003eBacklog (Last messages accounting for this alert)\u003c/th\u003e\u003c/tr\u003e\n${foreach backlog message}\n\u003ctr\u003e\u003ctd\u003e${message}\u003c/td\u003e\u003c/tr\u003e\n${end}\n\u003c/tbody\u003e\u003c/table\u003e\n${end}\n","email_recipients":{{ (graylog_alerts_mailing_list|default(alerts_mailing_list)).split(',') | to_json | replace(" ", "") }},"user_recipients":[]}} +{"_id":{"$oid":"6182707560a469693765f391"},"title":"DataProducts Output Size Notification","description":"DP Output File Size Notification","config":{"type":"email-notification-v1","sender":"{{ sunbird_mail_server_from_email }}","subject":"{{ env }} | DataProduct output file size notification: ${event.fields.DP_name} | ${event.fields.DP_output_file_size}","body_template":"--- [Event] --------------------------------------\nTimestamp: ${event.timestamp}\nMessage: ${event.message}\nSource: ${event.source}\nKey: ${event.key}\nPriority: ${event.priority}\nAlert: ${event.alert}\nFields:\n${foreach event.fields field} ${field.key}: ${field.value}\n${end}\n${if backlog}\n--- [Backlog] ------------------------------------\nLast messages accounting for this alert:\n${foreach backlog message}\n${message}\n${end}\n${end}\n","html_body_template":"\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr style=\"line-height:1.5\"\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6\"\u003eEvent Definition\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTitle\u003c/td\u003e\u003ctd\u003e${event_definition_title}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eDescription\u003c/td\u003e\u003ctd\u003e${event_definition_description}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eType\u003c/td\u003e\u003ctd\u003e${event_definition_type}\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth colspan=\"2\" style=\"background-color:#e6e6e6;line-height:1.5\"\u003eEvent\u003c/th\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd width=\"200px\"\u003eTimestamp\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eMessage\u003c/td\u003e\u003ctd\u003e${event.message}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource\u003c/td\u003e\u003ctd\u003e${event.source}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eKey\u003c/td\u003e\u003ctd\u003e${event.key}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003ePriority\u003c/td\u003e\u003ctd\u003e${event.priority}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eAlert\u003c/td\u003e\u003ctd\u003e${event.alert}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimestamp Processing\u003c/td\u003e\u003ctd\u003e${event.timestamp}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange Start\u003c/td\u003e\u003ctd\u003e${event.timerange_start}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eTimerange End\u003c/td\u003e\u003ctd\u003e${event.timerange_end}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eSource Streams\u003c/td\u003e\u003ctd\u003e${event.source_streams}\u003c/td\u003e\u003c/tr\u003e\n\u003ctr\u003e\u003ctd\u003eFields\u003c/td\u003e\u003ctd\u003e\u003cul style=\"list-style-type:square;\"\u003e${foreach event.fields field}\u003cli\u003e${field.key}:${field.value}\u003c/li\u003e${end}\u003cul\u003e\u003c/td\u003e\u003c/tr\u003e\n\u003c/tbody\u003e\u003c/table\u003e\n${if backlog}\n\u003cbr /\u003e\u003ctable width=\"100%\" border=\"0\" cellpadding=\"10\" cellspacing=\"0\" style=\"background-color:#f9f9f9;border:none;line-height:1.2\"\u003e\u003ctbody\u003e\n\u003ctr\u003e\u003cth style=\"background-color:#e6e6e6;line-height:1.5\"\u003eBacklog (Last messages accounting for this alert)\u003c/th\u003e\u003c/tr\u003e\n${foreach backlog message}\n\u003ctr\u003e\u003ctd\u003e${message}\u003c/td\u003e\u003c/tr\u003e\n${end}\n\u003c/tbody\u003e\u003c/table\u003e\n${end}\n","email_recipients":{{ (graylog_alerts_mailing_list|default(alerts_mailing_list)).split(',') | to_json | replace(" ", "") }},"user_recipients":[]}} diff --git a/ansible/roles/graylog-mongodb-import/templates/views.json b/ansible/roles/graylog-mongodb-import/templates/views.json index 12f3175673..d8fc377393 100644 --- a/ansible/roles/graylog-mongodb-import/templates/views.json +++ b/ansible/roles/graylog-mongodb-import/templates/views.json @@ -1,6 +1,6 @@ -{"_id":{"$oid":"606d3ebc4341c41b1b10e6a2"},"type":"DASHBOARD","title":"Log Sources Overview and GeoIP","summary":"Log Sources Overview and GeoIP","description":"Log Sources Overview and GeoIP","search_id":"60891dc5c74816d14249e216","properties":[],"requires":{},"state":{"a1647eb6-a064-4fe6-b459-1e4267d3f659":{"titles":{"tab":{"title":"Sources Overview"},"widget":{"6c127c5d-be75-4157-b43f-ac0194ac0586":" Count of Messages Per Source","92d63811-e4dd-47db-bd3b-db03c8a9bd53":"Messages per Source","00637e63-d728-4b3e-932b-7c8696b4855d":"Messages over time","9944c6c9-c9f8-4065-9c38-1a02e9eff0e7":"Response code over time","34bad054-49fc-4ad8-a744-16459fd405e3":"Response code count"}},"widgets":[{"id":"00637e63-d728-4b3e-932b-7c8696b4855d","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"timestamp","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[],"visualization":"bar","rollup":true,"event_annotation":false}},{"id":"9944c6c9-c9f8-4065-9c38-1a02e9eff0e7","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":"kubernetes_labels_app:\"nginx-public-ingress\""},"streams":[],"config":{"row_pivots":[{"field":"timestamp","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[{"field":"n_rscode","type":"values","config":{"limit":15}}],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[],"visualization":"line","rollup":true,"event_annotation":false}},{"id":"6c127c5d-be75-4157-b43f-ac0194ac0586","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"source","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"table","rollup":true,"event_annotation":false}},{"id":"92d63811-e4dd-47db-bd3b-db03c8a9bd53","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"source","type":"values","config":{"limit":10}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"pie","rollup":true,"event_annotation":false}},{"id":"34bad054-49fc-4ad8-a744-16459fd405e3","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":"kubernetes_labels_app:\"nginx-public-ingress\""},"streams":[],"config":{"row_pivots":[{"field":"n_rscode","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count(n_rscode)"}],"sort":[{"type":"series","field":"count(n_rscode)","direction":"Descending"}],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"00637e63-d728-4b3e-932b-7c8696b4855d":["652de1c8-b0f4-40ef-8b8e-e2139f5b2da7"],"9944c6c9-c9f8-4065-9c38-1a02e9eff0e7":["29399975-6f33-4383-8072-b7372a444a04"],"6c127c5d-be75-4157-b43f-ac0194ac0586":["b16e6001-8553-44d6-8b89-bf2c9502407e"],"92d63811-e4dd-47db-bd3b-db03c8a9bd53":["f872138d-aaab-4210-aa28-82d47013cdba"],"34bad054-49fc-4ad8-a744-16459fd405e3":["1d15d453-d008-461e-8a9d-931947482765"]},"positions":{"6c127c5d-be75-4157-b43f-ac0194ac0586":{"col":1,"row":14,"height":4,"width":6},"92d63811-e4dd-47db-bd3b-db03c8a9bd53":{"col":7,"row":14,"height":4,"width":6},"00637e63-d728-4b3e-932b-7c8696b4855d":{"col":1,"row":10,"height":4,"width":4},"9944c6c9-c9f8-4065-9c38-1a02e9eff0e7":{"col":5,"row":10,"height":4,"width":4},"34bad054-49fc-4ad8-a744-16459fd405e3":{"col":9,"row":10,"height":4,"width":4}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"71a16fbd-0018-406a-b44e-b5d3c086a3c8":{"titles":{"widget":{"3d5398a1-378e-4a34-8cd4-71ee9c2973ce":"IP address map","93901e5b-7162-4d0a-a671-618fb8b4d3d5":"Count of requests country and city wise","f8c4a7eb-422f-4b60-a6e5-50cc15e844ea":"Percentage of requests country and city wise"},"tab":{"title":"Geolocation Map of IP's"}},"widgets":[{"id":"3d5398a1-378e-4a34-8cd4-71ee9c2973ce","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":"_exists_: n_cip_geo_country_name"},"streams":[],"config":{"row_pivots":[{"field":"n_cip_geolocation","type":"values","config":{"limit":15}}],"column_pivots":[{"field":"n_cip_geo_city_name","type":"values","config":{"limit":15}}],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"map","rollup":true,"event_annotation":false}},{"id":"f8c4a7eb-422f-4b60-a6e5-50cc15e844ea","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"n_cip_geo_country_name","type":"values","config":{"limit":15}},{"field":"n_cip_geo_city_name","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"pie","rollup":true,"event_annotation":false}},{"id":"93901e5b-7162-4d0a-a671-618fb8b4d3d5","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"n_cip_geo_country_name","type":"values","config":{"limit":15}},{"field":"n_cip_geo_city_name","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"3d5398a1-378e-4a34-8cd4-71ee9c2973ce":["641ce507-92a1-4cd6-b57c-9f2346405418"],"93901e5b-7162-4d0a-a671-618fb8b4d3d5":["a819d06d-8122-4c12-8e22-4546afa294e0"],"f8c4a7eb-422f-4b60-a6e5-50cc15e844ea":["ca17b326-8be6-4b50-810b-d6bd906b39de"]},"positions":{"3d5398a1-378e-4a34-8cd4-71ee9c2973ce":{"col":1,"row":7,"height":4,"width":+Infinity},"93901e5b-7162-4d0a-a671-618fb8b4d3d5":{"col":1,"row":11,"height":4,"width":6},"f8c4a7eb-422f-4b60-a6e5-50cc15e844ea":{"col":7,"row":11,"height":4,"width":6}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"admin","created_at":{"$date":"2019-11-22T10:54:50.950Z"}} -{"_id":{"$oid":"606dbfe90363da32a534131e"},"type":"SEARCH","title":"Home Page","summary":"","description":"","search_id":"606ed19fe7202f30e9856553","properties":[],"requires":{},"state":{"bafc4c40-177c-433d-ba3f-8934eacbfceb":{"titles":{"widget":{"6b798062-541e-4763-b076-c7f3b5a8daaa":"Message Count","d8a0c654-efc4-4504-947a-75f346226ced":"All Messages"}},"widgets":[{"id":"d8a0c654-efc4-4504-947a-75f346226ced","type":"messages","timerange":null,"query":null,"streams":[],"config":{"fields":["timestamp","source"],"show_message_row":true,"decorators":[],"sort":[{"type":"pivot","field":"timestamp","direction":"Descending"}]}},{"id":"6b798062-541e-4763-b076-c7f3b5a8daaa","type":"aggregation","timerange":null,"query":null,"streams":[],"config":{"row_pivots":[{"field":"timestamp","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"bar","rollup":true,"event_annotation":false}},{"id":"49002afd-be41-4725-8066-16374a907196","type":"aggregation","timerange":null,"query":null,"streams":[],"config":{"row_pivots":[],"column_pivots":[],"series":[{"config":{"name":"Message Count"},"function":"count()"}],"sort":[],"visualization":"numeric","rollup":true,"event_annotation":false}}],"widget_mapping":{"6b798062-541e-4763-b076-c7f3b5a8daaa":["02eabfc0-8b65-4f83-8005-11747a203aa5"],"d8a0c654-efc4-4504-947a-75f346226ced":["6129f3d0-43eb-4bc3-9931-cbe057cf9e25"],"49002afd-be41-4725-8066-16374a907196":["7754427c-1c7c-4923-9db5-d67de6d1ebbd"]},"positions":{"6b798062-541e-4763-b076-c7f3b5a8daaa":{"col":1,"row":1,"height":2,"width":10},"d8a0c654-efc4-4504-947a-75f346226ced":{"col":1,"row":3,"height":6,"width":+Infinity},"49002afd-be41-4725-8066-16374a907196":{"col":11,"row":1,"height":2,"width":2}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"admin","created_at":{"$date":"2021-04-07T14:19:36.396Z"}} -{"_id":{"$oid":"6087f91807a3b704f5793257"},"type":"DASHBOARD","title":"Sources","summary":"This is a list of all sources that sent in messages to Graylog.","description":"This is a list of all sources that sent in messages to Graylog. You can narrow the timerange by zooming in on the message histogram, or you can increase the time range by specifying a broader one in the controls at the top. You can also specify filters to limit the results you are seeing. You can also add additional widgets to this dashboard, or adapt the appearance of existing widgets to suit your needs.","search_id":"6087f91807a3b704f5793255","properties":[],"requires":{},"state":{"a1647eb6-a064-4fe6-b459-1e4267d3f659":{"titles":{"tab":{"title":"Sources Overview"},"widget":{"6c127c5d-be75-4157-b43f-ac0194ac0586":"Selected sources","92d63811-e4dd-47db-bd3b-db03c8a9bd53":"Messages per Source","00637e63-d728-4b3e-932b-7c8696b4855d":"Messages over time"}},"widgets":[{"id":"92d63811-e4dd-47db-bd3b-db03c8a9bd53","type":"aggregation","timerange":{"type":"relative","range":300},"query":null,"streams":[],"config":{"row_pivots":[{"field":"source","type":"values","config":{"limit":10}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"pie","rollup":true,"event_annotation":false}},{"id":"00637e63-d728-4b3e-932b-7c8696b4855d","type":"aggregation","timerange":{"type":"relative","range":300},"query":null,"streams":[],"config":{"row_pivots":[{"field":"timestamp","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[],"visualization":"line","rollup":true,"event_annotation":false}},{"id":"6c127c5d-be75-4157-b43f-ac0194ac0586","type":"aggregation","timerange":{"type":"relative","range":300},"query":null,"streams":[],"config":{"row_pivots":[{"field":"source","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"6c127c5d-be75-4157-b43f-ac0194ac0586":["011b2894-49e5-44d8-aab6-8c4d4457a886"],"92d63811-e4dd-47db-bd3b-db03c8a9bd53":["a964f1c5-e108-4b5e-a907-ffe0b0f0683c"],"00637e63-d728-4b3e-932b-7c8696b4855d":["481de18f-938e-40d5-8ab2-6eaf6a28f091"]},"positions":{"6c127c5d-be75-4157-b43f-ac0194ac0586":{"col":1,"row":5,"height":4,"width":6},"92d63811-e4dd-47db-bd3b-db03c8a9bd53":{"col":7,"row":5,"height":4,"width":6},"00637e63-d728-4b3e-932b-7c8696b4855d":{"col":1,"row":1,"height":4,"width":+Infinity}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"admin","created_at":{"$date":"2019-11-22T10:54:50.950Z"}} -{"_id":{"$oid":"60b876fb337c3046d7c2e6f5"},"type":"DASHBOARD","title":"Functional Errors Dashboard","summary":"Functional Errors Dashboard","description":"Functional Errors Dashboard","search_id":"60c9a1ac10b85d9d03d63b30","properties":[],"requires":{},"state":{"91fcb809-6b04-4590-9263-30d3295372c4":{"titles":{"widget":{"dc0ec351-93ad-4c05-95ac-044c8fed9a03":"Percentage of Errors","23148474-33aa-45a6-b946-e9c0f322c346":"Count of Errors"},"tab":{"title":"Threshold \u003e 0.1% and \u003c= 1%"}},"widgets":[{"id":"23148474-33aa-45a6-b946-e9c0f322c346","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e0.1 AND metric_value: \u003c=1"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}},{"id":"dc0ec351-93ad-4c05-95ac-044c8fed9a03","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e0.1 AND metric_value: \u003c=1"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"pie","rollup":true,"event_annotation":false}}],"widget_mapping":{"23148474-33aa-45a6-b946-e9c0f322c346":["540657cf-781e-4332-a34a-925922b4a4aa"],"dc0ec351-93ad-4c05-95ac-044c8fed9a03":["130a424e-94bf-40fc-b775-814b677c2800"]},"positions":{"23148474-33aa-45a6-b946-e9c0f322c346":{"col":1,"row":71,"height":4,"width":+Infinity},"dc0ec351-93ad-4c05-95ac-044c8fed9a03":{"col":1,"row":22,"height":4,"width":+Infinity}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"806f5a1b-0923-4963-8fc2-057d5886d7dc":{"titles":{"widget":{"749f189f-fcba-4073-b3b4-200f73dea858":"Percentage of Errors","8e88585c-029b-4787-b94c-316d47817e0e":"Count of Errors"},"tab":{"title":"Threshold \u003e 1% and \u003c= 2%"}},"widgets":[{"id":"749f189f-fcba-4073-b3b4-200f73dea858","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e1 AND metric_value: \u003c=2"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"pie","rollup":true,"event_annotation":false}},{"id":"8e88585c-029b-4787-b94c-316d47817e0e","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e1 AND metric_value: \u003c=2"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"8e88585c-029b-4787-b94c-316d47817e0e":["04f08263-bf1a-447a-a991-41c3271f6d1d"],"749f189f-fcba-4073-b3b4-200f73dea858":["218d19af-4959-4c6c-a975-e5dc4ba0eb0c"]},"positions":{"8e88585c-029b-4787-b94c-316d47817e0e":{"col":1,"row":73,"height":4,"width":+Infinity},"749f189f-fcba-4073-b3b4-200f73dea858":{"col":1,"row":24,"height":4,"width":+Infinity}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"fbd1868d-c074-4361-9130-1d1714c8ea67":{"titles":{"widget":{"e6a276b8-a246-4cf6-8960-d1beecddf918":"Percentage of Errors","6f98b771-54b0-49cc-92a3-187ada526394":"Count of Errors"},"tab":{"title":"Threshold \u003e 2% and \u003c= 5%"}},"widgets":[{"id":"e6a276b8-a246-4cf6-8960-d1beecddf918","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e2 AND metric_value: \u003c=5"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"pie","rollup":true,"event_annotation":false}},{"id":"6f98b771-54b0-49cc-92a3-187ada526394","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e2 AND metric_value: \u003c=5"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"6f98b771-54b0-49cc-92a3-187ada526394":["83f15909-9faf-4dca-9453-fb88662f56b0"],"e6a276b8-a246-4cf6-8960-d1beecddf918":["518ea27c-dac8-445e-b959-0b0fe392fb0c"]},"positions":{"6f98b771-54b0-49cc-92a3-187ada526394":{"col":1,"row":73,"height":4,"width":+Infinity},"e6a276b8-a246-4cf6-8960-d1beecddf918":{"col":1,"row":24,"height":4,"width":+Infinity}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"cf249cae-17b7-498a-98a1-386de3931899":{"titles":{"widget":{"9759d11a-060d-46ad-93aa-e07b0bbe2940":"Percentage of Errors","0bd6ee4c-961a-4465-ad45-08a3c61a2723":"Count of Errors"},"tab":{"title":"Threshold \u003e 5%"}},"widgets":[{"id":"0bd6ee4c-961a-4465-ad45-08a3c61a2723","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e5"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}},{"id":"9759d11a-060d-46ad-93aa-e07b0bbe2940","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e5"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"pie","rollup":true,"event_annotation":false}}],"widget_mapping":{"0bd6ee4c-961a-4465-ad45-08a3c61a2723":["cb8f072c-b322-4670-bfba-cac53975b980"],"9759d11a-060d-46ad-93aa-e07b0bbe2940":["54c5de0d-8dc5-4e55-87b0-5e06837f8b0f"]},"positions":{"0bd6ee4c-961a-4465-ad45-08a3c61a2723":{"col":1,"row":73,"height":4,"width":+Infinity},"9759d11a-060d-46ad-93aa-e07b0bbe2940":{"col":1,"row":24,"height":4,"width":+Infinity}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"admin","created_at":{"$date":"2021-06-03T06:19:02.711Z"}} -{"_id":{"$oid":"60c1d20505a0719038bacdab"},"type":"SEARCH","title":"Show Logs without timestamp","summary":"","description":"","search_id":"60c1d1f705a0719038bacdaa","properties":[],"requires":{},"state":{"7b59069e-c82a-4d5a-8740-784fc57826fb":{"titles":{"widget":{"4e45ba55-2ac3-4eba-88f6-70a5fbe1ce5d":"Message Count","a2e06b69-d9d9-409d-b1de-dffae349c008":"All Messages"}},"widgets":[{"id":"4e45ba55-2ac3-4eba-88f6-70a5fbe1ce5d","type":"aggregation","timerange":null,"query":null,"streams":[],"config":{"row_pivots":[{"field":"timestamp","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"bar","rollup":true,"event_annotation":false}},{"id":"a2e06b69-d9d9-409d-b1de-dffae349c008","type":"messages","timerange":null,"query":null,"streams":[],"config":{"fields":[],"show_message_row":true,"decorators":[],"sort":[{"type":"pivot","field":"timestamp","direction":"Descending"}]}}],"widget_mapping":{"4e45ba55-2ac3-4eba-88f6-70a5fbe1ce5d":["eb787319-596c-4c9b-8ea9-643048137beb"],"a2e06b69-d9d9-409d-b1de-dffae349c008":["bb6cd31a-c26f-48d9-a7b5-7eee245b3744"]},"positions":{"4e45ba55-2ac3-4eba-88f6-70a5fbe1ce5d":{"col":1,"row":3,"height":2,"width":+Infinity},"a2e06b69-d9d9-409d-b1de-dffae349c008":{"col":1,"row":5,"height":6,"width":+Infinity}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"viewer","created_at":{"$date":"2021-06-10T08:48:16.501Z"}} -{"_id":{"$oid":"60ed949db96722aa70a93421"},"type":"DASHBOARD","title":"DataProducts Dashboard","summary":"DataProducts Dashboard","description":"DataProducts Dashboard ","search_id":"61445c70965dbceb0025d60e","properties":[],"requires":{},"state":{"68cda4cf-0fff-4abb-ac6a-7dbec66f0cde":{"titles":{"widget":{"a4e61cc5-7c46-44ab-877e-f0625e77f915":"Data Products Execution Status"},"tab":{"title":"DataProducts Status"}},"widgets":[{"id":"a4e61cc5-7c46-44ab-877e-f0625e77f915","type":"aggregation","timerange":{"type":"relative","range":28800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"id-report-date","type":"time","config":{"interval":{"type":"auto","scaling":null}}},{"field":"subsystem","type":"values","config":{"limit":15}},{"field":"status","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"a4e61cc5-7c46-44ab-877e-f0625e77f915":["cb2c0d8d-1ca1-4a33-8dec-01bfc478a878"]},"positions":{"a4e61cc5-7c46-44ab-877e-f0625e77f915":{"col":1,"row":26,"height":4,"width":+Infinity}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"e34ac556-9172-4a5f-adf0-24d0f9d62273":{"titles":{"tab":{"title":"Exhaust Reports Dashboard"},"widget":{"a0df09d4-16a7-4a5c-a0db-97e3447995dd":"Exhaust Jobs"}},"widgets":[{"id":"a0df09d4-16a7-4a5c-a0db-97e3447995dd","type":"aggregation","timerange":{"type":"relative","range":28800},"query":{"type":"elasticsearch","query_string":"subsystem:ProgressExhaustJob || subsystem: ResponseExhaustJob || subsystem: UserInfoExhaustJob || subsystem: OnDemandDruidExhaustJob"},"streams":[],"config":{"row_pivots":[{"field":"subsystem","type":"values","config":{"limit":15}},{"field":"id-report-date","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[],"series":[{"config":{},"function":"sum(total_requests)"},{"config":{},"function":"sum(success_requests)"},{"config":{},"function":"sum(failed_requests)"},{"config":{},"function":"sum(time_taken_secs)"},{"config":{},"function":"max(time_taken_secs)"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"a0df09d4-16a7-4a5c-a0db-97e3447995dd":["ff8d162a-1144-4ee4-a7d3-abd5f370ecfe"]},"positions":{"a0df09d4-16a7-4a5c-a0db-97e3447995dd":{"col":1,"row":35,"height":4,"width":+Infinity}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"1c5e00d1-ae3a-4401-9897-e4cef2d963ad":{"titles":{"tab":{"title":"WorkFlow Summary Dashboard"},"widget":{"a0df09d4-16a7-4a5c-a0db-97e3447995dd":"Exhaust Jobs","0543269c-00c6-4b6c-911e-455fa3ff2c40":"WFS Job"}},"widgets":[{"id":"0543269c-00c6-4b6c-911e-455fa3ff2c40","type":"aggregation","timerange":{"type":"relative","range":28800},"query":{"type":"elasticsearch","query_string":"subsystem:WorkFlowSummaryModel"},"streams":[],"config":{"row_pivots":[{"field":"id-report-date","type":"time","config":{"interval":{"type":"auto","scaling":null}}},{"field":"subsystem","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"sum(input_events)"},{"config":{},"function":"sum(output_events)"},{"config":{},"function":"sum(time_taken_secs)"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"0543269c-00c6-4b6c-911e-455fa3ff2c40":["86b12452-e6a8-4f16-95e9-1cdde471b28a"]},"positions":{"0543269c-00c6-4b6c-911e-455fa3ff2c40":{"col":1,"row":13,"height":4,"width":+Infinity}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"admin","created_at":{"$date":"2021-07-13T12:38:15.831Z"}} +{"_id":{"$oid":"606d3ebc4341c41b1b10e6a2"},"type":"DASHBOARD","title":"Log Sources Overview and GeoIP","summary":"Log Sources Overview and GeoIP","description":"Log Sources Overview and GeoIP","search_id":"60891dc5c74816d14249e216","properties":[],"requires":{},"state":{"a1647eb6-a064-4fe6-b459-1e4267d3f659":{"titles":{"tab":{"title":"Sources Overview"},"widget":{"6c127c5d-be75-4157-b43f-ac0194ac0586":" Count of Messages Per Source","92d63811-e4dd-47db-bd3b-db03c8a9bd53":"Messages per Source","00637e63-d728-4b3e-932b-7c8696b4855d":"Messages over time","9944c6c9-c9f8-4065-9c38-1a02e9eff0e7":"Response code over time","34bad054-49fc-4ad8-a744-16459fd405e3":"Response code count"}},"widgets":[{"id":"00637e63-d728-4b3e-932b-7c8696b4855d","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"timestamp","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[],"visualization":"bar","rollup":true,"event_annotation":false}},{"id":"9944c6c9-c9f8-4065-9c38-1a02e9eff0e7","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":"kubernetes_labels_app:\"nginx-public-ingress\""},"streams":[],"config":{"row_pivots":[{"field":"timestamp","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[{"field":"n_rscode","type":"values","config":{"limit":15}}],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[],"visualization":"line","rollup":true,"event_annotation":false}},{"id":"6c127c5d-be75-4157-b43f-ac0194ac0586","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"source","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"table","rollup":true,"event_annotation":false}},{"id":"92d63811-e4dd-47db-bd3b-db03c8a9bd53","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"source","type":"values","config":{"limit":10}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"pie","rollup":true,"event_annotation":false}},{"id":"34bad054-49fc-4ad8-a744-16459fd405e3","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":"kubernetes_labels_app:\"nginx-public-ingress\""},"streams":[],"config":{"row_pivots":[{"field":"n_rscode","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count(n_rscode)"}],"sort":[{"type":"series","field":"count(n_rscode)","direction":"Descending"}],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"00637e63-d728-4b3e-932b-7c8696b4855d":["652de1c8-b0f4-40ef-8b8e-e2139f5b2da7"],"9944c6c9-c9f8-4065-9c38-1a02e9eff0e7":["29399975-6f33-4383-8072-b7372a444a04"],"6c127c5d-be75-4157-b43f-ac0194ac0586":["b16e6001-8553-44d6-8b89-bf2c9502407e"],"92d63811-e4dd-47db-bd3b-db03c8a9bd53":["f872138d-aaab-4210-aa28-82d47013cdba"],"34bad054-49fc-4ad8-a744-16459fd405e3":["1d15d453-d008-461e-8a9d-931947482765"]},"positions":{"6c127c5d-be75-4157-b43f-ac0194ac0586":{"col":1,"row":14,"height":4,"width":6},"92d63811-e4dd-47db-bd3b-db03c8a9bd53":{"col":7,"row":14,"height":4,"width":6},"00637e63-d728-4b3e-932b-7c8696b4855d":{"col":1,"row":10,"height":4,"width":4},"9944c6c9-c9f8-4065-9c38-1a02e9eff0e7":{"col":5,"row":10,"height":4,"width":4},"34bad054-49fc-4ad8-a744-16459fd405e3":{"col":9,"row":10,"height":4,"width":4}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"71a16fbd-0018-406a-b44e-b5d3c086a3c8":{"titles":{"widget":{"3d5398a1-378e-4a34-8cd4-71ee9c2973ce":"IP address map","93901e5b-7162-4d0a-a671-618fb8b4d3d5":"Count of requests country and city wise","f8c4a7eb-422f-4b60-a6e5-50cc15e844ea":"Percentage of requests country and city wise"},"tab":{"title":"Geolocation Map of IP's"}},"widgets":[{"id":"3d5398a1-378e-4a34-8cd4-71ee9c2973ce","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":"_exists_: n_cip_geo_country_name"},"streams":[],"config":{"row_pivots":[{"field":"n_cip_geolocation","type":"values","config":{"limit":15}}],"column_pivots":[{"field":"n_cip_geo_city_name","type":"values","config":{"limit":15}}],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"map","rollup":true,"event_annotation":false}},{"id":"f8c4a7eb-422f-4b60-a6e5-50cc15e844ea","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"n_cip_geo_country_name","type":"values","config":{"limit":15}},{"field":"n_cip_geo_city_name","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"pie","rollup":true,"event_annotation":false}},{"id":"93901e5b-7162-4d0a-a671-618fb8b4d3d5","type":"aggregation","timerange":{"type":"relative","range":1800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"n_cip_geo_country_name","type":"values","config":{"limit":15}},{"field":"n_cip_geo_city_name","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"3d5398a1-378e-4a34-8cd4-71ee9c2973ce":["641ce507-92a1-4cd6-b57c-9f2346405418"],"93901e5b-7162-4d0a-a671-618fb8b4d3d5":["a819d06d-8122-4c12-8e22-4546afa294e0"],"f8c4a7eb-422f-4b60-a6e5-50cc15e844ea":["ca17b326-8be6-4b50-810b-d6bd906b39de"]},"positions":{"3d5398a1-378e-4a34-8cd4-71ee9c2973ce":{"col":1,"row":7,"height":4,"width":"+Infinity"},"93901e5b-7162-4d0a-a671-618fb8b4d3d5":{"col":1,"row":11,"height":4,"width":6},"f8c4a7eb-422f-4b60-a6e5-50cc15e844ea":{"col":7,"row":11,"height":4,"width":6}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"admin","created_at":{"$date":"2019-11-22T10:54:50.950Z"}} +{"_id":{"$oid":"606dbfe90363da32a534131e"},"type":"SEARCH","title":"Home Page","summary":"","description":"","search_id":"606ed19fe7202f30e9856553","properties":[],"requires":{},"state":{"bafc4c40-177c-433d-ba3f-8934eacbfceb":{"titles":{"widget":{"6b798062-541e-4763-b076-c7f3b5a8daaa":"Message Count","d8a0c654-efc4-4504-947a-75f346226ced":"All Messages"}},"widgets":[{"id":"d8a0c654-efc4-4504-947a-75f346226ced","type":"messages","timerange":null,"query":null,"streams":[],"config":{"fields":["timestamp","source"],"show_message_row":true,"decorators":[],"sort":[{"type":"pivot","field":"timestamp","direction":"Descending"}]}},{"id":"6b798062-541e-4763-b076-c7f3b5a8daaa","type":"aggregation","timerange":null,"query":null,"streams":[],"config":{"row_pivots":[{"field":"timestamp","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"bar","rollup":true,"event_annotation":false}},{"id":"49002afd-be41-4725-8066-16374a907196","type":"aggregation","timerange":null,"query":null,"streams":[],"config":{"row_pivots":[],"column_pivots":[],"series":[{"config":{"name":"Message Count"},"function":"count()"}],"sort":[],"visualization":"numeric","rollup":true,"event_annotation":false}}],"widget_mapping":{"6b798062-541e-4763-b076-c7f3b5a8daaa":["02eabfc0-8b65-4f83-8005-11747a203aa5"],"d8a0c654-efc4-4504-947a-75f346226ced":["6129f3d0-43eb-4bc3-9931-cbe057cf9e25"],"49002afd-be41-4725-8066-16374a907196":["7754427c-1c7c-4923-9db5-d67de6d1ebbd"]},"positions":{"6b798062-541e-4763-b076-c7f3b5a8daaa":{"col":1,"row":1,"height":2,"width":10},"d8a0c654-efc4-4504-947a-75f346226ced":{"col":1,"row":3,"height":6,"width":"+Infinity"},"49002afd-be41-4725-8066-16374a907196":{"col":11,"row":1,"height":2,"width":2}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"admin","created_at":{"$date":"2021-04-07T14:19:36.396Z"}} +{"_id":{"$oid":"6087f91807a3b704f5793257"},"type":"DASHBOARD","title":"Sources","summary":"This is a list of all sources that sent in messages to Graylog.","description":"This is a list of all sources that sent in messages to Graylog. You can narrow the timerange by zooming in on the message histogram, or you can increase the time range by specifying a broader one in the controls at the top. You can also specify filters to limit the results you are seeing. You can also add additional widgets to this dashboard, or adapt the appearance of existing widgets to suit your needs.","search_id":"6087f91807a3b704f5793255","properties":[],"requires":{},"state":{"a1647eb6-a064-4fe6-b459-1e4267d3f659":{"titles":{"tab":{"title":"Sources Overview"},"widget":{"6c127c5d-be75-4157-b43f-ac0194ac0586":"Selected sources","92d63811-e4dd-47db-bd3b-db03c8a9bd53":"Messages per Source","00637e63-d728-4b3e-932b-7c8696b4855d":"Messages over time"}},"widgets":[{"id":"92d63811-e4dd-47db-bd3b-db03c8a9bd53","type":"aggregation","timerange":{"type":"relative","range":300},"query":null,"streams":[],"config":{"row_pivots":[{"field":"source","type":"values","config":{"limit":10}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"pie","rollup":true,"event_annotation":false}},{"id":"00637e63-d728-4b3e-932b-7c8696b4855d","type":"aggregation","timerange":{"type":"relative","range":300},"query":null,"streams":[],"config":{"row_pivots":[{"field":"timestamp","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[],"visualization":"line","rollup":true,"event_annotation":false}},{"id":"6c127c5d-be75-4157-b43f-ac0194ac0586","type":"aggregation","timerange":{"type":"relative","range":300},"query":null,"streams":[],"config":{"row_pivots":[{"field":"source","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{"name":"Message count"},"function":"count()"}],"sort":[{"type":"series","field":"count()","direction":"Descending"}],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"6c127c5d-be75-4157-b43f-ac0194ac0586":["011b2894-49e5-44d8-aab6-8c4d4457a886"],"92d63811-e4dd-47db-bd3b-db03c8a9bd53":["a964f1c5-e108-4b5e-a907-ffe0b0f0683c"],"00637e63-d728-4b3e-932b-7c8696b4855d":["481de18f-938e-40d5-8ab2-6eaf6a28f091"]},"positions":{"6c127c5d-be75-4157-b43f-ac0194ac0586":{"col":1,"row":5,"height":4,"width":6},"92d63811-e4dd-47db-bd3b-db03c8a9bd53":{"col":7,"row":5,"height":4,"width":6},"00637e63-d728-4b3e-932b-7c8696b4855d":{"col":1,"row":1,"height":4,"width":"+Infinity"}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"admin","created_at":{"$date":"2019-11-22T10:54:50.950Z"}} +{"_id":{"$oid":"60b876fb337c3046d7c2e6f5"},"type":"DASHBOARD","title":"Functional Errors Dashboard","summary":"Functional Errors Dashboard","description":"Functional Errors Dashboard","search_id":"60c9a1ac10b85d9d03d63b30","properties":[],"requires":{},"state":{"91fcb809-6b04-4590-9263-30d3295372c4":{"titles":{"widget":{"dc0ec351-93ad-4c05-95ac-044c8fed9a03":"Percentage of Errors","23148474-33aa-45a6-b946-e9c0f322c346":"Count of Errors"},"tab":{"title":"Threshold \u003e 0.1% and \u003c= 1%"}},"widgets":[{"id":"23148474-33aa-45a6-b946-e9c0f322c346","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e0.1 AND metric_value: \u003c=1"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}},{"id":"dc0ec351-93ad-4c05-95ac-044c8fed9a03","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e0.1 AND metric_value: \u003c=1"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"pie","rollup":true,"event_annotation":false}}],"widget_mapping":{"23148474-33aa-45a6-b946-e9c0f322c346":["540657cf-781e-4332-a34a-925922b4a4aa"],"dc0ec351-93ad-4c05-95ac-044c8fed9a03":["130a424e-94bf-40fc-b775-814b677c2800"]},"positions":{"23148474-33aa-45a6-b946-e9c0f322c346":{"col":1,"row":71,"height":4,"width":"+Infinity"},"dc0ec351-93ad-4c05-95ac-044c8fed9a03":{"col":1,"row":22,"height":4,"width":"+Infinity"}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"806f5a1b-0923-4963-8fc2-057d5886d7dc":{"titles":{"widget":{"749f189f-fcba-4073-b3b4-200f73dea858":"Percentage of Errors","8e88585c-029b-4787-b94c-316d47817e0e":"Count of Errors"},"tab":{"title":"Threshold \u003e 1% and \u003c= 2%"}},"widgets":[{"id":"749f189f-fcba-4073-b3b4-200f73dea858","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e1 AND metric_value: \u003c=2"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"pie","rollup":true,"event_annotation":false}},{"id":"8e88585c-029b-4787-b94c-316d47817e0e","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e1 AND metric_value: \u003c=2"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"8e88585c-029b-4787-b94c-316d47817e0e":["04f08263-bf1a-447a-a991-41c3271f6d1d"],"749f189f-fcba-4073-b3b4-200f73dea858":["218d19af-4959-4c6c-a975-e5dc4ba0eb0c"]},"positions":{"8e88585c-029b-4787-b94c-316d47817e0e":{"col":1,"row":73,"height":4,"width":"+Infinity"},"749f189f-fcba-4073-b3b4-200f73dea858":{"col":1,"row":24,"height":4,"width":"+Infinity"}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"fbd1868d-c074-4361-9130-1d1714c8ea67":{"titles":{"widget":{"e6a276b8-a246-4cf6-8960-d1beecddf918":"Percentage of Errors","6f98b771-54b0-49cc-92a3-187ada526394":"Count of Errors"},"tab":{"title":"Threshold \u003e 2% and \u003c= 5%"}},"widgets":[{"id":"e6a276b8-a246-4cf6-8960-d1beecddf918","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e2 AND metric_value: \u003c=5"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"pie","rollup":true,"event_annotation":false}},{"id":"6f98b771-54b0-49cc-92a3-187ada526394","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e2 AND metric_value: \u003c=5"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"6f98b771-54b0-49cc-92a3-187ada526394":["83f15909-9faf-4dca-9453-fb88662f56b0"],"e6a276b8-a246-4cf6-8960-d1beecddf918":["518ea27c-dac8-445e-b959-0b0fe392fb0c"]},"positions":{"6f98b771-54b0-49cc-92a3-187ada526394":{"col":1,"row":73,"height":4,"width":"+Infinity"},"e6a276b8-a246-4cf6-8960-d1beecddf918":{"col":1,"row":24,"height":4,"width":"+Infinity"}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"cf249cae-17b7-498a-98a1-386de3931899":{"titles":{"widget":{"9759d11a-060d-46ad-93aa-e07b0bbe2940":"Percentage of Errors","0bd6ee4c-961a-4465-ad45-08a3c61a2723":"Count of Errors"},"tab":{"title":"Threshold \u003e 5%"}},"widgets":[{"id":"0bd6ee4c-961a-4465-ad45-08a3c61a2723","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e5"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}},{"id":"9759d11a-060d-46ad-93aa-e07b0bbe2940","type":"aggregation","timerange":{"type":"relative","range":3600},"query":{"type":"elasticsearch","query_string":"metric_value: \u003e5"},"streams":[],"config":{"row_pivots":[{"field":"dimensions_id","type":"values","config":{"limit":15}},{"field":"dimensions_value","type":"values","config":{"limit":15}},{"field":"metric_id","type":"values","config":{"limit":15}},{"field":"metric_value","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"pie","rollup":true,"event_annotation":false}}],"widget_mapping":{"0bd6ee4c-961a-4465-ad45-08a3c61a2723":["cb8f072c-b322-4670-bfba-cac53975b980"],"9759d11a-060d-46ad-93aa-e07b0bbe2940":["54c5de0d-8dc5-4e55-87b0-5e06837f8b0f"]},"positions":{"0bd6ee4c-961a-4465-ad45-08a3c61a2723":{"col":1,"row":73,"height":4,"width":"+Infinity"},"9759d11a-060d-46ad-93aa-e07b0bbe2940":{"col":1,"row":24,"height":4,"width":"+Infinity"}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"admin","created_at":{"$date":"2021-06-03T06:19:02.711Z"}} +{"_id":{"$oid":"60c1d20505a0719038bacdab"},"type":"SEARCH","title":"Show Logs without timestamp","summary":"","description":"","search_id":"60c1d1f705a0719038bacdaa","properties":[],"requires":{},"state":{"7b59069e-c82a-4d5a-8740-784fc57826fb":{"titles":{"widget":{"4e45ba55-2ac3-4eba-88f6-70a5fbe1ce5d":"Message Count","a2e06b69-d9d9-409d-b1de-dffae349c008":"All Messages"}},"widgets":[{"id":"4e45ba55-2ac3-4eba-88f6-70a5fbe1ce5d","type":"aggregation","timerange":null,"query":null,"streams":[],"config":{"row_pivots":[{"field":"timestamp","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[],"series":[{"config":{},"function":"count()"}],"sort":[],"visualization":"bar","rollup":true,"event_annotation":false}},{"id":"a2e06b69-d9d9-409d-b1de-dffae349c008","type":"messages","timerange":null,"query":null,"streams":[],"config":{"fields":[],"show_message_row":true,"decorators":[],"sort":[{"type":"pivot","field":"timestamp","direction":"Descending"}]}}],"widget_mapping":{"4e45ba55-2ac3-4eba-88f6-70a5fbe1ce5d":["eb787319-596c-4c9b-8ea9-643048137beb"],"a2e06b69-d9d9-409d-b1de-dffae349c008":["bb6cd31a-c26f-48d9-a7b5-7eee245b3744"]},"positions":{"4e45ba55-2ac3-4eba-88f6-70a5fbe1ce5d":{"col":1,"row":3,"height":2,"width":"+Infinity"},"a2e06b69-d9d9-409d-b1de-dffae349c008":{"col":1,"row":5,"height":6,"width":"+Infinity"}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"viewer","created_at":{"$date":"2021-06-10T08:48:16.501Z"}} +{"_id":{"$oid":"60ed949db96722aa70a93421"},"type":"DASHBOARD","title":"DataProducts Dashboard","summary":"DataProducts Dashboard","description":"DataProducts Dashboard ","search_id":"61445c70965dbceb0025d60e","properties":[],"requires":{},"state":{"68cda4cf-0fff-4abb-ac6a-7dbec66f0cde":{"titles":{"widget":{"a4e61cc5-7c46-44ab-877e-f0625e77f915":"Data Products Execution Status"},"tab":{"title":"DataProducts Status"}},"widgets":[{"id":"a4e61cc5-7c46-44ab-877e-f0625e77f915","type":"aggregation","timerange":{"type":"relative","range":28800},"query":{"type":"elasticsearch","query_string":""},"streams":[],"config":{"row_pivots":[{"field":"id-report-date","type":"time","config":{"interval":{"type":"auto","scaling":null}}},{"field":"subsystem","type":"values","config":{"limit":15}},{"field":"status","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"a4e61cc5-7c46-44ab-877e-f0625e77f915":["cb2c0d8d-1ca1-4a33-8dec-01bfc478a878"]},"positions":{"a4e61cc5-7c46-44ab-877e-f0625e77f915":{"col":1,"row":26,"height":4,"width":"+Infinity"}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"e34ac556-9172-4a5f-adf0-24d0f9d62273":{"titles":{"tab":{"title":"Exhaust Reports Dashboard"},"widget":{"a0df09d4-16a7-4a5c-a0db-97e3447995dd":"Exhaust Jobs"}},"widgets":[{"id":"a0df09d4-16a7-4a5c-a0db-97e3447995dd","type":"aggregation","timerange":{"type":"relative","range":28800},"query":{"type":"elasticsearch","query_string":"subsystem:ProgressExhaustJob || subsystem: ResponseExhaustJob || subsystem: UserInfoExhaustJob || subsystem: OnDemandDruidExhaustJob"},"streams":[],"config":{"row_pivots":[{"field":"subsystem","type":"values","config":{"limit":15}},{"field":"id-report-date","type":"time","config":{"interval":{"type":"auto","scaling":null}}}],"column_pivots":[],"series":[{"config":{},"function":"sum(total_requests)"},{"config":{},"function":"sum(success_requests)"},{"config":{},"function":"sum(failed_requests)"},{"config":{},"function":"sum(time_taken_secs)"},{"config":{},"function":"max(time_taken_secs)"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"a0df09d4-16a7-4a5c-a0db-97e3447995dd":["ff8d162a-1144-4ee4-a7d3-abd5f370ecfe"]},"positions":{"a0df09d4-16a7-4a5c-a0db-97e3447995dd":{"col":1,"row":35,"height":4,"width":"+Infinity"}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}},"1c5e00d1-ae3a-4401-9897-e4cef2d963ad":{"titles":{"tab":{"title":"WorkFlow Summary Dashboard"},"widget":{"a0df09d4-16a7-4a5c-a0db-97e3447995dd":"Exhaust Jobs","0543269c-00c6-4b6c-911e-455fa3ff2c40":"WFS Job"}},"widgets":[{"id":"0543269c-00c6-4b6c-911e-455fa3ff2c40","type":"aggregation","timerange":{"type":"relative","range":28800},"query":{"type":"elasticsearch","query_string":"subsystem:WorkFlowSummaryModel"},"streams":[],"config":{"row_pivots":[{"field":"id-report-date","type":"time","config":{"interval":{"type":"auto","scaling":null}}},{"field":"subsystem","type":"values","config":{"limit":15}}],"column_pivots":[],"series":[{"config":{},"function":"sum(input_events)"},{"config":{},"function":"sum(output_events)"},{"config":{},"function":"sum(time_taken_secs)"}],"sort":[],"visualization":"table","rollup":true,"event_annotation":false}}],"widget_mapping":{"0543269c-00c6-4b6c-911e-455fa3ff2c40":["86b12452-e6a8-4f16-95e9-1cdde471b28a"]},"positions":{"0543269c-00c6-4b6c-911e-455fa3ff2c40":{"col":1,"row":13,"height":4,"width":"+Infinity"}},"formatting":{"highlighting":[]},"display_mode_settings":{"positions":{}}}},"owner":"admin","created_at":{"$date":"2021-07-13T12:38:15.831Z"}} diff --git a/ansible/roles/keycloak-deploy/tasks/bootstrap.yml b/ansible/roles/keycloak-deploy/tasks/bootstrap.yml index 7df5d1e503..dcefe36e30 100644 --- a/ansible/roles/keycloak-deploy/tasks/bootstrap.yml +++ b/ansible/roles/keycloak-deploy/tasks/bootstrap.yml @@ -28,7 +28,6 @@ shell: "nohup {{keycloak_home}}/bin/standalone.sh -Dkeycloak.profile.feature.upload_scripts=enabled -b={{ansible_default_ipv4.address}} -bprivate={{ansible_default_ipv4.address}} --server-config standalone-ha.xml &" become: yes become_user: "{{ wildfly_user }}" - run_once: true notify: wait for keycloak to start - meta: flush_handlers @@ -37,7 +36,6 @@ apt: name: ["python-setuptools"] update_cache: true - run_once: true - name: Ensure python packages are installed apt: @@ -55,38 +53,32 @@ copy: src: "{{ role_path }}/files/python-keycloak-0.12.0" dest: /tmp/ - run_once: true - name: Initialize python library to run keycloak bootstrap script shell: cd /tmp/python-keycloak-0.12.0 && python setup.py install - run_once: true - name: Save keycloak vars to json template: src: "keycloak-bootstrap.conf.j2" dest: "/tmp/keycloak-bootstrap.conf.json" mode: "0644" - run_once: true - name: Copy realm json file to tmp location template: src: "keycloak-realm.j2" dest: "/tmp/keycloak-realm.json" mode: "0644" - run_once: true - name: Copy user manager roles file to tmp location copy: src: "files/python-keycloak-0.12.0/roles.json" dest: "/tmp/roles.json" mode: "0644" - run_once: true - name: Copy the keycloak bootstrap script copy: src: "{{ role_path }}/files/python-keycloak-0.12.0/keycloak" dest: /tmp - run_once: true - name: Run the keycloak bootstrap script shell: cd /tmp/keycloak/ && python keycloak_main.py /tmp/keycloak-bootstrap.conf.json diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index b03e09e837..3ae6e47417 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -94,6 +94,7 @@ user_extension_prefix: /user-extension uci_admin_prefix: /uci notification_service_prefix: /notification registry_service_prefix: /rc +assessment_prefix: /assessment # Service URLs knowledge_mw_service_url: "http://knowledge-mw-service:5000" @@ -5913,6 +5914,27 @@ kong_apis: config.required: false config.enabled: false +- name: assessmentItemRead + uris: "{{ assessment_prefix }}/v1/items/read" + upstream_url: "{{ vm_learning_service_url }}/assessment/v3/items/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - assessmentAccess + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + - name: collectionUpdate uris: "{{ collection_prefix }}/v1/update" upstream_url: "{{ content_service_url }}/collection/v4/update" @@ -9551,3 +9573,24 @@ kong_apis: - name: opa-checks config.required: true config.enabled: true + +- name: getReportDatasets + uris: "{{ report_service_prefix }}/report/datasets/get" + upstream_url: "{{ report_service_url }}/report/datasets/get" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - reportsAccess + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: opa-checks + config.required: true + config.enabled: true diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 4379f99681..1553e1b00d 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -48,6 +48,21 @@ ml_analytics_project_output_dir: "{{ WORKDIR }}/source/projects/output" ml_analytics_observation_status_output_dir: "{{ WORKDIR }}/source/observations/status/output" ml_analytics_api_authorization_key: "{{ml_api_auth_token | default('sunbird_api_auth_token')}}" ml_analytics_api_access_token: "{{ml_api_access_token | default('ml_core_internal_access_token')}}" -ml_analytics_druid_observation_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_id","school_externalId","state_name","state_id","state_externalId","district_name","district_id","district_externalId","zone_name","zone_id","zone_externalId","parent_name","parent_id","block_name","block_id","block_externalId","cluster_name","cluster_id","cluster_externalId","teacher_name","teacher_id","teacher_externalId","schoolLeader_name","schoolLeader_id","schoolLeader_externalId","taluk_id","taluk_externalId","taluk_name","hub_name","hub_id","hub_externalId","mandal_name","mandal_id","mandal_externalId","region_name","region_id","region_externalId","complex_name","complex_id","complex_externalId","completedDate","beat_name","beat_id","beat_externalId","division_name","division_id","division_externalId","Village/Ward_name","Village/Ward_id","Village/Ward_externalId","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name", "ecm_marked_na","board_name","updatedAt"]},"metricsSpec":[]}}}' +ml_analytics_druid_observation_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_id","school_externalId","state_name","state_id","state_externalId","district_name","district_id","district_externalId","zone_name","zone_id","zone_externalId","parent_name","parent_id","block_name","block_id","block_externalId","cluster_name","cluster_id","cluster_externalId","teacher_name","teacher_id","teacher_externalId","schoolLeader_name","schoolLeader_id","schoolLeader_externalId","taluk_id","taluk_externalId","taluk_name","hub_name","hub_id","hub_externalId","mandal_name","mandal_id","mandal_externalId","region_name","region_id","region_externalId","complex_name","complex_id","complex_externalId","completedDate","beat_name","beat_id","beat_externalId","division_name","division_id","division_externalId","Village/Ward_name","Village/Ward_id","Village/Ward_externalId","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name", "ecm_marked_na","board_name","updatedAt","organisation_id","user_type"]},"metricsSpec":[]}}}' ml_analytics_druid_project_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/projects/sl_projects.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}' ml_analytics_azure_sas_token: "{{ sunbird_private_storage_account_key }}" +ml_analytics_druid_distinctCnt_obs_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount/ml_observation_distinctCount_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_obs_domain_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain/ml_observation_distinctCount_domain.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain_criteria/ml_observation_distinctCount_domain_criteria.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCount/ml_projects_distinctCount.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' +ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCountPrglevel/ml_projects_distinctCount_prgmlevel.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}' +ml_analytics_obs_distinctCnt_azure_blob_path: "observation/distinctCount/" +ml_analytics_obs_distinctCnt_domain_azure_blob_path: "observation/distinctCount_domain/" +ml_analytics_obs_distinctCnt_domain_criteria_azure_blob_path: "observation/distinctCount_domain_criteria/" +ml_analytics_projects_distinctCnt_azure_blob_path: "projects/distinctCount/" +ml_analytics_obs_distinctCnt_output_dir: "{{ WORKDIR }}/source/observations/distinctCount/output" +ml_analytics_obs_distinctCnt_domain_output_dir: "{{ WORKDIR }}/source/observations/distinctCount_domain/output" +ml_analytics_obs_distinctCnt_domain_criteria_output_dir: "{{ WORKDIR }}/source/observations/distinctCount_domain_criteria/output" +ml_analytics_projects_distinctCnt_output_dir: "{{ WORKDIR }}/source/projects/distinctCount/output" +ml_analytics_projects_distinctCnt_prglevel_output_dir: "{{ WORKDIR }}/source/projects/distinctCountPrglevel/output" +ml_analytics_projects_distinctCnt_prglevel_azure_blob_path: "projects/distinctCountPrglevel/" diff --git a/ansible/roles/ml-analytics-service/tasks/main.yml b/ansible/roles/ml-analytics-service/tasks/main.yml index b5406faaa9..9c9bf79a90 100755 --- a/ansible/roles/ml-analytics-service/tasks/main.yml +++ b/ansible/roles/ml-analytics-service/tasks/main.yml @@ -40,10 +40,23 @@ - "{{ BASEPATH }}" - "{{ WORKDIR }}" - "{{ WORKDIR }}/faust_as_service" - -- name: Change user and create working directory under opt dir and install python virtual environment + +- name: Delete the virtualenv DIR + shell: "rm -rf {{ WORKDIR }}/spark_env" + become: true + +- name: Install python virtual environment shell: "cd {{ WORKDIR }} && virtualenv --python=python3.8 spark_venv" become: true + +- name: Change the ownership of virtual env + become: yes + file: + path: "{{ WORKDIR }}/spark_env" + state: directory + owner: "{{ USER }}" + group: "{{ USER }}" + mode: "0755" - name: Create necessary logs folders for pipeline become: yes @@ -113,4 +126,4 @@ user: "{{ USER }}" minute: "30" hour: "18" - job: "{{ BASEPATH }}/ml-analytics-service/run.sh" + job: "{{ BASEPATH }}/ml-analytics-service/run.sh > {{ BASEPATH }}/ml-analytics-service/crontab_job.log" diff --git a/ansible/roles/ml-analytics-service/templates/config.j2 b/ansible/roles/ml-analytics-service/templates/config.j2 index 06692f1de7..8693935e9c 100644 --- a/ansible/roles/ml-analytics-service/templates/config.j2 +++ b/ansible/roles/ml-analytics-service/templates/config.j2 @@ -78,6 +78,16 @@ observation_status_injestion_spec = {{ ml_analytics_druid_observation_status_inj project_injestion_spec = {{ ml_analytics_druid_project_injestion_spec }} +ml_distinctCnt_obs_status_spec = {{ ml_analytics_druid_distinctCnt_obs_injestion_spec }} + +ml_distinctCnt_obs_domain_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_injestion_spec }} + +ml_distinctCnt_obs_domain_criteria_spec = {{ ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec }} + +ml_distinctCnt_projects_status_spec = {{ ml_analytics_druid_distinctCnt_projects_status_injestion_spec }} + +ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec }} + [KAFKA] url = {{ ml_analytics_kafka_url }} @@ -132,16 +142,28 @@ observation_blob_path = {{ ml_analytics_observation_azure_blob_path }} projects_blob_path = {{ ml_analytics_project_azure_blob_path }} -[REDIS] +observation_distinctCount_blob_path = {{ ml_analytics_obs_distinctCnt_azure_blob_path }} + +observation_distinctCount_domain_blob_path = {{ ml_analytics_obs_distinctCnt_domain_azure_blob_path }} -host = {{ ml_analytics_redis_host }} +observation_distinctCount_domain_criteria_blob_path = {{ ml_analytics_obs_distinctCnt_domain_criteria_azure_blob_path }} -port = {{ ml_analytics_redis_port }} +projects_distinctCnt_blob_path = {{ ml_analytics_projects_distinctCnt_azure_blob_path }} -db_name = {{ ml_analytics_redis_db_name }} +projects_distinctCnt_prgmlevel_blob_path = {{ ml_analytics_projects_distinctCnt_prglevel_azure_blob_path }} [OUTPUT_DIR] project = {{ ml_analytics_project_output_dir }} observation_status = {{ ml_analytics_observation_status_output_dir }} + +observation_distinctCount_status = {{ ml_analytics_obs_distinctCnt_output_dir }} + +observation_distinctCount_domain = {{ ml_analytics_obs_distinctCnt_domain_output_dir }} + +observation_distinctCount_domain_criteria = {{ ml_analytics_obs_distinctCnt_domain_criteria_output_dir }} + +projects_distinctCount = {{ ml_analytics_projects_distinctCnt_output_dir }} + +projects_distinctCount_prgmlevel = {{ ml_analytics_projects_distinctCnt_prglevel_output_dir }} diff --git a/ansible/roles/stack-sunbird/defaults/main.yml b/ansible/roles/stack-sunbird/defaults/main.yml index 5ded8b6813..12d98086b3 100644 --- a/ansible/roles/stack-sunbird/defaults/main.yml +++ b/ansible/roles/stack-sunbird/defaults/main.yml @@ -607,7 +607,7 @@ nodebb_liveness_readiness: initialDelaySeconds: 45 periodSeconds: 15 timeoutSeconds: 5 - failureThreshold: 2 + failureThreshold: 20 livenessProbe: httpGet: path: /discussions/api/category/2 @@ -615,7 +615,7 @@ nodebb_liveness_readiness: initialDelaySeconds: 45 periodSeconds: 15 timeoutSeconds: 5 - failureThreshold: 2 + failureThreshold: 20 player_liveness_readiness: healthcheck: true @@ -1051,4 +1051,4 @@ kong_desktop_device_consumer_names_for_opa: '["desktop"]' # Audience claim check is disabled as of now # List of keycloak clients as these can come in audience field of a JWT token -# keycloak_allowed_aud: '"{{ keycloak_auth_server_url }}/realms/{{ keycloak_realm }}", "account", "realm-management"' \ No newline at end of file +# keycloak_allowed_aud: '"{{ keycloak_auth_server_url }}/realms/{{ keycloak_realm }}", "account", "realm-management"' diff --git a/ansible/roles/stack-sunbird/templates/assessment-service_application.conf b/ansible/roles/stack-sunbird/templates/assessment-service_application.conf index a7a1729f83..851433f160 100644 --- a/ansible/roles/stack-sunbird/templates/assessment-service_application.conf +++ b/ansible/roles/stack-sunbird/templates/assessment-service_application.conf @@ -415,4 +415,14 @@ import { master.category.validation.enabled="{{ master_category_validation_enabled }}" question.cache.enable=true -questionset.cache.enable=true \ No newline at end of file +questionset.cache.enable=true +assessment.copy.origin_data=["name", "author", "license", "organisation"] +assessment.copy.props_to_remove=["downloadUrl", "artifactUrl", "variants", + "createdOn", "collections", "children", "lastUpdatedOn", "SYS_INTERNAL_LAST_UPDATED_ON", + "versionKey", "s3Key", "status", "pkgVersion", "toc_url", "mimeTypesCount", + "contentTypesCount", "leafNodesCount", "childNodes", "prevState", "lastPublishedOn", + "flagReasons", "compatibilityLevel", "size", "publishChecklist", "publishComment", + "LastPublishedBy", "rejectReasons", "rejectComment", "gradeLevel", "subject", + "medium", "board", "topic", "purpose", "subtopic", "contentCredits", + "owner", "collaborators", "creators", "contributors", "badgeAssertions", "dialcodes", + "concepts", "keywords", "reservedDialcodes", "dialcodeRequired", "leafNodes", "sYS_INTERNAL_LAST_UPDATED_ON", "prevStatus", "lastPublishedBy", "streamingUrl"] \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/inbound.env b/ansible/roles/stack-sunbird/templates/inbound.env index 7b48fe1153..c8ed1a5157 100644 --- a/ansible/roles/stack-sunbird/templates/inbound.env +++ b/ansible/roles/stack-sunbird/templates/inbound.env @@ -12,6 +12,7 @@ KAFKA_OUTBOUND_TOPIC={{env_name}}.outbound KAFKA_TELEMETRY_TOPIC={{env_name}}.uci.telemetry KAFKA_ODK_TRANSFORMER_TOPIC={{env_name}}.odk.transformer KAFKA_ODK_TRANSFORMER_TOPIC_PATTERN={{env_name}}.odk.* +KAFKA_MESSAGE_REPORT_TOPIC={{env_name}}.message-report #FormsDB FORMS_DB_URL=postgresql://{{uci_postgres_host}}:5432/{{uci_forms_postgres_database}} @@ -56,6 +57,7 @@ REDIS_DB_INDEX={{redis_db_index_uci | default('7')}} AZURE_BLOB_STORE_CONTAINER={{sunbird_azure_uci_container_name | default('uci-' + env )}} AZURE_BLOB_STORE_ACCOUNT_KEY={{sunbird_private_storage_account_name}} AZURE_BLOB_STORE_ACCOUNT_NAME={{sunbird_private_storage_account_key}} +SELECTED_FILE_CDN=azure #Netcore NETCORE_WHATSAPP_AUTH_TOKEN={{uci_netcore_whatsapp_token}} diff --git a/ansible/roles/stack-sunbird/templates/orchestrator.env b/ansible/roles/stack-sunbird/templates/orchestrator.env index bd6b974a11..4165f7dceb 100644 --- a/ansible/roles/stack-sunbird/templates/orchestrator.env +++ b/ansible/roles/stack-sunbird/templates/orchestrator.env @@ -12,6 +12,9 @@ KAFKA_OUTBOUND_TOPIC={{env_name}}.outbound KAFKA_TELEMETRY_TOPIC={{env_name}}.uci.telemetry KAFKA_ODK_TRANSFORMER_TOPIC={{env_name}}.odk.transformer KAFKA_ODK_TRANSFORMER_TOPIC_PATTERN={{env_name}}.odk.* +KAFKA_PROCESS_OUTBOUND={{env_name}}.process-outbound +KAFKA_BROADCAST_TRANSFORMER_TOPIC={{env_name}}.broadcast-transformer +KAFKA_GENERIC_TRANSFORMER_TOPIC={{env_name}}.generic-transformer #Dummy config: To be removed later KAFKA_LOGS_TOPIC={{env_name}}.inbound-unprocessed @@ -21,11 +24,14 @@ FORMS_DB_URL=postgresql://{{uci_postgres_host}}:5432/{{uci_forms_postgres_databa FORMS_DB_USERNAME={{uci_postgres_user}} FORMS_DB_PASSWORD={{uci_postgres_password}} FORMS_DB_NAME={{uci_forms_postgres_database}} +FORMS_DB_HOST={{uci_postgres_host}} +FORMS_DB_PORT=5432 #Cassandra CASSANDRA_URL={{sunbird_cassandra_host}} CASSANDRA_PORT=9042 CASSANDRA_KEYSPACE={{env}}_uci_store +CASSANDRA_MIGRATION_COUNT={{uci_cassandra_migration_count | default(1)}} #Ports ORCHESTRATOR_INTERNAL_PORT=8686 diff --git a/ansible/roles/stack-sunbird/templates/outbound.env b/ansible/roles/stack-sunbird/templates/outbound.env index a76df2cce3..bac1d980e8 100644 --- a/ansible/roles/stack-sunbird/templates/outbound.env +++ b/ansible/roles/stack-sunbird/templates/outbound.env @@ -18,6 +18,8 @@ FORMS_DB_URL=postgresql://{{uci_postgres_host}}:5432/{{uci_forms_postgres_databa FORMS_DB_USERNAME={{uci_postgres_user}} FORMS_DB_PASSWORD={{uci_postgres_password}} FORMS_DB_NAME={{uci_forms_postgres_database}} +FORMS_DB_HOST={{uci_postgres_host}} +FORMS_DB_PORT=5432 #Cassandra CASSANDRA_URL={{sunbird_cassandra_host}} @@ -55,3 +57,4 @@ REDIS_DB_INDEX={{redis_db_index_uci | default('7')}} AZURE_BLOB_STORE_CONTAINER={{sunbird_azure_uci_container_name | default('uci-' + env )}} AZURE_BLOB_STORE_ACCOUNT_KEY={{sunbird_private_storage_account_name}} AZURE_BLOB_STORE_ACCOUNT_NAME={{sunbird_private_storage_account_key}} +SELECTED_FILE_CDN=azure diff --git a/ansible/roles/stack-sunbird/templates/sunbird_discussions-mw-service.env b/ansible/roles/stack-sunbird/templates/sunbird_discussions-mw-service.env index 6e3fafc506..f972a40d80 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_discussions-mw-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_discussions-mw-service.env @@ -11,4 +11,6 @@ authorization_token={{ discussionsmw_nodebb_authorization_token }} TELEMETRY_EVENTS_BATCH_SIZE={{ telemetry_events_batch_size | default(1) }} TELEMETRY_SERVICE_API_SLUG={{ telemetry_service_api_slug | default('/v1/telemetry') }} TELEMETRY_SERVICE_URL={{ telemetry_service_url | default('http://telemetry-service:9001') }} -API_AUTH_TOKEN={{ api_auth_token | default('') }} \ No newline at end of file +API_AUTH_TOKEN={{ api_auth_token | default('') }} +enable_audit_event={{ enable_audit_event | default('true') }} +moderation_flag={{ moderation_flag | default('false') }} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_report-service.env b/ansible/roles/stack-sunbird/templates/sunbird_report-service.env index 9d609a0cc4..a2ebfce02a 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_report-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_report-service.env @@ -9,4 +9,23 @@ SUNBIRD_REPORTS_TABLE_NAME=report SUNBIRD_REPORT_SUMMARY_TABLE_NAME={{sunbird_report_summary_table_name}} SUNBIRD_ENV="https://{{domain_name}}" DEACTIVATE_JOB_API_HOST=http://kong:8000/data/v1/report/jobs/deactivate -DEACTIVATE_JOB_API_KEY={{sunbird_api_auth_token}} \ No newline at end of file +DEACTIVATE_JOB_API_KEY={{sunbird_api_auth_token}} + +# 4.10.0 +sunbird_azure_report_container_name={{sunbird_azure_report_container_name}} +sunbird_azure_account_name={{sunbird_private_storage_account_name}} +sunbird_azure_account_key={{sunbird_private_storage_account_key}} +sunbird_api_auth_token={{sunbird_api_auth_token}} +sunbird_super_admin_slug={{sunbird_super_admin_slug|default('sunbird')}} +sunbird_portal_auth_server_url={{keycloak_auth_server_url}} +sunbird_keycloak_public_key={{core_vault_sunbird_sso_publickey}} +sunbird_portal_auth_server_client={{sunbird_portal_auth_server_client}} +sunbird_keycloak_public={{sunbird_keycloak_public}} +sunbird_learner_player_url={{sunbird_learner_player_url}} +sunbird_content_proxy_url=http://knowledge-mw-service:5000 +sunbird_keycloak_realm={{keycloak_realm}} +sunbird_cache_store={{sunbird_cache_store}} +sunbird_cache_ttl={{sunbird_cache_ttl}} +sunbird_report_sas_expiry_in_minutes=3600 +sunbird_dataservice_url={{sunbird_dataservice_url}} + diff --git a/ansible/roles/stack-sunbird/templates/transformer.env b/ansible/roles/stack-sunbird/templates/transformer.env index 30963cb81f..fd2e6d00aa 100644 --- a/ansible/roles/stack-sunbird/templates/transformer.env +++ b/ansible/roles/stack-sunbird/templates/transformer.env @@ -13,6 +13,8 @@ KAFKA_TELEMETRY_TOPIC={{env_name}}.uci.telemetry KAFKA_ODK_TRANSFORMER_TOPIC={{env_name}}.odk.transformer KAFKA_ODK_TRANSFORMER_TOPIC_PATTERN={{env_name}}.odk.* KAFKA_LOGS_TOPIC={{env_name}}.inbound-unprocessed +KAFKA_PROCESS_OUTBOUND={{env_name}}.process-outbound +KAFKA_BROADCAST_TRANSFORMER_TOPIC={{env_name}}.broadcast-transformer #FormsDB FORMS_DB_URL=postgresql://{{uci_postgres_host}}:5432/{{uci_forms_postgres_database}} @@ -26,6 +28,7 @@ FORMS_DB_PORT=5432 CASSANDRA_URL={{sunbird_cassandra_host}} CASSANDRA_PORT=9042 CASSANDRA_KEYSPACE={{env}}_uci_store +CASSANDRA_MIGRATION_COUNT=1 #Ports TRANSFORMER_INTERNAL_PORT=9091 @@ -61,4 +64,14 @@ ENV={{env}} REDIS_HOST={{sunbird_redis_host}} REDIS_PASS={{sunbird_redis_pass | default('')}} REDIS_PORT={{sunbird_redis_port | default(6379)}} -REDIS_DB_INDEX={{redis_db_index_uci | default('7')}} \ No newline at end of file +REDIS_DB_INDEX={{redis_db_index_uci | default('7')}} + +# Events +EXHAUST_TELEMETRY_ENABLED=TRUE +POSTHOG_EVENT_ENABLED=FALSE + +#Azure Config +AZURE_BLOB_STORE_CONTAINER={{sunbird_azure_uci_container_name | default('uci-' + env )}} +AZURE_BLOB_STORE_ACCOUNT_KEY={{sunbird_private_storage_account_name}} +AZURE_BLOB_STORE_ACCOUNT_NAME={{sunbird_private_storage_account_key}} +SELECTED_FILE_CDN=azure \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/GraylogMongoImport/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/GraylogMongoImport/config.xml index 206745d5c5..abc4d93760 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/GraylogMongoImport/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/GraylogMongoImport/config.xml @@ -91,7 +91,6 @@ NOTE:<br> 'content_packs_installations', 'event_definitions', 'event_notifications', -'event_notification_status', 'grants', 'grok_patterns', 'index_sets', diff --git a/kubernetes/ansible/api-manager.yml b/kubernetes/ansible/api-manager.yml index 77562241f1..6c1d7de7c4 100644 --- a/kubernetes/ansible/api-manager.yml +++ b/kubernetes/ansible/api-manager.yml @@ -4,6 +4,7 @@ become: yes environment: KUBECONFIG: "{{ kubeconfig_path }}" + ANSIBLE_PYTHON_INTERPRETER: "/usr/bin/python2" vars_files: - "{{inventory_dir}}/secrets.yml" roles: diff --git a/kubernetes/ansible/namespace.yml b/kubernetes/ansible/namespace.yml new file mode 100644 index 0000000000..bc62234c2c --- /dev/null +++ b/kubernetes/ansible/namespace.yml @@ -0,0 +1,39 @@ +--- +# tasks file for creating namespace and imagepull secrets + +- hosts: local + gather_facts: false + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + serial: 1 + vars: + helm_vars: + reloader: + watchGlobally: false + vars_files: + - "{{inventory_dir}}/secrets.yml" + pre_tasks: + - name: Creating tmp file + tempfile: + state: file + suffix: helm + register: helm_file + - name: Creating helm_charts vars file + copy: + content: "{{ helm_vars|to_yaml }}" + dest: "{{ helm_file.path }}" + - name: Creating namespace + shell: "kubectl create namespace {{ item }} " + with_items: + - "{{ bootstrap_namespace.split(',') }}" + ignore_errors: yes + - name: Creating docker secrets + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + when: imagepullsecrets|length > 0 + with_items: + - "{{ bootstrap_namespace.split(',') }}" + + - name: Installing reloader for configmaps reload + shell: helm upgrade --install --atomic reloader "{{ playbook_dir }}"/../helm_charts/core/reloader --namespace "{{ item }}" -f "{{ helm_file.path }}" + with_items: + - "{{ bootstrap_namespace.split(',') }}" diff --git a/kubernetes/ansible/roles/kong-api/tasks/main.yml b/kubernetes/ansible/roles/kong-api/tasks/main.yml index 82a9f77439..d85575dece 100644 --- a/kubernetes/ansible/roles/kong-api/tasks/main.yml +++ b/kubernetes/ansible/roles/kong-api/tasks/main.yml @@ -21,3 +21,6 @@ - name: Run script to save apis shell: "python /tmp/kong-api-scripts/kong_apis.py /tmp/kong_apis.json --kong-admin-api-url=http://{{ private_ingressgateway_ip }}/admin-api" + +- name: Reload kong + shell: kubectl get pod -l app=apimanager --namespace={{namespace}} -o name | xargs -I{} kubectl exec {} -- kong reload \ No newline at end of file diff --git a/kubernetes/ansible/roles/kong-consumer/tasks/main.yml b/kubernetes/ansible/roles/kong-consumer/tasks/main.yml index 2300a372da..edb1b4c0a4 100644 --- a/kubernetes/ansible/roles/kong-consumer/tasks/main.yml +++ b/kubernetes/ansible/roles/kong-consumer/tasks/main.yml @@ -48,3 +48,6 @@ with_items: - /tmp/jwt_token_output.txt - /tmp/jwt_token.txt + +- name: Reload kong + shell: kubectl get pod -l app=apimanager --namespace={{namespace}} -o name | xargs -I{} kubectl exec {} -- kong reload \ No newline at end of file diff --git a/kubernetes/ansible/roles/opa-test-coverage/defaults/main.yml b/kubernetes/ansible/roles/opa-test-coverage/defaults/main.yml index 96e556a904..2da3a4ed45 100644 --- a/kubernetes/ansible/roles/opa-test-coverage/defaults/main.yml +++ b/kubernetes/ansible/roles/opa-test-coverage/defaults/main.yml @@ -24,7 +24,7 @@ keycloak_public_key: stdout: '-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwTS+srpbA/n7I5fDjxcf\nH1QUSiFMP7B4yyLxir3VigKtai+YW7ZqMF853O3OiE/QTKuT0ZsR7aDtCsoZaGqq\n2NSyfbc3m339dBQ+0YgM5UdqvUHubNkgQwkRwb1lGlMWSHMYk0iIKJEIw1MCdCH0\nOBwjzlDxHb78lCKd/GvBvRLMXOJbAP72xJ/gjaNFHQ/GzskhrapDuMXaH+S7vtox\nBDG74fQwXCG+nDJ9ryV2bbWxWMINLU82x3+L6YYnWK80loucm2fzG5l1W/Wz9DIa\nGiWwP0JHlZf9GM/raydB4kDEq3jB22LfdLCdQanMkxbw7bkmvjBT2NFwqgFv3q3m\nVQIDAQAB\n-----END PUBLIC KEY-----' keycloak_auth_server_url: "https://sunbirded.org/auth" keycloak_realm: "sunbird" -private_ingressgateway_ip: "1.2.3.4" +private_ingressgateway_ip: "2.3.4.5" # Audience claim check is disabled as of now # keycloak_allowed_aud: '"{{ keycloak_auth_server_url }}/realms/{{ keycloak_realm }}", "account", "realm-management"' diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml index 6c217b1832..f4422e608b 100644 --- a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml +++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml @@ -125,6 +125,7 @@ data: set $target http://report-service.{{ .Values.namespace }}.svc.cluster.local:3030; rewrite ^/report/(.*) /$1 break; proxy_http_version 1.1; + proxy_set_header Host $server_name; proxy_pass $target; } location /search/ { diff --git a/kubernetes/helm_charts/sunbird-RC/registry/values.j2 b/kubernetes/helm_charts/sunbird-RC/registry/values.j2 index b7b4bbbb73..e7e6e6f31c 100644 --- a/kubernetes/helm_charts/sunbird-RC/registry/values.j2 +++ b/kubernetes/helm_charts/sunbird-RC/registry/values.j2 @@ -80,4 +80,4 @@ serviceMonitor: enabled: true labels: # labels with which the prometheus choose the serviceMonitor app: prometheus-operator - release: prometheus-operator \ No newline at end of file + release: prometheus-operator diff --git a/kubernetes/opa/common/common.rego b/kubernetes/opa/common/common.rego index 9023b7250d..63b3158886 100644 --- a/kubernetes/opa/common/common.rego +++ b/kubernetes/opa/common/common.rego @@ -22,11 +22,11 @@ ROLES := { "PROGRAM_DESIGNER": ["submitDataExhaustRequest", "getDataExhaustRequest", "listDataExhaustRequest"], - "ORG_ADMIN": ["acceptTnc", "assignRole", "submitDataExhaustRequest", "getDataExhaustRequest", "listDataExhaustRequest", "getUserProfileV5", "updateUserV2", "readUserConsent", "createTenantPreferences", "updateTenantPreferences", "getReport", "listReports", "createReport", "deleteReport", "updateReport", "publishReport", "retireReport", "getReportSummary", "listReportSummary", "createReportSummary"], + "ORG_ADMIN": ["acceptTnc", "assignRole", "submitDataExhaustRequest", "getDataExhaustRequest", "listDataExhaustRequest", "getUserProfileV5", "updateUserV2", "readUserConsent", "createTenantPreferences", "updateTenantPreferences", "createReport", "deleteReport", "updateReport", "publishReport", "retireReport", "getReportSummary", "listReportSummary", "createReportSummary"], - "REPORT_VIEWER": ["acceptTnc", "getReport", "listReports", "getReportSummary", "listReportSummary"], + "REPORT_VIEWER": ["acceptTnc", "getReportSummary", "listReportSummary"], - "REPORT_ADMIN": ["submitDataExhaustRequest", "getDataExhaustRequest", "listDataExhaustRequest", "acceptTnc", "getReport", "listReports", "createReport", "deleteReport", "updateReport", "publishReport", "retireReport", "getReportSummary", "listReportSummary", "createReportSummary"], + "REPORT_ADMIN": ["submitDataExhaustRequest", "getDataExhaustRequest", "listDataExhaustRequest", "acceptTnc", "createReport", "deleteReport", "updateReport", "publishReport", "retireReport", "getReportSummary", "listReportSummary", "createReportSummary"], "PUBLIC": ["PUBLIC"] } diff --git a/kubernetes/opa/learner/policies.rego b/kubernetes/opa/learner/policies.rego index 218faee2b1..6bdfd97441 100644 --- a/kubernetes/opa/learner/policies.rego +++ b/kubernetes/opa/learner/policies.rego @@ -30,32 +30,85 @@ urls_to_action_mapping := { "/v2/org/preferences/update": "updateTenantPreferences" } +# Tnc API policy updates to handle different scenarios as explained below +# When some or all payloads are missing: +# 1. Missing userid and tnc type +# 2. Missing tnc type +# 3. Missing userid and tnc type not as orgAdminTnc / reportViewerTnc +# 4. Missing userid but tnc type as orgAdminTnc / reportViewerTnc +# When all payloads are present: +# 5. Both userid, tnc type present and tnc type not as orgAdminTnc / reportViewerTnc +# 6. Both userid, tnc type present and tnc type as orgAdminTnc / reportViewerTnc +# Issue identified as part of - +# - https://project-sunbird.atlassian.net/browse/SB-29723 +# - https://project-sunbird.atlassian.net/browse/SB-29996 + +# Point #1 +acceptTermsAndCondition { + super.public_role_check + not input.parsed_body.request.userId + not input.parsed_body.request.tncType +} + +# Point #2 +acceptTermsAndCondition { + super.public_role_check + input.parsed_body.request.userId == super.userid + not input.parsed_body.request.tncType +} + +# Point #3 +acceptTermsAndCondition { + super.public_role_check + not input.parsed_body.request.userId + not input.parsed_body.request.tncType in ["orgAdminTnc", "reportViewerTnc"] +} + +# Point #4 - As orgAdminTnc acceptTermsAndCondition { acls := ["acceptTnc"] roles := ["ORG_ADMIN"] super.acls_check(acls) super.role_check(roles) + not input.parsed_body.request.userId "orgAdminTnc" == input.parsed_body.request.tncType } +# Point #4 - As reportViewerTnc acceptTermsAndCondition { acls := ["acceptTnc"] roles := ["REPORT_VIEWER", "REPORT_ADMIN"] super.acls_check(acls) super.role_check(roles) + not input.parsed_body.request.userId "reportViewerTnc" == input.parsed_body.request.tncType } +# Point #5 acceptTermsAndCondition { super.public_role_check input.parsed_body.request.userId == super.userid + not input.parsed_body.request.tncType in ["orgAdminTnc", "reportViewerTnc"] } -# Optional request.userId - https://project-sunbird.atlassian.net/browse/SB-29723 +# Point #6 - As orgAdminTnc acceptTermsAndCondition { - super.public_role_check - not input.parsed_body.request.tncType - not input.parsed_body.request.userId + acls := ["acceptTnc"] + roles := ["ORG_ADMIN"] + super.acls_check(acls) + super.role_check(roles) + input.parsed_body.request.userId == super.userid + "orgAdminTnc" == input.parsed_body.request.tncType +} + +# Point #6 - As reportViewerTnc +acceptTermsAndCondition { + acls := ["acceptTnc"] + roles := ["REPORT_VIEWER", "REPORT_ADMIN"] + super.acls_check(acls) + super.role_check(roles) + input.parsed_body.request.userId == super.userid + "reportViewerTnc" == input.parsed_body.request.tncType } updateUser { @@ -88,6 +141,15 @@ assignRoleV2 { payload_orgs == matching_orgs } +# https://project-sunbird.atlassian.net/browse/SB-30186 +# Allow the request to go through if the organisationId is an array type in order to receive a 400 Bad Request error from backend +assignRoleV2 { + acls := ["assignRole"] + roles := ["ORG_ADMIN"] + super.acls_check(acls) + type_name(input.parsed_body.request.roles[_].scope[_].organisationId) == "array" +} + getUserProfile { super.public_role_check user_id := split(http_request.path, "/")[4] @@ -138,6 +200,7 @@ userFeed { split(user_id, "?")[0] == super.userid } +# https://project-sunbird.atlassian.net/browse/SB-29951 # Temporary fix as all feed url's begin with /v1/user/feed # Having only the userFeed (/v1/user/feed/:userid) block is causing issues for other similar routes like /v1/user/feed/create, /v1/user/feed/delete and /v1/user/feed/update # Adding the other url blocks below and making them a pass through to avoid rejecting the API incorrectly diff --git a/kubernetes/opa/learner/policies_test.rego b/kubernetes/opa/learner/policies_test.rego index 5b031411b3..ca2ab15ee0 100644 --- a/kubernetes/opa/learner/policies_test.rego +++ b/kubernetes/opa/learner/policies_test.rego @@ -32,6 +32,32 @@ test_accept_terms_and_conditions_as_org_admin { } } +test_accept_terms_and_conditions_as_org_admin_with_userid { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with input as + { + "attributes": { + "request": { + "http": { + "headers": { + "x-authenticated-user-token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6ImFjY2Vzc3YxX2tleTEifQ.eyJhdWQiOiJodHRwczovL3N1bmJpcmRlZC5vcmcvYXV0aC9yZWFsbXMvc3VuYmlyZCIsInN1YiI6ImY6NWJiNmM4N2MtN2M4OC00ZDJiLWFmN2UtNTM0YTJmZWY5NzhkOjI4YjBkMDhmLWMyZWEtNDBkMS1iY2QwLThhZTAwZmNhNjZiZSIsInJvbGVzIjpbeyJyb2xlIjoiQk9PS19DUkVBVE9SIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6IkNPTlRFTlRfQ1JFQVRPUiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn1dfSx7InJvbGUiOiJDT05URU5UX1JFVklFV0VSIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6IkNPVVJTRV9NRU5UT1IiLCJzY29wZSI6W3sib3JnYW5pc2F0aW9uSWQiOiIwMTM2OTg3ODc5NzUwMzY5MjgxMCJ9XX0seyJyb2xlIjoiUFJPR1JBTV9ERVNJR05FUiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn1dfSx7InJvbGUiOiJSRVBPUlRfVklFV0VSIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6Ik9SR19BRE1JTiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn0seyJvcmdhbmlzYXRpb25JZCI6IjAxNDcxOTIzNTY3ODEyMzQ1Njc4In1dfSx7InJvbGUiOiJQVUJMSUMiLCJzY29wZSI6W119XSwiaXNzIjoiaHR0cHM6Ly9zdW5iaXJkZWQub3JnL2F1dGgvcmVhbG1zL3N1bmJpcmQiLCJuYW1lIjoiZGVtbyIsInR5cCI6IkJlYXJlciIsImV4cCI6MTY0MDIzNjEwMiwiaWF0IjoxNjQwMTQ5NzA1fQ.B3-TSdYSOlawPHjFdiRjXwvRbYQ_eH_HTiLKlH7vGS0rCOJ6HQbYyWOhZ7vbZPb3virkuyfhykFcYCEHBCkHY-fwGAeU58Pmhi0dnNJkR59Fa9y_75W98JXZW68HROp62ntEAKCA1oot_U4tYi-8UNoR17Gszj9iYzFEBc6TZA4Lrom_9gqhBOYzL0ISFWSS6oG94EaaKDYHyWzCSjU2nYRB_fn-tODmnVJ12GRJAc1oM9y54o8neNYsl4T_xPyD34v-CinUJM8jzDjFqK5_O3HnAbcmXvkZjFRgfk4mF1V4s5hlsTJGyhi2JOPh90C5N-HbAY8QsPBnzgYFQU_sww" + }, + "path": "/v1/user/tnc/accept" + } + } + }, + "parsed_body": { + "request": { + "userId": "28b0d08f-c2ea-40d1-bcd0-8ae00fca66be", + "tncType": "orgAdminTnc", + "version": "4.7.0" + } + } + } +} + test_accept_terms_and_conditions_as_report_viewer { data.main.allow.allowed with data.common.current_time as current_time @@ -57,6 +83,32 @@ test_accept_terms_and_conditions_as_report_viewer { } } +test_accept_terms_and_conditions_as_report_viewer_with_userid { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with input as + { + "attributes": { + "request": { + "http": { + "headers": { + "x-authenticated-user-token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6ImFjY2Vzc3YxX2tleTEifQ.eyJhdWQiOiJodHRwczovL3N1bmJpcmRlZC5vcmcvYXV0aC9yZWFsbXMvc3VuYmlyZCIsInN1YiI6ImY6NWJiNmM4N2MtN2M4OC00ZDJiLWFmN2UtNTM0YTJmZWY5NzhkOjI4YjBkMDhmLWMyZWEtNDBkMS1iY2QwLThhZTAwZmNhNjZiZSIsInJvbGVzIjpbeyJyb2xlIjoiQk9PS19DUkVBVE9SIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6IkNPTlRFTlRfQ1JFQVRPUiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn1dfSx7InJvbGUiOiJDT05URU5UX1JFVklFV0VSIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6IkNPVVJTRV9NRU5UT1IiLCJzY29wZSI6W3sib3JnYW5pc2F0aW9uSWQiOiIwMTM2OTg3ODc5NzUwMzY5MjgxMCJ9XX0seyJyb2xlIjoiUFJPR1JBTV9ERVNJR05FUiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn1dfSx7InJvbGUiOiJSRVBPUlRfVklFV0VSIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6Ik9SR19BRE1JTiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn0seyJvcmdhbmlzYXRpb25JZCI6IjAxNDcxOTIzNTY3ODEyMzQ1Njc4In1dfSx7InJvbGUiOiJQVUJMSUMiLCJzY29wZSI6W119XSwiaXNzIjoiaHR0cHM6Ly9zdW5iaXJkZWQub3JnL2F1dGgvcmVhbG1zL3N1bmJpcmQiLCJuYW1lIjoiZGVtbyIsInR5cCI6IkJlYXJlciIsImV4cCI6MTY0MDIzNjEwMiwiaWF0IjoxNjQwMTQ5NzA1fQ.B3-TSdYSOlawPHjFdiRjXwvRbYQ_eH_HTiLKlH7vGS0rCOJ6HQbYyWOhZ7vbZPb3virkuyfhykFcYCEHBCkHY-fwGAeU58Pmhi0dnNJkR59Fa9y_75W98JXZW68HROp62ntEAKCA1oot_U4tYi-8UNoR17Gszj9iYzFEBc6TZA4Lrom_9gqhBOYzL0ISFWSS6oG94EaaKDYHyWzCSjU2nYRB_fn-tODmnVJ12GRJAc1oM9y54o8neNYsl4T_xPyD34v-CinUJM8jzDjFqK5_O3HnAbcmXvkZjFRgfk4mF1V4s5hlsTJGyhi2JOPh90C5N-HbAY8QsPBnzgYFQU_sww" + }, + "path": "/v1/user/tnc/accept" + } + } + }, + "parsed_body": { + "request": { + "userId": "28b0d08f-c2ea-40d1-bcd0-8ae00fca66be", + "tncType": "reportViewerTnc", + "version": "4.7.0" + } + } + } +} + test_accept_terms_and_conditions_as_public_user { data.main.allow.allowed with data.common.current_time as current_time @@ -106,6 +158,57 @@ test_accept_terms_and_conditions_as_public_user_without_userid { } } +test_accept_terms_and_conditions_as_public_user_without_userid_other_tnc_types { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with input as + { + "attributes": { + "request": { + "http": { + "headers": { + "x-authenticated-user-token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6ImFjY2Vzc3YxX2tleTEifQ.eyJhdWQiOiJodHRwczovL3N1bmJpcmRlZC5vcmcvYXV0aC9yZWFsbXMvc3VuYmlyZCIsInN1YiI6ImY6NWJiNmM4N2MtN2M4OC00ZDJiLWFmN2UtNTM0YTJmZWY5NzhkOjI4YjBkMDhmLWMyZWEtNDBkMS1iY2QwLThhZTAwZmNhNjZiZSIsInJvbGVzIjpbeyJyb2xlIjoiUFVCTElDIiwic2NvcGUiOltdfV0sImlzcyI6Imh0dHBzOi8vc3VuYmlyZGVkLm9yZy9hdXRoL3JlYWxtcy9zdW5iaXJkIiwibmFtZSI6ImRlbW8iLCJ0eXAiOiJCZWFyZXIiLCJleHAiOjE2NDAyMzYxMDIsImlhdCI6MTY0MDE0OTcwNX0.iyFqdJG_9xF07S94bkfVDiWHmDWAmhCEmapu37Mto78s5OkOJQy-agXFjtQtgV5rFudHiVRukNpKXqlJ8GhasmW7fSEPL-fDKMilMIi4JCZi7d19AkFeq8mX0rI31m3zjCv-TcMPPWWNM4udR7kSj-tUOB-vupGZ1iRAtQU2lqrUCl1A84UYDqnJTokz6RVlr_Z4lRCzFn__aGsDZXO8h7juM4mAepVMy3wVhmbKR2R5WF5xQIvVjzEveRYj8P26VUg73wo_RtyRI5mQjbxyBaIX287pWe3kCu1KKwYQkBlRLx9da39g9TKZWXxD5ArCYMC83EmEeFI0LJicYDTXFg" + }, + "path": "/v1/user/tnc/accept" + } + } + }, + "parsed_body": { + "request": { + "version": "v8", + "tncType": "groupsTnc" + } + } + } +} + +test_accept_terms_and_conditions_as_public_user_with_userid_other_tnc_types { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with input as + { + "attributes": { + "request": { + "http": { + "headers": { + "x-authenticated-user-token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6ImFjY2Vzc3YxX2tleTEifQ.eyJhdWQiOiJodHRwczovL3N1bmJpcmRlZC5vcmcvYXV0aC9yZWFsbXMvc3VuYmlyZCIsInN1YiI6ImY6NWJiNmM4N2MtN2M4OC00ZDJiLWFmN2UtNTM0YTJmZWY5NzhkOjI4YjBkMDhmLWMyZWEtNDBkMS1iY2QwLThhZTAwZmNhNjZiZSIsInJvbGVzIjpbeyJyb2xlIjoiUFVCTElDIiwic2NvcGUiOltdfV0sImlzcyI6Imh0dHBzOi8vc3VuYmlyZGVkLm9yZy9hdXRoL3JlYWxtcy9zdW5iaXJkIiwibmFtZSI6ImRlbW8iLCJ0eXAiOiJCZWFyZXIiLCJleHAiOjE2NDAyMzYxMDIsImlhdCI6MTY0MDE0OTcwNX0.iyFqdJG_9xF07S94bkfVDiWHmDWAmhCEmapu37Mto78s5OkOJQy-agXFjtQtgV5rFudHiVRukNpKXqlJ8GhasmW7fSEPL-fDKMilMIi4JCZi7d19AkFeq8mX0rI31m3zjCv-TcMPPWWNM4udR7kSj-tUOB-vupGZ1iRAtQU2lqrUCl1A84UYDqnJTokz6RVlr_Z4lRCzFn__aGsDZXO8h7juM4mAepVMy3wVhmbKR2R5WF5xQIvVjzEveRYj8P26VUg73wo_RtyRI5mQjbxyBaIX287pWe3kCu1KKwYQkBlRLx9da39g9TKZWXxD5ArCYMC83EmEeFI0LJicYDTXFg" + }, + "path": "/v1/user/tnc/accept" + } + } + }, + "parsed_body": { + "request": { + "userId": "28b0d08f-c2ea-40d1-bcd0-8ae00fca66be", + "version": "v8", + "tncType": "groupsTnc" + } + } + } +} + test_update_user { data.main.allow.allowed with data.common.current_time as current_time @@ -194,6 +297,44 @@ test_assign_role_v2 { } } +test_assign_role_v2 { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with input as + { + "attributes": { + "request": { + "http": { + "headers": { + "x-authenticated-user-token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6ImFjY2Vzc3YxX2tleTEifQ.eyJhdWQiOiJodHRwczovL3N1bmJpcmRlZC5vcmcvYXV0aC9yZWFsbXMvc3VuYmlyZCIsInN1YiI6ImY6NWJiNmM4N2MtN2M4OC00ZDJiLWFmN2UtNTM0YTJmZWY5NzhkOjI4YjBkMDhmLWMyZWEtNDBkMS1iY2QwLThhZTAwZmNhNjZiZSIsInJvbGVzIjpbeyJyb2xlIjoiQk9PS19DUkVBVE9SIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6IkNPTlRFTlRfQ1JFQVRPUiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn1dfSx7InJvbGUiOiJDT05URU5UX1JFVklFV0VSIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6IkNPVVJTRV9NRU5UT1IiLCJzY29wZSI6W3sib3JnYW5pc2F0aW9uSWQiOiIwMTM2OTg3ODc5NzUwMzY5MjgxMCJ9XX0seyJyb2xlIjoiUFJPR1JBTV9ERVNJR05FUiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn1dfSx7InJvbGUiOiJSRVBPUlRfVklFV0VSIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6Ik9SR19BRE1JTiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn0seyJvcmdhbmlzYXRpb25JZCI6IjAxNDcxOTIzNTY3ODEyMzQ1Njc4In1dfSx7InJvbGUiOiJQVUJMSUMiLCJzY29wZSI6W119XSwiaXNzIjoiaHR0cHM6Ly9zdW5iaXJkZWQub3JnL2F1dGgvcmVhbG1zL3N1bmJpcmQiLCJuYW1lIjoiZGVtbyIsInR5cCI6IkJlYXJlciIsImV4cCI6MTY0MDIzNjEwMiwiaWF0IjoxNjQwMTQ5NzA1fQ.B3-TSdYSOlawPHjFdiRjXwvRbYQ_eH_HTiLKlH7vGS0rCOJ6HQbYyWOhZ7vbZPb3virkuyfhykFcYCEHBCkHY-fwGAeU58Pmhi0dnNJkR59Fa9y_75W98JXZW68HROp62ntEAKCA1oot_U4tYi-8UNoR17Gszj9iYzFEBc6TZA4Lrom_9gqhBOYzL0ISFWSS6oG94EaaKDYHyWzCSjU2nYRB_fn-tODmnVJ12GRJAc1oM9y54o8neNYsl4T_xPyD34v-CinUJM8jzDjFqK5_O3HnAbcmXvkZjFRgfk4mF1V4s5hlsTJGyhi2JOPh90C5N-HbAY8QsPBnzgYFQU_sww" + }, + "path": "/v2/user/assign/role" + } + } + }, + "parsed_body": { + "request": { + "userId": "fcae65a6-8a48-11ec-8c82-c7075e84952d", + "roles": [{ + "role": "COURSE_CREATOR", + "operation":"remove", + "scope": [{ + "organisationId": "01369878797503692810" + }] + }, + { + "role": "CONTENT_CREATOR", + "operation":"add", + "scope": [{ + "organisationId": ["01471923567812345678"] + }] + }] + } + } + } +} + test_get_user_profile { data.main.allow.allowed with data.common.current_time as current_time diff --git a/kubernetes/opa/registry/policies_test.rego b/kubernetes/opa/registry/policies_test.rego index 01bd3c7a32..1e37608805 100644 --- a/kubernetes/opa/registry/policies_test.rego +++ b/kubernetes/opa/registry/policies_test.rego @@ -6,11 +6,13 @@ package tests current_time := 1640235102 iss := "https://sunbirded.org/auth/realms/sunbird" +private_ingressgateway_ip := "1.2.3.4" test_rc_certificate_create_internal_request { data.main.allow.allowed with data.common.current_time as current_time with data.common.iss as iss + with data.common.private_ingressgateway_ip as private_ingressgateway_ip with input as { "attributes": { @@ -65,6 +67,7 @@ test_rc_certificate_delete_internal_request { data.main.allow.allowed with data.common.current_time as current_time with data.common.iss as iss + with data.common.private_ingressgateway_ip as private_ingressgateway_ip with input as { "attributes": { diff --git a/kubernetes/opa/report/policies.rego b/kubernetes/opa/report/policies.rego index 728bb245bc..6b9a96d48b 100644 --- a/kubernetes/opa/report/policies.rego +++ b/kubernetes/opa/report/policies.rego @@ -1,6 +1,9 @@ package policies import data.common as super +import input.attributes.request.http as http_request + +x_authenticated_user_token := http_request.headers["x-authenticated-user-token"] urls_to_action_mapping := { "/report/get": "getReport", @@ -12,21 +15,28 @@ urls_to_action_mapping := { "/report/retire": "retireReport", "/report/summary": "getReportSummary", "/report/summary/list": "listReportSummary", - "/report/summary/create": "createReportSummary" + "/report/summary/create": "createReportSummary", + "/report/datasets/get": "getReportDatasets" } getReport { - acls := ["getReport"] - roles := ["REPORT_ADMIN", "REPORT_VIEWER", "ORG_ADMIN"] - super.acls_check(acls) - super.role_check(roles) + super.public_role_check +} + +getReport { + not x_authenticated_user_token +} + +getReport { + super.is_an_internal_request } listReports { - acls := ["listReports"] - roles := ["REPORT_ADMIN", "REPORT_VIEWER", "ORG_ADMIN"] - super.acls_check(acls) - super.role_check(roles) + super.public_role_check +} + +listReports { + not x_authenticated_user_token } createReport { @@ -37,6 +47,10 @@ createReport { input.parsed_body.request.report.createdby == super.userid } +createReport { + super.is_an_internal_request +} + deleteReport { acls := ["deleteReport"] roles := ["REPORT_ADMIN", "ORG_ADMIN"] @@ -51,6 +65,10 @@ updateReport { super.role_check(roles) } +updateReport { + super.is_an_internal_request +} + publishReport { acls := ["publishReport"] roles := ["REPORT_ADMIN", "ORG_ADMIN"] @@ -85,4 +103,12 @@ createReportSummary { super.acls_check(acls) super.role_check(roles) input.parsed_body.request.summary.createdby == super.userid +} + +getReportDatasets { + super.public_role_check +} + +getReportDatasets { + not x_authenticated_user_token } \ No newline at end of file diff --git a/kubernetes/opa/report/policies_test.rego b/kubernetes/opa/report/policies_test.rego index 161c13bd70..10bd59faf7 100644 --- a/kubernetes/opa/report/policies_test.rego +++ b/kubernetes/opa/report/policies_test.rego @@ -6,6 +6,7 @@ package tests current_time := 1640235102 iss := "https://sunbirded.org/auth/realms/sunbird" +private_ingressgateway_ip := "1.2.3.4" test_get_report { data.main.allow.allowed @@ -26,6 +27,25 @@ test_get_report { } } +test_get_report_internal_request { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with data.common.private_ingressgateway_ip as private_ingressgateway_ip + with input as + { + "attributes": { + "request": { + "http": { + "headers": {}, + "path": "/report/get/1656a060-bf3a-11ec-b495-9fb99cdeb463", + "host": "1.2.3.4" + } + } + } + } +} + test_list_reports { data.main.allow.allowed with data.common.current_time as current_time @@ -50,6 +70,28 @@ test_list_reports { } } +test_list_reports_without_user_token { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with input as + { + "attributes": { + "request": { + "http": { + "headers": {}, + "path": "/report/list" + } + } + }, + "parsed_body": { + "request": { + "filters": {} + } + } + } +} + test_create_report { data.main.allow.allowed with data.common.current_time as current_time @@ -90,6 +132,46 @@ test_create_report { } } +test_create_report_internal_request { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with data.common.private_ingressgateway_ip as private_ingressgateway_ip + with input as + { + "attributes": { + "request": { + "http": { + "headers": {}, + "path": "/report/create", + "host": "1.2.3.4" + } + } + }, + "parsed_body": { + "request": { + "report": { + "title": "string", + "description": "string", + "authorizedroles": ["string"], + "status": "string", + "type": "string", + "createdby": "28b0d08f-c2ea-40d1-bcd0-8ae00fca66be", + "reportconfig": { + "id": "string", + "label": "string", + "title": "string", + "description": "string", + }, + "slug": "string", + "reportgenerateddate": "string", + "updatefrequency": "string" + } + } + } + } +} + test_delete_report { data.main.allow.allowed with data.common.current_time as current_time @@ -133,6 +215,30 @@ test_update_report { } } +test_update_report_internal_request { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with data.common.private_ingressgateway_ip as private_ingressgateway_ip + with input as + { + "attributes": { + "request": { + "http": { + "headers": {}, + "path": "/report/update", + "host": "1.2.3.4" + } + } + }, + "parsed_body": { + "request": { + "report": {} + } + } + } +} + test_publish_report { data.main.allow.allowed with data.common.current_time as current_time @@ -242,4 +348,40 @@ test_create_report_summary { } } } +} + +test_get_report_datasets { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with input as + { + "attributes": { + "request": { + "http": { + "headers": { + "x-authenticated-user-token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6ImFjY2Vzc3YxX2tleTEifQ.eyJhdWQiOiJodHRwczovL3N1bmJpcmRlZC5vcmcvYXV0aC9yZWFsbXMvc3VuYmlyZCIsInN1YiI6ImY6NWJiNmM4N2MtN2M4OC00ZDJiLWFmN2UtNTM0YTJmZWY5NzhkOjI4YjBkMDhmLWMyZWEtNDBkMS1iY2QwLThhZTAwZmNhNjZiZSIsInJvbGVzIjpbeyJyb2xlIjoiQk9PS19DUkVBVE9SIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6IkNPTlRFTlRfQ1JFQVRPUiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn1dfSx7InJvbGUiOiJDT05URU5UX1JFVklFV0VSIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6IkNPVVJTRV9NRU5UT1IiLCJzY29wZSI6W3sib3JnYW5pc2F0aW9uSWQiOiIwMTM2OTg3ODc5NzUwMzY5MjgxMCJ9XX0seyJyb2xlIjoiUFJPR1JBTV9ERVNJR05FUiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn1dfSx7InJvbGUiOiJSRVBPUlRfVklFV0VSIiwic2NvcGUiOlt7Im9yZ2FuaXNhdGlvbklkIjoiMDEzNjk4Nzg3OTc1MDM2OTI4MTAifV19LHsicm9sZSI6Ik9SR19BRE1JTiIsInNjb3BlIjpbeyJvcmdhbmlzYXRpb25JZCI6IjAxMzY5ODc4Nzk3NTAzNjkyODEwIn0seyJvcmdhbmlzYXRpb25JZCI6IjAxNDcxOTIzNTY3ODEyMzQ1Njc4In1dfSx7InJvbGUiOiJQVUJMSUMiLCJzY29wZSI6W119XSwiaXNzIjoiaHR0cHM6Ly9zdW5iaXJkZWQub3JnL2F1dGgvcmVhbG1zL3N1bmJpcmQiLCJuYW1lIjoiZGVtbyIsInR5cCI6IkJlYXJlciIsImV4cCI6MTY0MDIzNjEwMiwiaWF0IjoxNjQwMTQ5NzA1fQ.B3-TSdYSOlawPHjFdiRjXwvRbYQ_eH_HTiLKlH7vGS0rCOJ6HQbYyWOhZ7vbZPb3virkuyfhykFcYCEHBCkHY-fwGAeU58Pmhi0dnNJkR59Fa9y_75W98JXZW68HROp62ntEAKCA1oot_U4tYi-8UNoR17Gszj9iYzFEBc6TZA4Lrom_9gqhBOYzL0ISFWSS6oG94EaaKDYHyWzCSjU2nYRB_fn-tODmnVJ12GRJAc1oM9y54o8neNYsl4T_xPyD34v-CinUJM8jzDjFqK5_O3HnAbcmXvkZjFRgfk4mF1V4s5hlsTJGyhi2JOPh90C5N-HbAY8QsPBnzgYFQU_sww" + }, + "path": "/report/datasets/get/1656a060-bf3a-11ec-b495-9fb99cdeb463" + } + } + } + } +} + +test_get_report_datasets_without_user_token { + data.main.allow.allowed + with data.common.current_time as current_time + with data.common.iss as iss + with input as + { + "attributes": { + "request": { + "http": { + "headers": {}, + "path": "/report/datasets/get/1656a060-bf3a-11ec-b495-9fb99cdeb463" + } + } + } + } } \ No newline at end of file diff --git a/kubernetes/pipelines/bootstrap_minimal/Jenkinsfile.namespace b/kubernetes/pipelines/bootstrap_minimal/Jenkinsfile.namespace new file mode 100644 index 0000000000..3109c8dddb --- /dev/null +++ b/kubernetes/pipelines/bootstrap_minimal/Jenkinsfile.namespace @@ -0,0 +1,46 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + stage('deploy') { + values = [:] + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + currentWs = sh(returnStdout: true, script: 'pwd').trim() + ansiblePlaybook = "$currentWs/kubernetes/ansible/namespace.yml" + ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + ansible_playbook_run(values) + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}" + } + summary() + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml index 3e35beb556..b004ab86d2 100644 --- a/private_repo/ansible/inventory/dev/Core/secrets.yml +++ b/private_repo/ansible/inventory/dev/Core/secrets.yml @@ -65,7 +65,6 @@ adminutil_refresh_token_public_key_kid: "" # get after keycloak deployment, go #SELECT value FROM component_config CC INNER JOIN component C ON(CC.component_id = C.id) WHERE C.realm_id = 'sunbird' and provider_id = 'hmac-generated' AND CC.name = 'secret'; adminutil_refresh_token_secret_key: "" # get after Keycloak deployment from postgres using the above query - # mongodb keyfile content generated using # 'openssl rand -base64 741' # eg: @@ -102,6 +101,7 @@ core_vault_sunbird_google_oauth_clientId_portal: # Google oauth client i core_vault_sunbird_google_oauth_clientSecret_portal: # Google oauth client secret core_vault_sunbird_google_captcha_site_key_portal: # Google recaptch site key google_captcha_private_key: # Google recaptch private key +learning_content_drive_apiKey: # Google drive api key # ------------------------------------------------------------------------------------------------------------ # # Optional variables - Can be left blank if you dont plan to use the intended features