Skip to content

Commit

Permalink
Added new vars (#3392)
Browse files Browse the repository at this point in the history
  • Loading branch information
@raghupathiguduri
raghupathiguduri authored Jun 27, 2022
1 parent 3c82e28 commit cba2d5e
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 1 deletion.
5 changes: 4 additions & 1 deletion ansible/roles/ml-analytics-service/defaults/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,14 @@ ml_analytics_project_output_dir: "{{ WORKDIR }}/source/projects/output"
ml_analytics_observation_status_output_dir: "{{ WORKDIR }}/source/observations/status/output"
ml_analytics_api_authorization_key: "{{ml_api_auth_token | default('sunbird_api_auth_token')}}"
ml_analytics_api_access_token: "{{ml_api_access_token | default('ml_core_internal_access_token')}}"
ml_analytics_druid_observation_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/observation/status/sl_observation_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":["status","entity_externalId","entity_id","entity_type","solution_id","solution_externalId","submission_id","entity_name","solution_name","role_title","school_name","school_id","school_externalId","state_name","state_id","state_externalId","district_name","district_id","district_externalId","zone_name","zone_id","zone_externalId","parent_name","parent_id","block_name","block_id","block_externalId","cluster_name","cluster_id","cluster_externalId","teacher_name","teacher_id","teacher_externalId","schoolLeader_name","schoolLeader_id","schoolLeader_externalId","taluk_id","taluk_externalId","taluk_name","hub_name","hub_id","hub_externalId","mandal_name","mandal_id","mandal_externalId","region_name","region_id","region_externalId","complex_name","complex_id","complex_externalId","completedDate","beat_name","beat_id","beat_externalId","division_name","division_id","division_externalId","Village/Ward_name","Village/Ward_id","Village/Ward_externalId","channel","parent_channel","program_id","program_externalId","program_name","app_name","user_id","private_program","solution_type","organisation_name", "ecm_marked_na","board_name","updatedAt","organisation_id"]},"metricsSpec":[]}}}'
ml_analytics_druid_observation_status_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","firehose":{"type":"static-azure-blobstore","blobs":[{"container":"{{container_name}}","path":"/observation/status/sl_observation_status.json"}],"fetchTimeout":300000},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-observation-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"updatedAt","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"status"},{"type":"string","name":"entity_externalId"},{"type":"string","name":"entity_id"},{"type":"string","name":"entity_type"},{"type":"string","name":"solution_id"},{"type":"string","name":"solution_externalId"},{"type":"string","name":"submission_id"},{"type":"string","name":"entity_name"},{"type":"string","name":"solution_name"},{"type":"string","name":"role_title"},{"type":"string","name":"school_name"},{"type":"string","name":"school_id"},{"type":"string","name":"school_externalId"},{"type":"string","name":"state_name"},{"type":"string","name":"state_id"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_id"},{"type":"string","name":"district_externalId"},{"type":"string","name":"block_name"},{"type":"string","name":"block_id"},{"type":"string","name":"block_externalId"},{"type":"string","name":"cluster_name"},{"type":"string","name":"cluster_id"},{"type":"string","name":"cluster_externalId"},{"type":"string","name":"completedDate"},{"type":"string","name":"channel"},{"type":"string","name":"parent_channel"},{"type":"string","name":"program_id"},{"type":"string","name":"program_externalId"},{"type":"string","name":"program_name"},{"type":"string","name":"app_name"},{"type":"string","name":"user_id"},{"type":"string","name":"private_program"},{"type":"string","name":"solution_type"},{"type":"string","name":"organisation_name"},{"type":"string","name":"ecm_marked_na"},{"type":"string","name":"board_name"},{"type":"string","name":"updatedAt"},{"type":"string","name":"organisation_id","user_type"}]},"metricsSpec":[]}}}'
ml_analytics_druid_project_injestion_spec : '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type": "azure","uris": ["azure://telemetry-data-store/projects/sl_projects.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"sl-project","granularitySpec":{"type":"uniform","queryGranularity":"DAY","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"project_updated_date","format":"auto"},"dimensionsSpec":{"dimensions":[]},"metricsSpec":[]}}}'
ml_analytics_azure_sas_token: "{{ sunbird_private_storage_account_key }}"
ml_analytics_druid_distinctCnt_obs_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount/ml_observation_distinctCount_status.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"status"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
ml_analytics_druid_distinctCnt_obs_domain_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain/ml_observation_distinctCount_domain.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
ml_analytics_druid_distinctCnt_obs_domain_criteria_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/observation/distinctCount_domain_criteria/ml_observation_distinctCount_domain_criteria.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-obs-domain-criteria","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"solution_name"},{"type":"string","name":"solution_id"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"parent_channel"},{"type":"string","name":"solution_type"},{"type":"string","name":"private_program"},{"type":"string","name":"domain_name"},{"type":"string","name":"domain_externalId"},{"type":"string","name":"domain_level"},{"type":"string","name":"criteria_name"},{"type":"string","name":"criteria_score"},{"type":"string","name":"criteria_id"},{"type":"long","name":"unique_submissions"},{"type":"long","name":"unique_entities"},{"type":"long","name":"unique_users"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
ml_analytics_druid_distinctCnt_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCount/ml_projects_distinctCount.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"project_title"},{"type":"string","name":"solution_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"district_name"},{"type":"string","name":"district_externalId"},{"type":"string","name":"organisation_name"},{"type":"string","name":"organisation_id"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec: '{"type":"index","spec":{"ioConfig":{"type":"index","inputSource":{"type":"azure","uris":["azure://telemetry-data-store/projects/distinctCountPrglevel/ml_projects_distinctCount_prgmlevel.json"]},"inputFormat":{"type":"json"}},"tuningConfig":{"type":"index","partitionsSpec":{"type":"dynamic"}},"dataSchema":{"dataSource":"ml-project-programLevel-status","granularitySpec":{"type":"uniform","queryGranularity":"none","rollup":false,"segmentGranularity":"DAY"},"timestampSpec":{"column":"time_stamp","format":"auto"},"dimensionsSpec":{"dimensions":[{"type":"string","name":"program_name"},{"type":"string","name":"program_id"},{"type":"string","name":"status_of_project"},{"type":"string","name":"state_name"},{"type":"string","name":"state_externalId"},{"type":"string","name":"private_program"},{"type":"string","name":"project_created_type"},{"type":"string","name":"parent_channel"},{"type":"long","name":"unique_projects"},{"type":"long","name":"unique_users"},{"type":"long","name":"no_of_imp_with_evidence"},{"type":"string","name":"time_stamp"}]},"metricsSpec":[]}}}'
ml_analytics_obs_distinctCnt_azure_blob_path: "observation/distinctCount/"
ml_analytics_obs_distinctCnt_domain_azure_blob_path: "observation/distinctCount_domain/"
ml_analytics_obs_distinctCnt_domain_criteria_azure_blob_path: "observation/distinctCount_domain_criteria/"
Expand All @@ -63,3 +64,5 @@ ml_analytics_obs_distinctCnt_output_dir: "{{ WORKDIR }}/source/observations/dist
ml_analytics_obs_distinctCnt_domain_output_dir: "{{ WORKDIR }}/source/observations/distinctCount_domain/output"
ml_analytics_obs_distinctCnt_domain_criteria_output_dir: "{{ WORKDIR }}/source/observations/distinctCount_domain_criteria/output"
ml_analytics_projects_distinctCnt_output_dir: "{{ WORKDIR }}/source/projects/distinctCount/output"
ml_analytics_projects_distinctCnt_prglevel_output_dir: "{{ WORKDIR }}/source/projects/distinctCountPrglevel/output"
ml_analytics_projects_distinctCnt_prglevel_azure_blob_path: "projects/distinctCountPrglevel/"
6 changes: 6 additions & 0 deletions ansible/roles/ml-analytics-service/templates/config.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ ml_distinctCnt_obs_domain_criteria_spec = {{ ml_analytics_druid_distinctCnt_obs_

ml_distinctCnt_projects_status_spec = {{ ml_analytics_druid_distinctCnt_projects_status_injestion_spec }}

ml_distinctCnt_prglevel_projects_status_spec = {{ ml_analytics_druid_distinctCnt_prglevel_projects_status_injestion_spec }}

[KAFKA]

url = {{ ml_analytics_kafka_url }}
Expand Down Expand Up @@ -148,6 +150,8 @@ observation_distinctCount_domain_criteria_blob_path = {{ ml_analytics_obs_distin

projects_distinctCnt_blob_path = {{ ml_analytics_projects_distinctCnt_azure_blob_path }}

projects_distinctCnt_prgmlevel_blob_path = {{ ml_analytics_projects_distinctCnt_prglevel_azure_blob_path }}

[OUTPUT_DIR]

project = {{ ml_analytics_project_output_dir }}
Expand All @@ -161,3 +165,5 @@ observation_distinctCount_domain = {{ ml_analytics_obs_distinctCnt_domain_output
observation_distinctCount_domain_criteria = {{ ml_analytics_obs_distinctCnt_domain_criteria_output_dir }}

projects_distinctCount = {{ ml_analytics_projects_distinctCnt_output_dir }}

projects_distinctCount_prgmlevel = {{ ml_analytics_projects_distinctCnt_prglevel_output_dir }}

0 comments on commit cba2d5e

Please sign in to comment.