From 9c61b446ff39eb4e5f225c118c7e9dbfde13ad94 Mon Sep 17 00:00:00 2001 From: Jayasimha Raghavan <87547684+jayasimha-raghavan-unskript@users.noreply.github.com> Date: Wed, 18 Oct 2023 20:56:18 -0700 Subject: [PATCH] Bash completion script fix (#900) Co-authored-by: Jayasimha Raghavan --- .pylintrc | 1 + README_extending_docker.md | 6 +- unskript-ctl/add_notification.py | 9 +- .../bash_completion_unskript_ctl.bash | 96 +++- unskript-ctl/db_utils.py | 2 +- unskript-ctl/unskript-client.py | 476 +++++++++++++----- unskript-ctl/unskript_global.yaml.template | 5 +- 7 files changed, 430 insertions(+), 165 deletions(-) diff --git a/.pylintrc b/.pylintrc index 3e4529a20..ec7e2c42b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -154,6 +154,7 @@ disable=abstract-method, C0303, C0304, C0301, + C0412, E0203, R1732 diff --git a/README_extending_docker.md b/README_extending_docker.md index 3ea47fce5..9e7b55110 100644 --- a/README_extending_docker.md +++ b/README_extending_docker.md @@ -247,16 +247,16 @@ docker exec -it $CONTAINER_ID bash For example, if you want to get Slack Notification, here is the command to follow - ``` -add_notification.sh -c Slack -u https://hooks.slack.com/services/T12345/B12345/XXXXXXX --channel test-alerting +add_notification.sh -c Slack -u https://hooks.slack.com/services/T12345/B12345/XXXXXXX --channel-name test-alerting ``` > Here `-c` Option is used for creating a new Notification. Options are either Slack or SMTP (email) > `-u` Option is the webhook URL of Slack -> `--channel` Channel to where the notification should be sent +> `--channel-name` Channel to where the notification should be sent This snippet shows how to configure SMTP (email) Notification ``` -./add_notification.sh -c SMTP -s smtp.server.com -u username@domain.com -p -t receiver@example.com +add_notification.sh -c SMTP -s smtp.server.com -u username@domain.com -p -t receiver@example.com ``` > Here `-s` Option is to specify the SMTP server > `-u` Option is to specify the SMTP username. Note, you need to specify the username with the domain like username@domain.com diff --git a/unskript-ctl/add_notification.py b/unskript-ctl/add_notification.py index b3a28177e..0c9e6e296 100644 --- a/unskript-ctl/add_notification.py +++ b/unskript-ctl/add_notification.py @@ -11,6 +11,7 @@ import os import sys import json +import yaml from pathlib import Path import subprocess @@ -27,11 +28,15 @@ # Note, the earlier name we used was unskript_config.yaml. if os.path.exists('/unskript/data/action/unskript_config.yaml') is True: try: - os.makedirs(Path(GLOBAL_CONFIG_PATH).parent, exist_ok=True) + os.makedirs(Path(GLOBAL_CONFIG_PATH).parent, exist_ok=True) Path('/unskript/data/action/unskript_config.yaml').rename(GLOBAL_CONFIG_PATH) except: pass +# Lets ensure the Global Config file exists in the +# path +if os.path.exists(GLOBAL_CONFIG_PATH) is False: + Path(GLOBAL_CONFIG_PATH).touch() """ This class define basic parser that shall be extended based on each notification @@ -167,4 +172,4 @@ def read_notification(self, type): if __name__ == "__main__": """Notifcation class does the create of Crud""" - Notification() \ No newline at end of file + Notification() diff --git a/unskript-ctl/bash_completion_unskript_ctl.bash b/unskript-ctl/bash_completion_unskript_ctl.bash index 05da9e42f..e6c7e7a09 100755 --- a/unskript-ctl/bash_completion_unskript_ctl.bash +++ b/unskript-ctl/bash_completion_unskript_ctl.bash @@ -6,6 +6,7 @@ _unskript-client-completion() { COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" + connector_list=("aws" "k8s" "postgres" "mongodb" "elasticsearch" "vault" "ssh" "keycloak" "github" "redis") # Find the absolute path of unskript-client.py @@ -14,10 +15,12 @@ _unskript-client-completion() { if [ -n "$unskript_client_script" ]; then # Check if the script exists and save check names - /usr/bin/env python "$unskript_client_script" --save-check-names /tmp/checknames.txt - /usr/bin/env python "$unskript_client_script" -h > /tmp/allopts.txt - if [ -f "/tmp/checknames.txt" ]; then - opt2="$(cat /tmp/checknames.txt)" + if [ ! -f "/tmp/allopts.txt" ]; then + /usr/bin/env python "$unskript_client_script" -h > /tmp/allopts.txt + fi + if [ ! -f "/tmp/checknames.txt" ]; then + /usr/bin/env python "$unskript_client_script" --save-check-names /tmp/checknames.txt + fi fi # Define options with each option on a separate line using newline characters @@ -27,7 +30,7 @@ _unskript-client-completion() { case "${prev}" in -rr|--run-runbook) # Provide completion suggestions for runbook filenames - COMPREPLY=( $(compgen -f -- "${cur}") ) + COMPREPLY=( $(compgen -f -- "${cur}" -o nospace) ) return 0 ;; @@ -35,50 +38,111 @@ _unskript-client-completion() { case ${prev} in -rc|--run-checks) case ${cur} in - -f) - COMPREPLY=( ${opt2} ) + --check) + cur=${cur#--check} + opt2="$(cat /tmp/checknames.txt)" + COMPREPLY=( $(compgen -W "${opt2}" -o nospace) ) ;; *) - COMPREPLY=( $(compgen -W "--all | --type | --failed | --check" -- "${cur}") ) + #COMPREPLY=( $(compgen -W "--all --type --failed --check" -- "${cur}" -o nospace) ) + COMPREPLY=( $(compgen -W "--all --type --failed --check" -o nospace) ) ;; esac return 0 ;; *) - COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + COMPREPLY=( $(compgen -W "${opts}" "${cur}" -o nospace) ) ;; + esac return 0 ;; -lc|--list-checks) # Provide completion suggestions for list-checks options - COMPREPLY=( $(compgen -W "--all | --type" -- "${cur}") ) + COMPREPLY=( $(compgen -W "--all --type" -- "${cur}" -o nospace) ) + return 0 + ;; + + --credential-list) + # Provide completion suggestions for list-checks options + COMPREPLY=() return 0 ;; -sa|--show-audit-trail) # Provide completion suggestions for show-audit-trail options - COMPREPLY=( $(compgen -W "--all | --type | --execution_id" -- "${cur}") ) + COMPREPLY=( $(compgen -W "--all --type --execution_id" -- "${cur}" -o nospace) ) return 0 ;; -dl|--display-failed-logs) # Provide completion suggestions for display-failed-logs options - COMPREPLY=( $(compgen -W "--execution_id " -- "${cur}") ) + COMPREPLY=( $(compgen -W "--execution_id" -- "${cur}" -o nospace) ) return 0 ;; -cc|--create-credentials) # Provide completion suggestions for create-credentials options - COMPREPLY=( $(compgen -W "--type creds_file_path" -- "${cur}") ) + COMPREPLY=( $(compgen -W "--type" -- "${cur}" -o nospace) ) return 0 ;; *) # Default: Provide completion suggestions for global options - COMPREPLY=( $(compgen -W "${opts}" -- "${cur}" -o nospace) ) - return 0 - ;; + _cmd="${COMP_WORDS[COMP_CWORD-2]}" + if [ "${prev}" = "--check" ]; + then + cur=${cur#--check} + opt2="$(grep -E "^${cur}" /tmp/checknames.txt)" + COMPREPLY=( $(compgen -W "${opt2}" -o nospace) ) + compopt -o nospace + elif [ "${_cmd}" = "--check" ]; + then + COMPREPLY=( $(compgen -W "--report" -o nospace) ) + elif [ "${cur}" = "--report" ]; + then + COMPREPLY=() + elif [[ "${_cmd}" = '-cc' || "${_cmd}" = '--create-credentials' ]]; + then + COMPREPLY+=("-k8s ") + elif [[ "${_cmd}" = '-rc' || "${_cmd}" = '--run-checks' || "${_cmd}" = '--list-checks' || "${_cmd}" = '-lc' ]]; + then + if [ "${prev}" = "--type" ]; + then + COMPREPLY=( $(compgen -W "aws k8s postgres mongodb elasticsearch vault ssh keycloak github redis" -o nospace) ) + elif [[ "${prev}" = "--all" || "${prev}" = "--failed" ]]; + then + COMPREPLY+=('--report') + fi + elif [[ "${_cmd}" = "--type" && "${COMP_WORDS[COMP_CWORD-3]}" = '-rc' || "${COMP_WORDS[COMP_CWORD-3]}" = '--run-checks' ]]; + then + for item in "${connector_list[@]}"; + do + if [ "$item" == "${prev}" ]; + then + COMPREPLY+=("--report") + break + fi + done + elif [[ "${_cmd}" = '-sa' || "${_cmd}" = '--show-audit-trail' ]]; + then + if [ "${prev}" = '--type' ]; + then + COMPREPLY=( $(compgen -W "aws k8s postgres mongodb elasticsearch vault ssh keycloak github redis" -o nospace) ) + elif [ "${prev}" = '--execution_id' ]; + then + COMPREPLY=( $(compgen -W "" -o nospace) ) + fi + else + if [ "${#COMP_WORDS[@]}" != "2" ]; + then + return 0 + fi + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}" -o nospace) ) + fi + + return 0 + ;; esac } diff --git a/unskript-ctl/db_utils.py b/unskript-ctl/db_utils.py index bdeec7bd8..bf6af78f7 100644 --- a/unskript-ctl/db_utils.py +++ b/unskript-ctl/db_utils.py @@ -242,7 +242,7 @@ def get_checks_by_connector(connector_name: str, full_snippet: bool = False): if connector_name.lower() != 'all' and not re.match(connector_name.lower(), s_connector): continue if full_snippet is False: - list_checks.append([s_connector.capitalize(), d.get('name'), d.get('uuid')]) + list_checks.append([s_connector.capitalize(), d.get('name'), d.get('metadata').get('action_entry_function')]) else: list_checks.append(d) diff --git a/unskript-ctl/unskript-client.py b/unskript-ctl/unskript-client.py index e3f81d9bb..acb48e58b 100755 --- a/unskript-ctl/unskript-client.py +++ b/unskript-ctl/unskript-client.py @@ -17,6 +17,7 @@ import re import uuid import psutil +import pprint import subprocess import yaml import nbformat @@ -74,6 +75,7 @@ TBL_HDR_DSPL_CHKS_NAME="\033[35m Failed Check Name / TS \033[0m" TBL_HDR_DSPL_CHKS_UUID="\033[1m Failed Check UUID \033[0m" TBL_HDR_CHKS_UUID="\033[1m Check UUID \033[0m" +TBL_HDR_CHKS_FN="\033[1m Function Name \033[0m" TBL_HDR_LIST_CHKS_CONNECTOR="\033[36m Connector Name \033[0m" parser = ArgumentParser(prog='unskript-ctl') @@ -200,6 +202,8 @@ def insert_first_and_last_cell(nb: nbformat.NotebookNode) -> nbformat.NotebookNo return nb + + # These are all trigger functions that are # called based on argument passed def list_runbooks(): @@ -244,7 +248,7 @@ def run_ipynb(filename: str, status_list_of_dict: list = None, filter: str = Non "Get Failed Readiness Probes", "Get Failed Liveness Probes", "Get K8s DaemonSets Missing on Node" -] + ] nb = read_ipynb(filename) # We store the Status of runbook execution in status_dict @@ -275,6 +279,8 @@ def run_ipynb(filename: str, status_list_of_dict: list = None, filter: str = Non with open(output_file, "r") as f: new_nb = nbformat.read(f, as_version=4) outputs = get_last_code_cell_output(new_nb.dict()) + if len(outputs) == 0: + print("ERROR: Output of the cell execution is empty. Is the credential configured?") ids = get_code_cell_action_uuids(nb.dict()) result_table = [["Checks Name", "Result", "Failed Count", "Error"]] @@ -292,76 +298,78 @@ def run_ipynb(filename: str, status_list_of_dict: list = None, filter: str = Non connector, 'ERROR' ]) - results = {} + + results = [] if ids: - results = outputs[0] + results = outputs idx = 0 - r = results.get('text') failed_result_available = False failed_result = {} if ids: - for result in r.split('\n'): - if result == '': - continue - payload = json.loads(result) - - try: - if ids and CheckOutputStatus(payload.get('status')) == CheckOutputStatus.SUCCESS: - result_table.append([ - get_action_name_from_id(ids[idx], nb.dict()), - TBL_CELL_CONTENT_PASS, - 0, - 'N/A' - ]) - status_dict['result'].append([ - get_action_name_from_id(ids[idx], nb.dict()), - ids[idx], - get_connector_name_from_id(ids[idx], nb.dict()), - 'PASS'] - ) - elif ids and CheckOutputStatus(payload.get('status')) == CheckOutputStatus.FAILED: - failed_objects = payload.get('objects') - failed_result[get_action_name_from_id(ids[idx], nb.dict())] = failed_objects - result_table.append([ - get_action_name_from_id(ids[idx], nb.dict()), - TBL_CELL_CONTENT_FAIL, - len(failed_objects), - 'N/A' - ]) - failed_result_available = True - status_dict['result'].append([ - get_action_name_from_id(ids[idx], nb.dict()), - ids[idx], - get_connector_name_from_id(ids[idx], nb.dict()), - 'FAIL' - ]) - elif ids and CheckOutputStatus(payload.get('status')) == CheckOutputStatus.RUN_EXCEPTION: - if payload.get('error') is not None: - failed_objects = payload.get('error') + for output in outputs: + r = output.get('text') + for result in r.split('\n'): + if result == '': + continue + payload = json.loads(result) + + try: + if ids and CheckOutputStatus(payload.get('status')) == CheckOutputStatus.SUCCESS: + result_table.append([ + get_action_name_from_id(ids[idx], nb.dict()), + TBL_CELL_CONTENT_PASS, + 0, + 'N/A' + ]) + status_dict['result'].append([ + get_action_name_from_id(ids[idx], nb.dict()), + ids[idx], + get_connector_name_from_id(ids[idx], nb.dict()), + 'PASS'] + ) + elif ids and CheckOutputStatus(payload.get('status')) == CheckOutputStatus.FAILED: + failed_objects = payload.get('objects') failed_result[get_action_name_from_id(ids[idx], nb.dict())] = failed_objects - result_table.append([ - get_action_name_from_id(ids[idx], nb.dict()), - TBL_CELL_CONTENT_ERROR, - 0, - payload.get('error') - ]) - status_dict['result'].append([ - get_action_name_from_id(ids[idx], nb.dict()), - ids[idx], - get_connector_name_from_id(ids[idx], nb.dict()), - 'ERROR' - ]) - except Exception: - pass - if ids: + result_table.append([ + get_action_name_from_id(ids[idx], nb.dict()), + TBL_CELL_CONTENT_FAIL, + len(failed_objects), + 'N/A' + ]) + failed_result_available = True + status_dict['result'].append([ + get_action_name_from_id(ids[idx], nb.dict()), + ids[idx], + get_connector_name_from_id(ids[idx], nb.dict()), + 'FAIL' + ]) + elif ids and CheckOutputStatus(payload.get('status')) == CheckOutputStatus.RUN_EXCEPTION: + if payload.get('error') is not None: + failed_objects = payload.get('error') + failed_result[get_action_name_from_id(ids[idx], nb.dict())] = failed_objects + result_table.append([ + get_action_name_from_id(ids[idx], nb.dict()), + TBL_CELL_CONTENT_ERROR, + 0, + pprint.pformat(payload.get('error'), width=30) + ]) + status_dict['result'].append([ + get_action_name_from_id(ids[idx], nb.dict()), + ids[idx], + get_connector_name_from_id(ids[idx], nb.dict()), + 'ERROR' + ]) + except Exception: + pass + update_current_execution(payload.get('status'), ids[idx], nb.dict()) update_check_run_trail(ids[idx], get_action_name_from_id(ids[idx], nb.dict()), get_connector_name_from_id(ids[idx], nb.dict()), CheckOutputStatus(payload.get('status')), failed_result) - idx += 1 + idx += 1 if filter == 'k8s': for check in hardcoded_checks: result_table.append([check, "Coming soon", "Coming soon", "Coming soon"]) @@ -386,10 +394,50 @@ def run_checks(args: list): :rtype: None """ + parser = ArgumentParser(description='-rc | --run-checks') + parser.add_argument('-rc', '--run-checks', + required=True, + default=True, + help=SUPPRESS, + action="store_true") + parser.add_argument('--all', + help="Run All checks available in the System", + action="store_true") + parser.add_argument('--check', + type=str, + dest='function_name', + help="Run an individual check") + parser.add_argument('--failed', + help="Run Failed checks again", + action="store_true") + parser.add_argument('--type', + type=str, + nargs='*', + help='Run all checks in the given connectors') + parser.add_argument('--report', + action="store_true", + required=False, + help='Report check runs') + args = parser.parse_args() + + if len(sys.argv) == 2: + parser.print_help() + sys.exit(0) + + filter = '' + if args.all is True: + filter = '--all' + elif args.failed is True: + filter = '--failed' + elif args.type not in ([], None): + filter = '--type' + largs = args.type + elif args.function_name not in ('', None): + filter = '--check' + if not args: raise Exception("Run Checks needs filter to be specified.") - filter = args[0] runbooks = [] check_list = [] @@ -403,20 +451,23 @@ def run_checks(args: list): if tc.get('uuid') == k: check_list.append(tc) elif filter in ('-c', '--check'): - # If it is a `-f` option, lets get the second part of the args which + # If it is a `-c` option, lets get the second part of the args which # Will be the name of the check that need to be run and lets run it - check_name = args[1] + check_name = args.check if not check_name: - raise Exception("Option -f should have a check name specified") + raise Exception("Option --check should have a check name specified") # Lets call the db utils method to get the check by name check_list = get_check_by_name(check_name) + if len(check_list) == 0: + print(f"ERROR: Invalid Function name {check_name}") + parser.print_help() + print("Note: You can use TAB to autocomplete options available for the -rc --check") + return elif filter == '--all': check_list = get_checks_by_connector("all", True) elif filter == '--type': check_list = [] - all_connectors = args[1:] - if args[-1] in ('-r', '--report'): - all_connectors = args[1:-1] + all_connectors = largs # Handle case when more than one connector was given with # comma separation but no space in between them. @@ -425,15 +476,17 @@ def run_checks(args: list): all_connectors = all_connectors[0].split(',') for connector in all_connectors: + connector = connector.replace(',', '') temp_list = get_checks_by_connector(connector.strip(), True) for t in temp_list: if t not in check_list: check_list.append(t) else: - print(f"ERROR: WRONG OPTION: {args}") - + parser.print_help() + print("Note: You can use TAB to autocomplete options available for the -rc ") + return - if args[-1] in ('-r', '--report'): + if args.report: UNSKRIPT_GLOBALS['report'] = True if len(check_list) > 0: @@ -666,7 +719,9 @@ def create_jit_runbook(check_list: list): if os.path.exists(os.environ.get('EXECUTION_DIR') + '/workspace') is False: os.makedirs(os.environ.get('EXECUTION_DIR') + '/workspace') - failed_notebook = os.environ.get('EXECUTION_DIR', '/unskript/data/execution').strip('"') + '/workspace/' + str(uuid.uuid4()) + '.ipynb' + exec_id = str(uuid.uuid4()) + UNSKRIPT_GLOBALS['exec_id'] = exec_id + failed_notebook = os.environ.get('EXECUTION_DIR', '/unskript/data/execution').strip('"') + '/workspace/' + exec_id + '.ipynb' for check in check_list: s_connector = check.get('metadata').get('action_type') s_connector = s_connector.replace('LEGO', 'CONNECTOR') @@ -809,7 +864,10 @@ def update_audit_trail(status_dict_list: list): finally: k = str(datetime.now()) p = f = e = 0 - id = uuid.uuid4() + if UNSKRIPT_GLOBALS.get('exec_id') is None: + id = uuid.uuid4() + else: + id = UNSKRIPT_GLOBALS.get('exec_id') trail_data = {} trail_data[id] = {} trail_data[id]['time_stamp'] = k @@ -845,18 +903,36 @@ def list_checks_by_connector(args): :rtype: None """ - if not args: - print(f"ERROR: Either --all or --type is needed") - return - if args[0] == '--all': + parser = ArgumentParser(description='-lc | --list-checks') + parser.add_argument('-lc', + '--list-checks', + help=SUPPRESS, + required=True, + default=True, + action="store_true") + parser.add_argument('--all', + help="List all Checks across connectors in the system", + action="store_true") + parser.add_argument('--type', + type=str, + help="Type of connector for which the checks should be shown") + + args = parser.parse_args() + + if len(sys.argv) <= 2: + parser.print_help() + sys.exit(0) + + + if args.all is True: connector_name = 'all' - elif args[0] == '--type': - connector_name = args[-1] + elif args.type not in ('', None): + connector_name = args.type else: connector_name = 'all' list_connector_table = [ - [TBL_HDR_LIST_CHKS_CONNECTOR, TBL_HDR_CHKS_NAME, TBL_HDR_CHKS_UUID]] + [TBL_HDR_LIST_CHKS_CONNECTOR, TBL_HDR_CHKS_NAME, TBL_HDR_CHKS_FN]] for l in get_checks_by_connector(connector_name): list_connector_table.append(l) @@ -871,29 +947,51 @@ def display_failed_checks(args): :rtype: None """ + parser = ArgumentParser(description='-df | --display-failed-checks') + parser.add_argument('-df', + '--display-failed-checks', + help=SUPPRESS, + required=True, + default=True, + action="store_true") + parser.add_argument('--all', + help='Run all failed checks again', + action="store_true") + parser.add_argument('--type', + dest='connector_type', + help="Connector Type to display the failed check against", + type=str) + args = parser.parse_args() + + if len(sys.argv) == 2: + parser.print_help() + sys.exit(0) + if not args: print("ERROR: Either -all or --type connector needed") return connector = "all" - if args[0] == '--all': - connector = "all" - elif args[0] == '--type': - connector = args[1] + if args.all is True: + connector = 'all' + elif args.connector_type not in ('', None): + connector = args.connector_type + else: + connector = 'all' pss_content = get_pss_record('current_execution_status') exec_status = pss_content.get('exec_status') failed_exec_list = [] if exec_status is not None: for k,v in exec_status.items(): - if CheckOutputStatus(v.get('current_status')) == CheckOutputStatus.FAILED: + if v.get('current_status') == 2: d = {} d['check_name'] = v.get('check_name') d['timestamp'] = v.get('failed_timestamp') d['execution_id'] = k if connector == 'all': failed_exec_list.append(d) - elif connector not in ('', None) and v.get('connector_type') == connector.lower(): + elif connector and v.get('connector_type').lower() == connector.lower(): # Append only when connector type matches failed_exec_list.append(d) else: @@ -912,7 +1010,33 @@ def display_failed_checks(args): print("") -def display_failed_logs(exec_id: str = None): +def display_failed_logs(args): + + parser = ArgumentParser(description='-dl | --display-failed-logs') + + parser.add_argument('-dl', + '--display-failed-logs', + help=SUPPRESS, + required=True, + default=True, + action="store_true") + parser.add_argument('--execution_id', + help="Execution ID for which the Logs should be fetched", + type=str) + + args = parser.parse_args() + + if len(sys.argv) == 2: + parser.print_help() + sys.exit(0) + + if args.execution_id not in ('', None): + exec_id = args.execution_id + + if not args: + return + + # exec_id = args[-1] output = os.environ.get('EXECUTION_DIR', '/unskript/data/execution').strip( '"') + '/workspace/' + f"{exec_id}_output.ipynb" if not os.path.exists(output): @@ -937,15 +1061,36 @@ def show_audit_trail(args): """show_audit_trail This function reads the failed logs for a given execution ID When a check fails, the failed logs are saved in os.environ['EXECUTION_DIR']/failed/.log :type filter: string - :param filter: filter used to query audit_trail to get logs used to serach logs + :param filter: filter used to query audit_trail to get logs used to search logs """ - if not args: - print(f"ERROR: Audit Trial needs --all or --type ") - return - if args[0].replace('-','') == 'all': + parser = ArgumentParser(description='-sa | --show-audit-trail') + parser.add_argument('-sa', + '--show-audit-trail', + help=SUPPRESS, + required=True, + default=True, + action='store_true') + parser.add_argument('--all', + help="List Logs of All check across all connectors", + action="store_true") + parser.add_argument('--type', + dest='connector_type', + help='Show Audit trail for checks for the given connector', + type=str) + parser.add_argument('--execution_id', + type=str, + help='Execution ID for which the audit trail should be shown') + + args = parser.parse_args() + # if not args: + # print(f"ERROR: Audit Trial needs --all or --type ") + # return + if args.all is True: filter = 'all' - elif args[0].replace('-','') == 'type': - filter = args[-1] + elif args.connector_type not in ('', None): + filter = args.connector_type + elif args.execution_id not in ('', None): + filter = args.execution_id else: filter = 'all' @@ -1109,16 +1254,16 @@ def get_code_cell_action_uuids(content: dict) -> list: return retval -def get_last_code_cell_output(content: dict) -> dict: - """get_last_code_cell_output This function takes in the notenode dictionary +def get_last_code_cell_output(content: dict) -> list: + """get_last_code_cell_output This function takes in the notebook node dictionary finds out the last cell output and returns in the form of a dict :type content: dict - :param content: Notenode as Dictionary + :param content: NotebookNode as Dictionary - :rtype: Last output in the form of Python dictionary + :rtype: Last output in the form of Python list """ - retval = {} + retval = [] if content in ('', None): print("Content sent is empty") return retval @@ -1126,9 +1271,7 @@ def get_last_code_cell_output(content: dict) -> dict: for cell in content.get('cells'): if cell.get('cell_type') == 'code': if cell.get('id') == 'lastcell': - outputs = {} - outputs = cell.get('outputs') - retval = outputs + retval = cell.get('outputs') print("") return retval @@ -1292,6 +1435,9 @@ def get_runbook_metadata_contents(_runbook) -> dict: else: if os.path.exists(os.environ.get('PWD') + '/' + _runbook): file_name_to_read = os.environ.get('PWD') + '/' + _runbook + + if os.path.exists(_runbook) is True: + file_name_to_read = _runbook if not file_name_to_read: raise Exception(f"Runbook Not found {_runbook}") @@ -1518,22 +1664,39 @@ def parse_creds(args): """ # Check if creds that need to be created is for k8s, if yes # read the given kubecofnig and create the credential of it. - if args[0] != '--type': + parser = ArgumentParser(description='-cc | --create-credentials') + parser.add_argument('-cc', + '--create-credentials', + help=SUPPRESS, + required=True, + default=True, + action='store_true') + parser.add_argument('--type', + dest='connector_type', + type=str, + nargs=REMAINDER, + help='Type of connector to be created') + + args = parser.parse_args() + + if args.connector_type in ('', None): display_creds_ui() return - connector_type = args[1] + largs = args.connector_type + connector_type = largs[0] connector_type = connector_type.replace('-','') + if connector_type.lower() in ("k8s", "kubernetes"): - if len(args) == 1: + if len(largs) == 1: print("ERROR: Need a path for kubeconfig file as value for the k8s credential") parser.print_help() sys.exit(1) - if os.path.exists(args[-1]) is False: + if os.path.exists(largs[-1]) is False: print(f"ERROR: Credential File {args[1]} does not exist, please check path") parser.print_help() sys.exit(1) - with open(args[-1], 'r', encoding='utf-8') as f: + with open(largs[-1], 'r', encoding='utf-8') as f: creds_data = f.read() k8s_creds_file = os.environ.get('HOME') + CREDENTIAL_DIR + '/k8screds.json' with open(k8s_creds_file, 'r', encoding='utf-8') as f: @@ -1568,29 +1731,35 @@ def list_creds(): we display on the UI. ACTIVE means the credential data has been filled and ready to go INACTIVE means the credential is not yet ready to be used. """ + # Lets get the creds data from PSS instead of reading from the credentials + # json files. + creds_pss_data = get_pss_record('default_credential_id') + creds_data = [["#", "Connector Type", "Connector Name", "Status"]] creds_dir = os.environ.get('HOME') + CREDENTIAL_DIR creds_files = glob.glob(creds_dir + '/*.json',recursive=True) - creds_data = [["#", "Connector Type", "Connector Name", "Status"]] - index = 0 - for cf in creds_files: - with open(cf, 'r', encoding='utf-8') as f: - content = json.loads(f.read()) - if content.get('type'): - c_type = content.get('type').replace('CONNECTOR_TYPE_', '') - else: - c_type = "UNDEFINED" - - if content.get('metadata') and content.get('metadata').get('connectorData') == "{}": - status = "Incomplete" - else: - status = "Active" + if creds_pss_data: + index = 0 + list_of_creds_active = [] + for data in creds_pss_data.items(): + c_type, c_data = data + c_name = c_data.get('name') + list_of_creds_active.append(c_name) + creds_data.append([index, c_type.split('_')[-1], c_name, "Active"]) + index += 1 + for c_file in creds_files: + c_name = os.path.basename(c_file).replace('.json', '') + if c_name not in list_of_creds_active: + with open(c_file, 'r', encoding='utf-8') as f: + content = json.loads(f.read()) + if content.get('type'): + c_type = content.get('type').replace('CONNECTOR_TYPE_', '') + else: + c_type = "UNDEFINED" + creds_data.append([index, c_type, c_name, "Incomplete"]) + index += 1 + else: + print("ERROR: No Credential Data saaved in PSS DB. Nothing to display") - if content.get('display_name'): - name = content.get('display_name') - else: - name = "UNDEFINED" - creds_data.append([index, c_type, name, status]) - index += 1 print(tabulate(creds_data, headers='firstrow', tablefmt='fancy_grid')) def start_debug(args): @@ -1685,29 +1854,58 @@ def save_check_names(filename: str): description = description + str(f"\t\t VERSION: {version_number} \n") parser.description = description - parser.add_argument('-lr', '--list-runbooks', - help='List Available Runbooks', action='store_true') - parser.add_argument('-rr', '--run-runbook', type=str, nargs=REMAINDER, + parser.add_argument('-lr', + '--list-runbooks', + help='List Available Runbooks', + action='store_true') + parser.add_argument('-rr', + '--run-runbook', + type=str, + nargs=REMAINDER, help='Run the given runbook FILENAME [-RUNBOOK_PARM1 VALUE1] etc..') - parser.add_argument('-rc', '--run-checks', type=str, nargs=REMAINDER, + parser.add_argument('-rc', + '--run-checks', + type=str, + nargs=REMAINDER, help='Run all available checks [--all | --type | --failed | --check check_name]') - parser.add_argument('-df', '--display-failed-checks', type=str, nargs=REMAINDER, + parser.add_argument('-df', + '--display-failed-checks', + type=str, + nargs=REMAINDER, help='Display Failed Checks [--all | --type ]') - parser.add_argument('-lc', '--list-checks', type=str, nargs=REMAINDER, + parser.add_argument('-lc', + '--list-checks', + type=str, + nargs=REMAINDER, help='List available checks, [--all | --type ]') - parser.add_argument('-sa', '--show-audit-trail', type=str, nargs=REMAINDER, + parser.add_argument('-sa', + '--show-audit-trail', + type=str, + nargs=REMAINDER, help='Show audit trail [--all | --type | --execution_id ]') - parser.add_argument('-dl', '--display-failed-logs', - type=str, help='Display failed logs [execution_id]') - parser.add_argument('-cc', '--create-credentials', type=str, nargs=REMAINDER, + parser.add_argument('-dl', + '--display-failed-logs', + type=str, + nargs=REMAINDER, + help='Display failed logs [execution_id]') + parser.add_argument('-cc', + '--create-credentials', + type=str, + nargs=REMAINDER, help='Create Credential [-creds-type creds_file_path]') - parser.add_argument('-cl', '--credential-list', - help='Credential List', action='store_true') + parser.add_argument('--credential-list', + help='Credential List', + action='store_true') parser.add_argument('--start-debug', - help='Start Debug Session. Example: [--start-debug --config /tmp/config.ovpn]', type=str, nargs=REMAINDER) + help='Start Debug Session. Example: [--start-debug --config /tmp/config.ovpn]', + type=str, + nargs=REMAINDER) parser.add_argument('--stop-debug', - help='Stop Current Debug Session', action='store_true') - parser.add_argument('--save-check-names', type=str, help=SUPPRESS) + help='Stop Current Debug Session', + action='store_true') + parser.add_argument('--save-check-names', + type=str, + help=SUPPRESS) args = parser.parse_args() diff --git a/unskript-ctl/unskript_global.yaml.template b/unskript-ctl/unskript_global.yaml.template index cb3711147..0ff7aca18 100644 --- a/unskript-ctl/unskript_global.yaml.template +++ b/unskript-ctl/unskript_global.yaml.template @@ -1,7 +1,4 @@ globals: namespace: "awesome-ops" - threshold: "string" region: "us-west-2" - services: ["calendar", "audit"] - - + services: ["calendar", "audit"] \ No newline at end of file