diff --git a/.changelog/4688.yml b/.changelog/4688.yml new file mode 100644 index 00000000000..3eeec6f51fe --- /dev/null +++ b/.changelog/4688.yml @@ -0,0 +1,4 @@ +changes: +- description: Determining logger setup via environment variable. Writing pre-commit results to files. + type: feature +pr_number: 4688 diff --git a/demisto_sdk/__init__.py b/demisto_sdk/__init__.py index 5bf2275f6b7..81d1498c526 100644 --- a/demisto_sdk/__init__.py +++ b/demisto_sdk/__init__.py @@ -1,4 +1,10 @@ -if __name__ in ["__main__", "demisto_sdk"]: +import os + +if os.environ.get("DEMISTO_SDK_SKIP_LOGGER_SETUP", "False").lower() not in [ + "true", + "yes", + "1", +]: from demisto_sdk.commands.common.logger import logging_setup logging_setup(initial=True, calling_function="__init__") diff --git a/demisto_sdk/commands/common/tools.py b/demisto_sdk/commands/common/tools.py index 40ee9eea5cd..5032cce8f04 100644 --- a/demisto_sdk/commands/common/tools.py +++ b/demisto_sdk/commands/common/tools.py @@ -913,11 +913,13 @@ def get_file( def get_file_or_remote(file_path: Path, clear_cache=False): content_path = get_content_path() + logger.debug(f'get_file_or_remote {content_path=}') relative_file_path = None if file_path.is_absolute(): absolute_file_path = file_path try: relative_file_path = file_path.relative_to(content_path) + logger.debug(f'get_file_or_remote {relative_file_path=}') except ValueError: logger.debug( f"{file_path} is not a subpath of {content_path}. If the file does not exists locally, it could not be fetched." @@ -925,6 +927,7 @@ def get_file_or_remote(file_path: Path, clear_cache=False): else: absolute_file_path = content_path / file_path relative_file_path = file_path + logger.debug(f'get_file_or_remote {relative_file_path=}') try: return get_file(absolute_file_path, clear_cache=clear_cache) except FileNotFoundError: diff --git a/demisto_sdk/commands/pre_commit/pre_commit_command.py b/demisto_sdk/commands/pre_commit/pre_commit_command.py index 09c11066cc7..f5578dd5427 100644 --- a/demisto_sdk/commands/pre_commit/pre_commit_command.py +++ b/demisto_sdk/commands/pre_commit/pre_commit_command.py @@ -1,3 +1,4 @@ +import json import multiprocessing import os import re @@ -118,6 +119,7 @@ def run_hook( precommit_env: dict, verbose: bool = False, stdout: Optional[int] = subprocess.PIPE, + json_output_path: Optional[Path] = None ) -> int: """This function runs the pre-commit process and waits until finished. We run this function in multithread. @@ -127,17 +129,24 @@ def run_hook( precommit_env (dict): The pre-commit environment variables verbose (bool, optional): Whether print verbose output. Defaults to False. stdout (Optional[int], optional): The way to handle stdout. Defaults to subprocess.PIPE. - + json_output_path (Optional[Path]): Optional path to a JSON formatted output file/dir where pre-commit hooks + results are stored. None by deafult, and file is not created. Returns: int: return code - 0 if hook passed, 1 if failed """ logger.debug(f"Running hook {hook_id}") + + # Otherwise it's None and file won't be created or already a file's path. + if json_output_path and json_output_path.is_dir(): + json_output_path = json_output_path / f'{hook_id}.json' + process = PreCommitRunner._run_pre_commit_process( PRECOMMIT_CONFIG_MAIN_PATH, precommit_env, verbose, stdout, command=["run", "-a", hook_id], + json_output_path=json_output_path, ) if process.stdout: @@ -153,6 +162,7 @@ def _run_pre_commit_process( verbose: bool, stdout=None, command: Optional[List[str]] = None, + json_output_path: Optional[Path] = None ) -> subprocess.CompletedProcess: """Runs a process of pre-commit @@ -162,13 +172,15 @@ def _run_pre_commit_process( verbose (bool): whether to print verbose output stdout (optional): use `subprocess.PIPE` to capture stdout. Use None to print it. Defaults to None. command (Optional[List[str]], optional): The pre-commit command to run. Defaults to None. - + json_output_path (Optional[Path]): Optional path to a JSON formatted output file where pre-commit hooks + results are stored. None by deafult, and file is not created. Returns: _type_: _description_ """ if command is None: command = ["run", "-a"] - return subprocess.run( + output = subprocess.PIPE if json_output_path else stdout + completed_process = subprocess.run( list( filter( None, @@ -185,17 +197,24 @@ def _run_pre_commit_process( ), env=precommit_env, cwd=CONTENT_PATH, - stdout=stdout, - stderr=stdout, + stdout=output, + stderr=output, universal_newlines=True, ) + if json_output_path: + with open(json_output_path, "w") as json_file: + json.dump(completed_process.__dict__, json_file, indent=4) + + return completed_process + @staticmethod def run( pre_commit_context: PreCommitContext, precommit_env: dict, verbose: bool, show_diff_on_failure: bool, + json_output_path: Optional[Path] = None ) -> int: """Execute the pre-commit hooks on the files. @@ -204,7 +223,8 @@ def run( precommit_env (dict): The environment variables dict. verbose (bool): Whether run pre-commit in verbose mode. show_diff_on_failure (bool): Whether to show diff when a hook fail or not. - + json_output_path (Optional[Path]): Optional path to a JSON formatted output file/dir where pre-commit hooks + results are stored. None by deafult, and file is not created. Returns: int: The exit code - 0 if everything is valid. """ @@ -224,6 +244,8 @@ def run( precommit_env, verbose, command=["install-hooks"], + json_output_path=json_output_path if not json_output_path.is_dir() + else json_output_path / 'install-hooks.json', ) num_processes = cpu_count() @@ -236,6 +258,12 @@ def run( logger.debug(f"Running hook {original_hook_id} with {generated_hooks}") hook_ids = generated_hooks.hook_ids if generated_hooks.parallel and len(hook_ids) > 1: + # We shall not write results to the same file if running hooks in parallel, therefore, + # writing the results to a parallel directory. + if json_output_path and not json_output_path.is_dir(): + json_output_path = json_output_path.parent / json_output_path.stem + json_output_path.mkdir(exist_ok=True) + with ThreadPool(num_processes) as pool: current_hooks_exit_codes = pool.map( partial( @@ -243,6 +271,7 @@ def run( precommit_env=precommit_env, verbose=verbose, stdout=subprocess.PIPE, + json_output_path=json_output_path, ), hook_ids, ) @@ -252,7 +281,7 @@ def run( hook_id, precommit_env=precommit_env, verbose=verbose, - stdout=None, + json_output_path=json_output_path, ) for hook_id in hook_ids ] @@ -284,6 +313,7 @@ def prepare_and_run( show_diff_on_failure: bool = False, exclude_files: Optional[Set[Path]] = None, dry_run: bool = False, + json_output_path: Optional[Path] = None ) -> int: """Trigger the relevant hooks. @@ -293,7 +323,8 @@ def prepare_and_run( show_diff_on_failure (bool, optional): Whether to show diff when a hook fail or not. Defaults to False. exclude_files (Optional[Set[Path]], optional): Files to exclude when running. Defaults to None. dry_run (bool, optional): Whether to run the pre-commit hooks in dry-run mode. Defaults to False. - + json_output_path (Path, optional): Optional path to a JSON formatted output file/dir where pre-commit hooks + results are stored. None by default, and file is not created. Returns: int: The exit code, 0 if nothing failed. """ @@ -345,8 +376,9 @@ def prepare_and_run( f"Dry run, skipping pre-commit.\nConfig file saved to {PRECOMMIT_CONFIG_MAIN_PATH}" ) return ret_val + ret_val = PreCommitRunner.run( - pre_commit_context, precommit_env, verbose, show_diff_on_failure + pre_commit_context, precommit_env, verbose, show_diff_on_failure, json_output_path ) return ret_val @@ -399,13 +431,13 @@ def group_by_language( for integration_script_paths in more_itertools.chunked_even( integrations_scripts_mapping.keys(), INTEGRATIONS_BATCH ): - with multiprocessing.Pool(processes=cpu_count()) as pool: - content_items = pool.map(BaseContent.from_path, integration_script_paths) - for content_item in content_items: - if not content_item or not isinstance(content_item, IntegrationScript): - continue - # content-item is a script/integration - integrations_scripts.add(content_item) + # Process each path sequentially + content_items = [BaseContent.from_path(path) for path in integration_script_paths] + for content_item in content_items: + if not content_item or not isinstance(content_item, IntegrationScript): + continue + # content-item is a script/integration + integrations_scripts.add(content_item) logger.debug("Pre-Commit: Finished parsing all integrations and scripts") exclude_integration_script = set() for integration_script in integrations_scripts: @@ -413,7 +445,6 @@ def group_by_language( # add api modules to the api_modules list, we will handle them later api_modules.append(integration_script) continue - if api_modules: logger.debug("Pre-Commit: Starting to handle API Modules") with ContentGraphInterface() as graph: @@ -509,6 +540,7 @@ def pre_commit_manager( docker_image: Optional[str] = None, run_hook: Optional[str] = None, pre_commit_template_path: Optional[Path] = None, + json_output_path: Optional[Path] = None ) -> int: """Run pre-commit hooks . @@ -523,12 +555,15 @@ def pre_commit_manager( force_run_hooks (Optional[List[str]], optional): List for hooks to force run. Defaults to None. verbose (bool, optional): Whether run pre-commit in verbose mode. Defaults to False. show_diff_on_failure (bool, optional): Whether show git diff after pre-commit failure. Defaults to False. - dry_run (bool, optional): Whether to run the pre-commit hooks in dry-run mode, which will only create the config file. + dry_run (bool, optional): Whether to run the pre-commit hooks in dry-run mode, which will only create the + config file. run_docker_hooks (bool, optional): Whether to run docker based hooks or not. image_ref: (str, optional): Override the image from YAML / native config file with this image reference. - docker_image: (str, optional): Override the `docker_image` property in the template file. This is a comma separated list of: `from-yml`, `native:dev`, `native:ga`, `native:candidate`. + docker_image: (str, optional): Override the `docker_image` property in the template file. This is a comma + separated list of: `from-yml`, `native:dev`, `native:ga`, `native:candidate`. pre_commit_template_path (Path, optional): Path to the template pre-commit file. - + json_output_path (Path, optional): Optional path to a JSON formatted output file/dir where pre-commit hooks results + are stored. None by default, and file is not created. Returns: int: Return code of pre-commit. """ @@ -597,6 +632,7 @@ def pre_commit_manager( show_diff_on_failure, exclude_files, dry_run, + json_output_path, ) @@ -694,4 +730,4 @@ def preprocess_files( for file in files_to_run } # filter out files that are not in the content git repo (e.g in .gitignore) - return relative_paths & all_git_files + return relative_paths & all_git_files \ No newline at end of file diff --git a/demisto_sdk/commands/split/ymlsplitter.py b/demisto_sdk/commands/split/ymlsplitter.py index 74aa00640d5..f33d4fe5832 100644 --- a/demisto_sdk/commands/split/ymlsplitter.py +++ b/demisto_sdk/commands/split/ymlsplitter.py @@ -129,7 +129,7 @@ def extract_to_package_format( yaml_out = f"{output_path}/{base_name}.yml" logger.debug(f"Creating yml file: {yaml_out} ...") if self.yml_data: - yaml_obj = self.yml_data + yaml_obj = self.yml_data.copy() else: yaml_obj = get_file(self.input, raise_on_error=True)