diff --git a/dlt/common/runners/stdout.py b/dlt/common/runners/stdout.py index bb5251764c..adf1fdc0d3 100644 --- a/dlt/common/runners/stdout.py +++ b/dlt/common/runners/stdout.py @@ -38,7 +38,13 @@ def iter_std( Use -u in scripts_args for unbuffered python execution """ with venv.start_command( - command, *script_args, stdout=PIPE, stderr=PIPE, bufsize=1, text=True + command, + *script_args, + stdout=PIPE, + stderr=PIPE, + bufsize=1, + text=True, + errors="backslashreplace", ) as process: exit_code: int = None q_: queue.Queue[Tuple[OutputStdStreamNo, str]] = queue.Queue() @@ -72,7 +78,13 @@ def _r_q(std_: OutputStdStreamNo) -> None: def iter_stdout(venv: Venv, command: str, *script_args: Any) -> Iterator[str]: # start a process in virtual environment, assume that text comes from stdout with venv.start_command( - command, *script_args, stdout=PIPE, stderr=PIPE, bufsize=1, text=True + command, + *script_args, + stdout=PIPE, + stderr=PIPE, + bufsize=1, + text=True, + errors="backslashreplace", ) as process: exit_code: int = None line = "" diff --git a/dlt/common/runners/venv.py b/dlt/common/runners/venv.py index ad6448dd2c..eb12970e97 100644 --- a/dlt/common/runners/venv.py +++ b/dlt/common/runners/venv.py @@ -98,14 +98,18 @@ def run_command(self, entry_point: str, *script_args: Any) -> str: # runs one of installed entry points typically CLIs coming with packages and installed into PATH command = os.path.join(self.context.bin_path, entry_point) cmd = [command, *script_args] - return subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True) + return subprocess.check_output( + cmd, stderr=subprocess.STDOUT, text=True, errors="backslashreplace" + ) def run_script(self, script_path: str, *script_args: Any) -> str: """Runs a python `script` source with specified `script_args`. Current `os.environ` and cwd is passed to executed process""" # os.environ is passed to executed process cmd = [self.context.env_exe, os.path.abspath(script_path), *script_args] try: - return subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True) + return subprocess.check_output( + cmd, stderr=subprocess.STDOUT, text=True, errors="backslashreplace" + ) except subprocess.CalledProcessError as cpe: if cpe.returncode == 2: raise FileNotFoundError(script_path) @@ -115,7 +119,9 @@ def run_script(self, script_path: str, *script_args: Any) -> str: def run_module(self, module: str, *module_args: Any) -> str: """Runs a python `module` with specified `module_args`. Current `os.environ` and cwd is passed to executed process""" cmd = [self.context.env_exe, "-m", module, *module_args] - return subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True) + return subprocess.check_output( + cmd, stderr=subprocess.STDOUT, text=True, errors="backslashreplace" + ) def add_dependencies(self, dependencies: List[str] = None) -> None: Venv._install_deps(self.context, dependencies) @@ -134,7 +140,9 @@ def _install_deps(context: types.SimpleNamespace, dependencies: List[str]) -> No cmd = [context.env_exe, "-Im", Venv.PIP_TOOL, "install"] try: - subprocess.check_output(cmd + dependencies, stderr=subprocess.STDOUT) + subprocess.check_output( + cmd + dependencies, stderr=subprocess.STDOUT, errors="backslashreplace" + ) except subprocess.CalledProcessError as exc: raise CannotInstallDependencies(dependencies, context.env_exe, exc.output)