From 9b2d120c52b4e891f1200682e144f1469f3fe00c Mon Sep 17 00:00:00 2001 From: svittoz Date: Thu, 21 Dec 2023 09:05:14 +0000 Subject: [PATCH] pre-commit --- eds_scikit/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eds_scikit/__init__.py b/eds_scikit/__init__.py index faff0b82..357aa5eb 100644 --- a/eds_scikit/__init__.py +++ b/eds_scikit/__init__.py @@ -81,7 +81,7 @@ def set_env_variables() -> None: def improve_performances( to_add_conf: List[Tuple[str, str]] = [], quiet_spark: bool = True, - app_name: str = "" + app_name: str = "", ) -> Tuple[SparkSession, SparkContext, SparkSession.sql]: """ (Re)defines various Spark variable with some configuration changes @@ -125,10 +125,12 @@ def improve_performances( ("spark.sql.execution.arrow.pyspark.enabled", "true"), ] ) - + for key, value in to_add_conf: if not conf.contains(key): - logger.warning(f"{key} not in default spark config. Make sure it corresponds to existing config parameter.") + logger.warning( + f"{key} not in default spark config. Make sure it corresponds to existing config parameter." + ) conf.set(key, value) # Stopping context to add necessary env variables