From 1e311e88461a7bb3767281f38c1230ea9e5cc049 Mon Sep 17 00:00:00 2001
From: Lu Wang <lu.wang@databricks.com>
Date: Thu, 30 May 2024 01:27:47 -0700
Subject: [PATCH] add spark session

---
 joblibspark/backend.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/joblibspark/backend.py b/joblibspark/backend.py
index 87dbfb8..1841d91 100644
--- a/joblibspark/backend.py
+++ b/joblibspark/backend.py
@@ -40,6 +40,7 @@
 from pyspark import cloudpickle
 from pyspark.util import VersionUtils
 
+from .utils import get_spark_session
 
 def register():
     """
@@ -72,10 +73,7 @@ def __init__(self, **backend_args):
         super(SparkDistributedBackend, self).__init__(**backend_args)
         self._pool = None
         self._n_jobs = None
-        self._spark = SparkSession \
-            .builder \
-            .appName("JoblibSparkBackend") \
-            .getOrCreate()
+        self._spark = get_spark_session()
         self._spark_context = self._spark.sparkContext
         self._job_group = "joblib-spark-job-group-" + str(uuid.uuid4())
         self._spark_pinned_threads_enabled = isinstance(