autoscale with pool=thread crashes celery. remove and use concurrency… (

#1929) * autoscale with pool=thread crashes celery. remove and use concurrency instead (to be improved later) * update dev background script as well
onyx-dot-app · Jul 25, 2024 · 546bfbd · 546bfbd
1 parent 27824d6
commit 546bfbd
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 3 deletions.
diff --git a/backend/scripts/dev_run_background_jobs.py b/backend/scripts/dev_run_background_jobs.py
@@ -24,9 +24,8 @@ def run_jobs(exclude_indexing: bool) -> None:
         "ee.danswer.background.celery.celery_app",
         "worker",
         "--pool=threads",
-        "--autoscale=3,10",
+        "--concurrency=16",
         "--loglevel=INFO",
-        "--concurrency=1",
     ]
 
     cmd_beat = [

diff --git a/backend/supervisord.conf b/backend/supervisord.conf
@@ -25,7 +25,7 @@ autorestart=true
 # relatively compute-light (e.g. they tend to just make a bunch of requests to 
 # Vespa / Postgres)
 [program:celery_worker]
-command=celery -A danswer.background.celery.celery_run:celery_app worker --pool=threads --autoscale=3,10 --loglevel=INFO --logfile=/var/log/celery_worker.log
+command=celery -A danswer.background.celery.celery_run:celery_app worker --pool=threads --concurrency=16 --loglevel=INFO --logfile=/var/log/celery_worker.log
 stdout_logfile=/var/log/celery_worker_supervisor.log
 stdout_logfile_maxbytes=52428800
 redirect_stderr=true