Option to speed up DanswerBot by turning off chain of thought (#816)

backend/danswer/configs/danswerbot_configs.py

@@ -41,10 +41,14 @@
 )
 # Add a second LLM call post Answer to verify if the Answer is valid
 # Throws out answers that don't directly or fully answer the user query
-# This is the default for all DanswerBot channels unless the bot is configured individually
+# This is the default for all DanswerBot channels unless the channel is configured individually
+# Set/unset by "Hide Non Answers"
 ENABLE_DANSWERBOT_REFLEXION = (
     os.environ.get("ENABLE_DANSWERBOT_REFLEXION", "").lower() == "true"
 )
+DANSWER_BOT_DISABLE_COT = (
+    os.environ.get("DANSWER_BOT_DISABLE_COT", "").lower() == "true"
+)
 # Add the per document feedback blocks that affect the document rankings via boosting
 ENABLE_SLACK_DOC_FEEDBACK = (
     os.environ.get("ENABLE_SLACK_DOC_FEEDBACK", "").lower() == "true"

backend/danswer/danswerbot/slack/handlers/handle_message.py

@@ -7,6 +7,7 @@
 from sqlalchemy.orm import Session
 
 from danswer.configs.danswerbot_configs import DANSWER_BOT_ANSWER_GENERATION_TIMEOUT
+from danswer.configs.danswerbot_configs import DANSWER_BOT_DISABLE_COT
 from danswer.configs.danswerbot_configs import DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER
 from danswer.configs.danswerbot_configs import DANSWER_BOT_DISPLAY_ERROR_MSGS
 from danswer.configs.danswerbot_configs import DANSWER_BOT_NUM_RETRIES
@@ -75,6 +76,7 @@ def handle_message(
     disable_docs_only_answer: bool = DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER,
     disable_auto_detect_filters: bool = DISABLE_DANSWER_BOT_FILTER_DETECT,
     reflexion: bool = ENABLE_DANSWERBOT_REFLEXION,
+    disable_cot: bool = DANSWER_BOT_DISABLE_COT,
 ) -> bool:
     """Potentially respond to the user message depending on filters and if an answer was generated
 
@@ -219,7 +221,7 @@ def _get_answer(new_message_request: NewMessageRequest) -> QAResponse:
                 query=msg,
                 filters=filters,
                 enable_auto_detect_filters=not disable_auto_detect_filters,
-                real_time=False,
+                real_time=disable_cot,
             )
         )
     except Exception as e:

deployment/docker_compose/docker-compose.dev.yml

@@ -61,7 +61,7 @@ services:
       # Set to debug to get more fine-grained logs
       - LOG_LEVEL=${LOG_LEVEL:-info}
       # Log all of the prompts to the LLM
-      - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-info}
+      - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-}
       # If set to `true` will enable additional logs about Vespa query performance
       # (time spent on finding the right docs + time spent fetching summaries from disk)
       - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-}
@@ -107,6 +107,7 @@ services:
       - DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER=${DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER:-}
       - DANSWER_BOT_DISPLAY_ERROR_MSGS=${DANSWER_BOT_DISPLAY_ERROR_MSGS:-}
       - DANSWER_BOT_RESPOND_EVERY_CHANNEL=${DANSWER_BOT_RESPOND_EVERY_CHANNEL:-}
+      - DANSWER_BOT_DISABLE_COT=${DANSWER_BOT_DISABLE_COT:-}
       - NOTIFY_SLACKBOT_NO_ANSWER=${NOTIFY_SLACKBOT_NO_ANSWER:-}
       # Recency Bias for search results, decay at 1 / (1 + DOC_TIME_DECAY * x years)
       - DOC_TIME_DECAY=${DOC_TIME_DECAY:-}
@@ -134,7 +135,7 @@ services:
       # Set to debug to get more fine-grained logs
       - LOG_LEVEL=${LOG_LEVEL:-info}
       # Log all of the prompts to the LLM
-      - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-info}
+      - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-}
     volumes:
       - local_dynamic_storage:/home/storage
       - file_connector_tmp_storage:/home/file_connector_storage