From 9e9a5789215fe85381450396941468e03e6973ca Mon Sep 17 00:00:00 2001 From: Yuhong Sun Date: Tue, 5 Dec 2023 00:43:45 -0800 Subject: [PATCH] Option to speed up DanswerBot by turning off chain of thought (#816) --- backend/danswer/configs/danswerbot_configs.py | 6 +++++- backend/danswer/danswerbot/slack/handlers/handle_message.py | 4 +++- deployment/docker_compose/docker-compose.dev.yml | 5 +++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/backend/danswer/configs/danswerbot_configs.py b/backend/danswer/configs/danswerbot_configs.py index 4b7a1ec752b..86abcd44687 100644 --- a/backend/danswer/configs/danswerbot_configs.py +++ b/backend/danswer/configs/danswerbot_configs.py @@ -41,10 +41,14 @@ ) # Add a second LLM call post Answer to verify if the Answer is valid # Throws out answers that don't directly or fully answer the user query -# This is the default for all DanswerBot channels unless the bot is configured individually +# This is the default for all DanswerBot channels unless the channel is configured individually +# Set/unset by "Hide Non Answers" ENABLE_DANSWERBOT_REFLEXION = ( os.environ.get("ENABLE_DANSWERBOT_REFLEXION", "").lower() == "true" ) +DANSWER_BOT_DISABLE_COT = ( + os.environ.get("DANSWER_BOT_DISABLE_COT", "").lower() == "true" +) # Add the per document feedback blocks that affect the document rankings via boosting ENABLE_SLACK_DOC_FEEDBACK = ( os.environ.get("ENABLE_SLACK_DOC_FEEDBACK", "").lower() == "true" diff --git a/backend/danswer/danswerbot/slack/handlers/handle_message.py b/backend/danswer/danswerbot/slack/handlers/handle_message.py index 01d8122bd04..b611b78b4a5 100644 --- a/backend/danswer/danswerbot/slack/handlers/handle_message.py +++ b/backend/danswer/danswerbot/slack/handlers/handle_message.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import Session from danswer.configs.danswerbot_configs import DANSWER_BOT_ANSWER_GENERATION_TIMEOUT +from danswer.configs.danswerbot_configs import DANSWER_BOT_DISABLE_COT from danswer.configs.danswerbot_configs import DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER from danswer.configs.danswerbot_configs import DANSWER_BOT_DISPLAY_ERROR_MSGS from danswer.configs.danswerbot_configs import DANSWER_BOT_NUM_RETRIES @@ -75,6 +76,7 @@ def handle_message( disable_docs_only_answer: bool = DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER, disable_auto_detect_filters: bool = DISABLE_DANSWER_BOT_FILTER_DETECT, reflexion: bool = ENABLE_DANSWERBOT_REFLEXION, + disable_cot: bool = DANSWER_BOT_DISABLE_COT, ) -> bool: """Potentially respond to the user message depending on filters and if an answer was generated @@ -219,7 +221,7 @@ def _get_answer(new_message_request: NewMessageRequest) -> QAResponse: query=msg, filters=filters, enable_auto_detect_filters=not disable_auto_detect_filters, - real_time=False, + real_time=disable_cot, ) ) except Exception as e: diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml index 8c59ee028fa..9140602c62b 100644 --- a/deployment/docker_compose/docker-compose.dev.yml +++ b/deployment/docker_compose/docker-compose.dev.yml @@ -61,7 +61,7 @@ services: # Set to debug to get more fine-grained logs - LOG_LEVEL=${LOG_LEVEL:-info} # Log all of the prompts to the LLM - - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-info} + - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # If set to `true` will enable additional logs about Vespa query performance # (time spent on finding the right docs + time spent fetching summaries from disk) - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-} @@ -107,6 +107,7 @@ services: - DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER=${DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER:-} - DANSWER_BOT_DISPLAY_ERROR_MSGS=${DANSWER_BOT_DISPLAY_ERROR_MSGS:-} - DANSWER_BOT_RESPOND_EVERY_CHANNEL=${DANSWER_BOT_RESPOND_EVERY_CHANNEL:-} + - DANSWER_BOT_DISABLE_COT=${DANSWER_BOT_DISABLE_COT:-} - NOTIFY_SLACKBOT_NO_ANSWER=${NOTIFY_SLACKBOT_NO_ANSWER:-} # Recency Bias for search results, decay at 1 / (1 + DOC_TIME_DECAY * x years) - DOC_TIME_DECAY=${DOC_TIME_DECAY:-} @@ -134,7 +135,7 @@ services: # Set to debug to get more fine-grained logs - LOG_LEVEL=${LOG_LEVEL:-info} # Log all of the prompts to the LLM - - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-info} + - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} volumes: - local_dynamic_storage:/home/storage - file_connector_tmp_storage:/home/file_connector_storage