Skip to content

Commit

Permalink
Add more logging on existing jobs
Browse files Browse the repository at this point in the history
  • Loading branch information
Weves committed Nov 30, 2023
1 parent 7ea774f commit e983aae
Showing 1 changed file with 13 additions and 5 deletions.
18 changes: 13 additions & 5 deletions backend/danswer/background/update.py
Original file line number Diff line number Diff line change
Expand Up @@ -313,10 +313,14 @@ def update_loop(delay: int = 10, num_workers: int = NUM_INDEXING_WORKERS) -> Non
start = time.time()
start_time_utc = datetime.utcfromtimestamp(start).strftime("%Y-%m-%d %H:%M:%S")
logger.info(f"Running update, current UTC time: {start_time_utc}")
logger.debug(
"Found existing indexing jobs: "
f"{[(attempt_id, job.status) for attempt_id, job in existing_jobs.items()]}"
)

if existing_jobs:
# TODO: make this debug level once the "no jobs are being scheduled" issue is resolved
logger.info(
"Found existing indexing jobs: "
f"{[(attempt_id, job.status) for attempt_id, job in existing_jobs.items()]}"
)

try:
existing_jobs = cleanup_indexing_jobs(existing_jobs=existing_jobs)
create_indexing_jobs(existing_jobs=existing_jobs)
Expand All @@ -330,7 +334,7 @@ def update_loop(delay: int = 10, num_workers: int = NUM_INDEXING_WORKERS) -> Non
time.sleep(sleep_time)


if __name__ == "__main__":
def update__main() -> None:
# needed for CUDA to work with multiprocessing
# NOTE: needs to be done on application startup
# before any other torch code has been run
Expand All @@ -342,3 +346,7 @@ def update_loop(delay: int = 10, num_workers: int = NUM_INDEXING_WORKERS) -> Non
warm_up_models(indexer_only=True, skip_cross_encoders=True)
logger.info("Starting Indexing Loop")
update_loop()


if __name__ == "__main__":
update__main()

1 comment on commit e983aae

@vercel
Copy link

@vercel vercel bot commented on e983aae Nov 30, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.