From 3b9d4d9874fe33ae90382e8f2fd8fb727ba96e4f Mon Sep 17 00:00:00 2001 From: Alice Butcher Date: Mon, 8 Jul 2024 15:25:02 +0100 Subject: [PATCH 1/2] fix: remove hacky queue length estimate it fills up the logs and may have completely crashed the site --- .../versioned_datastore/lib/datastore_utils.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/ckanext/versioned_datastore/lib/datastore_utils.py b/ckanext/versioned_datastore/lib/datastore_utils.py index 9c68947..e80c684 100644 --- a/ckanext/versioned_datastore/lib/datastore_utils.py +++ b/ckanext/versioned_datastore/lib/datastore_utils.py @@ -290,31 +290,17 @@ def is_ingestible(resource): def get_queue_length(queue_name): """ - This is a *very* hacky way to get the length of a queue, including anything already + This will only get the pending jobs in a queue, not any jobs that are currently processing. :param queue_name: the name of the queue to check, e.g. 'download' :return: length of queue as int """ - # because only pending jobs are counted, not active/running, if you add to the queue - # and job_list can't see it, the queue was empty; if it can, something else is - # already running. - def _temp_job(): - time.sleep(1) - - job = toolkit.enqueue_job( - _temp_job, - queue=queue_name, - title=f'{queue_name} queue status test', - rq_kwargs={'ttl': '1s'}, - ) queued_jobs = toolkit.get_action('job_list')( {'ignore_auth': True}, {'queues': [queue_name]} ) - job.delete() - return len(queued_jobs) From 8e967767944cfb5541a4ee5622e192c6af3da7fa Mon Sep 17 00:00:00 2001 From: Alice Butcher Date: Mon, 8 Jul 2024 15:27:36 +0100 Subject: [PATCH 2/2] fix: increase status tolerance for queue length also updates help messages for queues --- ckanext/versioned_datastore/plugin.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/ckanext/versioned_datastore/plugin.py b/ckanext/versioned_datastore/plugin.py index 702f0c2..98672cb 100644 --- a/ckanext/versioned_datastore/plugin.py +++ b/ckanext/versioned_datastore/plugin.py @@ -188,12 +188,10 @@ def modify_status_reports(self, status_reports): 'label': toolkit._('Downloads'), 'value': queued_downloads, 'group': toolkit._('Queues'), - 'help': toolkit._( - 'Number of downloads either currently processing or waiting in the queue' - ), + 'help': toolkit._('Number of downloads waiting in the queue'), 'state': 'good' - if queued_downloads == 0 - else ('ok' if queued_downloads < 3 else 'bad'), + if queued_downloads < 2 + else ('ok' if queued_downloads < 4 else 'bad'), } ) @@ -204,12 +202,10 @@ def modify_status_reports(self, status_reports): 'label': toolkit._('Imports'), 'value': queued_imports, 'group': toolkit._('Queues'), - 'help': toolkit._( - 'Number of import jobs either currently processing or waiting in the queue' - ), + 'help': toolkit._('Number of import jobs waiting in the queue'), 'state': 'good' - if queued_imports == 0 - else ('ok' if queued_imports < 3 else 'bad'), + if queued_imports < 2 + else ('ok' if queued_imports < 4 else 'bad'), } )