From d72f1827c03ed5d9135a7d61ac66dfb52768f3dd Mon Sep 17 00:00:00 2001 From: David Schultz Date: Mon, 2 Dec 2024 12:49:37 -0600 Subject: [PATCH] minor priorty and x509 fixes (#408) * minor priority updates * fix proxy check when setting out file * update requirements-docs.txt * update requirements-tests.txt * update requirements.txt * fix tests * workaround for httpx 0.28 mocker bug * update requirements-docs.txt * update requirements-tests.txt * update requirements.txt --------- Co-authored-by: github-actions --- iceprod/roles_groups.py | 2 +- .../scheduled_tasks/update_task_priority.py | 9 ++++--- iceprod/server/globus.py | 15 ++++++----- iceprod/server/priority.py | 8 ++++++ iceprod/server/queue.py | 1 - requirements-docs.txt | 22 +++++++--------- requirements-tests.txt | 26 +++++++++---------- requirements.txt | 20 +++++++------- tests/credentials/test_service.py | 4 +++ .../update_task_priority_test.py | 4 +-- 10 files changed, 61 insertions(+), 50 deletions(-) diff --git a/iceprod/roles_groups.py b/iceprod/roles_groups.py index a5ea33ae9..09c5e9411 100644 --- a/iceprod/roles_groups.py +++ b/iceprod/roles_groups.py @@ -17,5 +17,5 @@ 'admin': 1., 'simprod': .5, 'filtering': .9, - 'users': .7, + 'users': .8, } diff --git a/iceprod/scheduled_tasks/update_task_priority.py b/iceprod/scheduled_tasks/update_task_priority.py index b056e2c70..fe6be2cac 100644 --- a/iceprod/scheduled_tasks/update_task_priority.py +++ b/iceprod/scheduled_tasks/update_task_priority.py @@ -12,19 +12,21 @@ logger = logging.getLogger('update_task_priority') -async def run(rest_client, dataset_id=None, debug=False): +async def run(rest_client, dataset_id=None, status=None, debug=False): """ Actual runtime / loop. Args: rest_client (:py:class:`iceprod.core.rest_client.Client`): rest client dataset_id (str): (optional) dataset id to update + status (list): list of task statuses to update debug (bool): debug flag to propagate exceptions """ + assert status prio = Priority(rest_client) try: args = { - 'status': 'idle|waiting', + 'status': '|'.join(status), 'keys': 'task_id|depends|dataset_id', } if dataset_id: @@ -92,6 +94,7 @@ def main(): parser = argparse.ArgumentParser(description='run a scheduled task once') add_auth_to_argparse(parser) parser.add_argument('-d', '--dataset', type=str, default=None, help='dataset id (optional)') + parser.add_argument('-s', '--status', action='append', help='task statuses to update (optional)') parser.add_argument('--log-level', default='info', help='log level') parser.add_argument('--debug', default=False, action='store_true', help='debug enabled') args = parser.parse_args() @@ -101,7 +104,7 @@ def main(): rest_client = create_rest_client(args) - asyncio.run(run(rest_client, dataset_id=args.dataset, debug=args.debug)) + asyncio.run(run(rest_client, dataset_id=args.dataset, status=args.status, debug=args.debug)) if __name__ == '__main__': diff --git a/iceprod/server/globus.py b/iceprod/server/globus.py index 87e6498fc..7bca66746 100644 --- a/iceprod/server/globus.py +++ b/iceprod/server/globus.py @@ -51,13 +51,16 @@ def update_proxy(self): if 'duration' not in self.cfg: raise Exception('duration missing') logger.info('duration: %r',self.cfg['duration']) + cmd = [ + 'grid-proxy-info', + '-e', + '-valid', + '%d:0'%self.cfg['duration'], + ] + if 'out' in self.cfg: + cmd += ['-f', self.cfg['out']] if subprocess.call( - [ - 'grid-proxy-info', - '-e', - '-valid', - '%d:0'%self.cfg['duration'], - ], + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ): diff --git a/iceprod/server/priority.py b/iceprod/server/priority.py index d0a823164..7005464fc 100644 --- a/iceprod/server/priority.py +++ b/iceprod/server/priority.py @@ -6,6 +6,8 @@ from datetime import datetime, UTC import logging +import wipac_dev_tools + from iceprod.client_auth import add_auth_to_argparse, create_rest_client from iceprod.roles_groups import GROUP_PRIORITIES from iceprod.server.util import str2datetime @@ -88,6 +90,7 @@ async def _get_max_dataset_prio_user(self, user): try: return max(d['priority'] for d in self.dataset_cache.values() if 'priority' in d and d['username'] == user) except ValueError: + logger.info('error getting max dataset prio', exc_info=True) return 1. async def _get_max_dataset_prio_group(self, group): @@ -264,6 +267,11 @@ def main(): logformat = '%(asctime)s %(levelname)s %(name)s %(module)s:%(lineno)s - %(message)s' logging.basicConfig(format=logformat, level=logging.DEBUG if args.debug else logging.INFO) + wipac_dev_tools.logging_tools.set_level( + level='DEBUG', + first_party_loggers='priority', + future_third_parties=['SavedDeviceGrantAuth'], + ) rest_client = create_rest_client(args) diff --git a/iceprod/server/queue.py b/iceprod/server/queue.py index 944773465..7b13902b9 100644 --- a/iceprod/server/queue.py +++ b/iceprod/server/queue.py @@ -145,7 +145,6 @@ def blocking(): self.cfg['queue']['x509proxy'] = self.proxy.get_proxy() except Exception: logger.warning('cannot setup x509 proxy', exc_info=True) - raise RuntimeError('cannot setup x509 proxy') while True: start = time.monotonic() diff --git a/requirements-docs.txt b/requirements-docs.txt index b2eee08d0..63b78fd28 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -16,9 +16,9 @@ attrs==24.2.0 # referencing babel==2.16.0 # via sphinx -boto3==1.35.68 +boto3==1.35.72 # via iceprod (setup.py) -botocore==1.35.68 +botocore==1.35.72 # via # boto3 # s3transfer @@ -37,7 +37,7 @@ cffi==1.17.1 # via cryptography charset-normalizer==3.4.0 # via requests -cryptography==43.0.3 +cryptography==44.0.0 # via # iceprod (setup.py) # pyjwt @@ -50,11 +50,11 @@ exceptiongroup==1.2.2 # via anyio h11==0.14.0 # via httpcore -htcondor==24.1.1 +htcondor==24.2.1 # via iceprod (setup.py) httpcore==1.0.7 # via httpx -httpx==0.27.2 +httpx==0.28.0 # via iceprod (setup.py) idna==3.10 # via @@ -89,13 +89,13 @@ pycparser==2.22 # via cffi pygments==2.18.0 # via sphinx -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via wipac-rest-tools pymongo==4.9.2 # via # iceprod (setup.py) # motor -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via iceprod (setup.py) python-dateutil==2.9.0.post0 # via @@ -123,7 +123,7 @@ requests-futures==1.0.2 # wipac-rest-tools requests-toolbelt==1.0.0 # via iceprod (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.0 # via # jsonschema # referencing @@ -134,9 +134,7 @@ setproctitle==1.3.4 six==1.16.0 # via python-dateutil sniffio==1.3.1 - # via - # anyio - # httpx + # via anyio snowballstemmer==2.2.0 # via sphinx sphinx==8.1.3 @@ -155,7 +153,7 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx statsd==4.0.1 # via iceprod (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via sphinx tornado==6.4.2 # via diff --git a/requirements-tests.txt b/requirements-tests.txt index 8a19ab7dd..ac7ce51b5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -14,11 +14,11 @@ attrs==24.2.0 # referencing beautifulsoup4==4.12.3 # via iceprod (setup.py) -boto3==1.35.68 +boto3==1.35.72 # via # iceprod (setup.py) # moto -botocore==1.35.68 +botocore==1.35.72 # via # boto3 # moto @@ -38,11 +38,11 @@ cffi==1.17.1 # via cryptography charset-normalizer==3.4.0 # via requests -coverage[toml]==7.6.7 +coverage[toml]==7.6.8 # via # iceprod (setup.py) # pytest-cov -cryptography==43.0.3 +cryptography==44.0.0 # via # iceprod (setup.py) # moto @@ -60,11 +60,11 @@ flexmock==0.12.1 # via iceprod (setup.py) h11==0.14.0 # via httpcore -htcondor==24.1.1 +htcondor==24.2.1 # via iceprod (setup.py) httpcore==1.0.7 # via httpx -httpx==0.27.2 +httpx==0.28.0 # via # iceprod (setup.py) # respx @@ -95,7 +95,7 @@ mccabe==0.7.0 # via flake8 mock==5.1.0 # via iceprod (setup.py) -moto[s3]==5.0.21 +moto[s3]==5.0.22 # via iceprod (setup.py) motor==3.6.0 # via iceprod (setup.py) @@ -115,13 +115,13 @@ pycparser==2.22 # via cffi pyflakes==3.2.0 # via flake8 -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via wipac-rest-tools pymongo==4.9.2 # via # iceprod (setup.py) # motor -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via iceprod (setup.py) pytest==8.0.2 # via @@ -173,7 +173,7 @@ responses==0.25.3 # via moto respx==0.21.1 # via iceprod (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.0 # via # jsonschema # referencing @@ -184,14 +184,12 @@ setproctitle==1.3.4 six==1.16.0 # via python-dateutil sniffio==1.3.1 - # via - # anyio - # httpx + # via anyio soupsieve==2.6 # via beautifulsoup4 statsd==4.0.1 # via iceprod (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via # coverage # pytest diff --git a/requirements.txt b/requirements.txt index 50125fa39..47899c1db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,9 +12,9 @@ attrs==24.2.0 # via # jsonschema # referencing -boto3==1.35.68 +boto3==1.35.72 # via iceprod (setup.py) -botocore==1.35.68 +botocore==1.35.72 # via # boto3 # s3transfer @@ -33,7 +33,7 @@ cffi==1.17.1 # via cryptography charset-normalizer==3.4.0 # via requests -cryptography==43.0.3 +cryptography==44.0.0 # via # iceprod (setup.py) # pyjwt @@ -44,11 +44,11 @@ exceptiongroup==1.2.2 # via anyio h11==0.14.0 # via httpcore -htcondor==24.1.1 +htcondor==24.2.1 # via iceprod (setup.py) httpcore==1.0.7 # via httpx -httpx==0.27.2 +httpx==0.28.0 # via iceprod (setup.py) idna==3.10 # via @@ -73,13 +73,13 @@ pyasn1==0.6.1 # via ldap3 pycparser==2.22 # via cffi -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via wipac-rest-tools pymongo==4.9.2 # via # iceprod (setup.py) # motor -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via iceprod (setup.py) python-dateutil==2.9.0.post0 # via @@ -106,7 +106,7 @@ requests-futures==1.0.2 # wipac-rest-tools requests-toolbelt==1.0.0 # via iceprod (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.0 # via # jsonschema # referencing @@ -117,9 +117,7 @@ setproctitle==1.3.4 six==1.16.0 # via python-dateutil sniffio==1.3.1 - # via - # anyio - # httpx + # via anyio statsd==4.0.1 # via iceprod (setup.py) tornado==6.4.2 diff --git a/tests/credentials/test_service.py b/tests/credentials/test_service.py index 997c0350f..9f9711809 100644 --- a/tests/credentials/test_service.py +++ b/tests/credentials/test_service.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock import motor.motor_asyncio import jwt +import pytest import iceprod.credentials.service @@ -40,6 +41,7 @@ async def test_credentials_service_refresh_empty(mongo_url, mongo_clear, respx_m await rs._run_once() +@pytest.mark.respx(using="httpx", assert_all_called=False) async def test_credentials_service_refresh_not_exp(mongo_url, mongo_clear, respx_mock): db = motor.motor_asyncio.AsyncIOMotorClient(mongo_url)['creds'] clients = '{}' @@ -66,6 +68,7 @@ async def test_credentials_service_refresh_not_exp(mongo_url, mongo_clear, respx assert rs.last_success_time is not None +@pytest.mark.respx(using="httpx") async def test_credentials_service_refresh_group(mongo_url, mongo_clear, respx_mock, monkeypatch): db = motor.motor_asyncio.AsyncIOMotorClient(mongo_url)['creds'] clients = json.dumps({ @@ -103,6 +106,7 @@ async def test_credentials_service_refresh_group(mongo_url, mongo_clear, respx_m assert ret['expiration'] == now+1000 +@pytest.mark.respx(using="httpx") async def test_credentials_service_refresh_user(mongo_url, mongo_clear, respx_mock, monkeypatch): db = motor.motor_asyncio.AsyncIOMotorClient(mongo_url)['creds'] clients = json.dumps({ diff --git a/tests/scheduled_tasks/update_task_priority_test.py b/tests/scheduled_tasks/update_task_priority_test.py index 708322d58..00c538989 100644 --- a/tests/scheduled_tasks/update_task_priority_test.py +++ b/tests/scheduled_tasks/update_task_priority_test.py @@ -35,7 +35,7 @@ async def client(method, url, args=None): client.called = False rc.request = client - await update_task_priority.run(rc, debug=True) + await update_task_priority.run(rc, status=['idle', 'waiting'], debug=True) assert client.called @@ -46,7 +46,7 @@ async def test_201_run(): rc.request = AsyncMock(side_effect=Exception()) # check it normally hides the error - await update_task_priority.run(rc, debug=False) + await update_task_priority.run(rc, status=['idle', 'waiting'], debug=False) with pytest.raises(Exception): await reset_tasks.run(rc, debug=True)