minor priorty and x509 fixes (#408)

* minor priority updates * fix proxy check when setting out file * <bot> update requirements-docs.txt * <bot> update requirements-tests.txt * <bot> update requirements.txt * fix tests * workaround for httpx 0.28 mocker bug * <bot> update requirements-docs.txt * <bot> update requirements-tests.txt * <bot> update requirements.txt --------- Co-authored-by: github-actions <[email protected]>
WIPACrepo · Dec 2, 2024 · d72f182 · d72f182
1 parent 6bc32fe
commit d72f182
Show file tree

Hide file tree

Showing 10 changed files with 61 additions and 50 deletions.
diff --git a/iceprod/roles_groups.py b/iceprod/roles_groups.py
@@ -17,5 +17,5 @@
     'admin': 1.,
     'simprod': .5,
     'filtering': .9,
-    'users': .7,
+    'users': .8,
 }
diff --git a/iceprod/scheduled_tasks/update_task_priority.py b/iceprod/scheduled_tasks/update_task_priority.py
@@ -12,19 +12,21 @@
 logger = logging.getLogger('update_task_priority')
 
 
-async def run(rest_client, dataset_id=None, debug=False):
+async def run(rest_client, dataset_id=None, status=None, debug=False):
     """
     Actual runtime / loop.
 
     Args:
         rest_client (:py:class:`iceprod.core.rest_client.Client`): rest client
         dataset_id (str): (optional) dataset id to update
+        status (list): list of task statuses to update
         debug (bool): debug flag to propagate exceptions
     """
+    assert status
     prio = Priority(rest_client)
     try:
         args = {
-            'status': 'idle|waiting',
+            'status': '|'.join(status),
             'keys': 'task_id|depends|dataset_id',
         }
         if dataset_id:
@@ -92,6 +94,7 @@ def main():
     parser = argparse.ArgumentParser(description='run a scheduled task once')
     add_auth_to_argparse(parser)
     parser.add_argument('-d', '--dataset', type=str, default=None, help='dataset id (optional)')
+    parser.add_argument('-s', '--status', action='append', help='task statuses to update (optional)')
     parser.add_argument('--log-level', default='info', help='log level')
     parser.add_argument('--debug', default=False, action='store_true', help='debug enabled')
     args = parser.parse_args()
@@ -101,7 +104,7 @@ def main():
 
     rest_client = create_rest_client(args)
 
-    asyncio.run(run(rest_client, dataset_id=args.dataset, debug=args.debug))
+    asyncio.run(run(rest_client, dataset_id=args.dataset, status=args.status, debug=args.debug))
 
 
 if __name__ == '__main__':

diff --git a/iceprod/server/globus.py b/iceprod/server/globus.py
@@ -51,13 +51,16 @@ def update_proxy(self):
         if 'duration' not in self.cfg:
             raise Exception('duration missing')
         logger.info('duration: %r',self.cfg['duration'])
+        cmd = [
+            'grid-proxy-info',
+            '-e',
+            '-valid',
+            '%d:0'%self.cfg['duration'],
+        ]
+        if 'out' in self.cfg:
+            cmd += ['-f', self.cfg['out']]
         if subprocess.call(
-            [
-                'grid-proxy-info',
-                '-e',
-                '-valid',
-                '%d:0'%self.cfg['duration'],
-            ],
+            cmd,
             stdout=subprocess.DEVNULL,
             stderr=subprocess.DEVNULL
         ):

diff --git a/iceprod/server/priority.py b/iceprod/server/priority.py
@@ -6,6 +6,8 @@
 from datetime import datetime, UTC
 import logging
 
+import wipac_dev_tools
+
 from iceprod.client_auth import add_auth_to_argparse, create_rest_client
 from iceprod.roles_groups import GROUP_PRIORITIES
 from iceprod.server.util import str2datetime
@@ -88,6 +90,7 @@ async def _get_max_dataset_prio_user(self, user):
         try:
             return max(d['priority'] for d in self.dataset_cache.values() if 'priority' in d and d['username'] == user)
         except ValueError:
+            logger.info('error getting max dataset prio', exc_info=True)
             return 1.
 
     async def _get_max_dataset_prio_group(self, group):
@@ -264,6 +267,11 @@ def main():
 
     logformat = '%(asctime)s %(levelname)s %(name)s %(module)s:%(lineno)s - %(message)s'
     logging.basicConfig(format=logformat, level=logging.DEBUG if args.debug else logging.INFO)
+    wipac_dev_tools.logging_tools.set_level(
+        level='DEBUG',
+        first_party_loggers='priority',
+        future_third_parties=['SavedDeviceGrantAuth'],
+    )
 
     rest_client = create_rest_client(args)
 

diff --git a/iceprod/server/queue.py b/iceprod/server/queue.py
@@ -145,7 +145,6 @@ def blocking():
                 self.cfg['queue']['x509proxy'] = self.proxy.get_proxy()
             except Exception:
                 logger.warning('cannot setup x509 proxy', exc_info=True)
-                raise RuntimeError('cannot setup x509 proxy')
 
         while True:
             start = time.monotonic()

diff --git a/requirements-docs.txt b/requirements-docs.txt
@@ -16,9 +16,9 @@ attrs==24.2.0
     #   referencing
 babel==2.16.0
     # via sphinx
-boto3==1.35.68
+boto3==1.35.72
     # via iceprod (setup.py)
-botocore==1.35.68
+botocore==1.35.72
     # via
     #   boto3
     #   s3transfer
@@ -37,7 +37,7 @@ cffi==1.17.1
     # via cryptography
 charset-normalizer==3.4.0
     # via requests
-cryptography==43.0.3
+cryptography==44.0.0
     # via
     #   iceprod (setup.py)
     #   pyjwt
@@ -50,11 +50,11 @@ exceptiongroup==1.2.2
     # via anyio
 h11==0.14.0
     # via httpcore
-htcondor==24.1.1
+htcondor==24.2.1
     # via iceprod (setup.py)
 httpcore==1.0.7
     # via httpx
-httpx==0.27.2
+httpx==0.28.0
     # via iceprod (setup.py)
 idna==3.10
     # via
@@ -89,13 +89,13 @@ pycparser==2.22
     # via cffi
 pygments==2.18.0
     # via sphinx
-pyjwt[crypto]==2.10.0
+pyjwt[crypto]==2.10.1
     # via wipac-rest-tools
 pymongo==4.9.2
     # via
     #   iceprod (setup.py)
     #   motor
-pyopenssl==24.2.1
+pyopenssl==24.3.0
     # via iceprod (setup.py)
 python-dateutil==2.9.0.post0
     # via
@@ -123,7 +123,7 @@ requests-futures==1.0.2
     #   wipac-rest-tools
 requests-toolbelt==1.0.0
     # via iceprod (setup.py)
-rpds-py==0.21.0
+rpds-py==0.22.0
     # via
     #   jsonschema
     #   referencing
@@ -134,9 +134,7 @@ setproctitle==1.3.4
 six==1.16.0
     # via python-dateutil
 sniffio==1.3.1
-    # via
-    #   anyio
-    #   httpx
+    # via anyio
 snowballstemmer==2.2.0
     # via sphinx
 sphinx==8.1.3
@@ -155,7 +153,7 @@ sphinxcontrib-serializinghtml==2.0.0
     # via sphinx
 statsd==4.0.1
     # via iceprod (setup.py)
-tomli==2.1.0
+tomli==2.2.1
     # via sphinx
 tornado==6.4.2
     # via

diff --git a/requirements-tests.txt b/requirements-tests.txt
@@ -14,11 +14,11 @@ attrs==24.2.0
     #   referencing
 beautifulsoup4==4.12.3
     # via iceprod (setup.py)
-boto3==1.35.68
+boto3==1.35.72
     # via
     #   iceprod (setup.py)
     #   moto
-botocore==1.35.68
+botocore==1.35.72
     # via
     #   boto3
     #   moto
@@ -38,11 +38,11 @@ cffi==1.17.1
     # via cryptography
 charset-normalizer==3.4.0
     # via requests
-coverage[toml]==7.6.7
+coverage[toml]==7.6.8
     # via
     #   iceprod (setup.py)
     #   pytest-cov
-cryptography==43.0.3
+cryptography==44.0.0
     # via
     #   iceprod (setup.py)
     #   moto
@@ -60,11 +60,11 @@ flexmock==0.12.1
     # via iceprod (setup.py)
 h11==0.14.0
     # via httpcore
-htcondor==24.1.1
+htcondor==24.2.1
     # via iceprod (setup.py)
 httpcore==1.0.7
     # via httpx
-httpx==0.27.2
+httpx==0.28.0
     # via
     #   iceprod (setup.py)
     #   respx
@@ -95,7 +95,7 @@ mccabe==0.7.0
     # via flake8
 mock==5.1.0
     # via iceprod (setup.py)
-moto[s3]==5.0.21
+moto[s3]==5.0.22
     # via iceprod (setup.py)
 motor==3.6.0
     # via iceprod (setup.py)
@@ -115,13 +115,13 @@ pycparser==2.22
     # via cffi
 pyflakes==3.2.0
     # via flake8
-pyjwt[crypto]==2.10.0
+pyjwt[crypto]==2.10.1
     # via wipac-rest-tools
 pymongo==4.9.2
     # via
     #   iceprod (setup.py)
     #   motor
-pyopenssl==24.2.1
+pyopenssl==24.3.0
     # via iceprod (setup.py)
 pytest==8.0.2
     # via
@@ -173,7 +173,7 @@ responses==0.25.3
     # via moto
 respx==0.21.1
     # via iceprod (setup.py)
-rpds-py==0.21.0
+rpds-py==0.22.0
     # via
     #   jsonschema
     #   referencing
@@ -184,14 +184,12 @@ setproctitle==1.3.4
 six==1.16.0
     # via python-dateutil
 sniffio==1.3.1
-    # via
-    #   anyio
-    #   httpx
+    # via anyio
 soupsieve==2.6
     # via beautifulsoup4
 statsd==4.0.1
     # via iceprod (setup.py)
-tomli==2.1.0
+tomli==2.2.1
     # via
     #   coverage
     #   pytest

diff --git a/requirements.txt b/requirements.txt
@@ -12,9 +12,9 @@ attrs==24.2.0
     # via
     #   jsonschema
     #   referencing
-boto3==1.35.68
+boto3==1.35.72
     # via iceprod (setup.py)
-botocore==1.35.68
+botocore==1.35.72
     # via
     #   boto3
     #   s3transfer
@@ -33,7 +33,7 @@ cffi==1.17.1
     # via cryptography
 charset-normalizer==3.4.0
     # via requests
-cryptography==43.0.3
+cryptography==44.0.0
     # via
     #   iceprod (setup.py)
     #   pyjwt
@@ -44,11 +44,11 @@ exceptiongroup==1.2.2
     # via anyio
 h11==0.14.0
     # via httpcore
-htcondor==24.1.1
+htcondor==24.2.1
     # via iceprod (setup.py)
 httpcore==1.0.7
     # via httpx
-httpx==0.27.2
+httpx==0.28.0
     # via iceprod (setup.py)
 idna==3.10
     # via
@@ -73,13 +73,13 @@ pyasn1==0.6.1
     # via ldap3
 pycparser==2.22
     # via cffi
-pyjwt[crypto]==2.10.0
+pyjwt[crypto]==2.10.1
     # via wipac-rest-tools
 pymongo==4.9.2
     # via
     #   iceprod (setup.py)
     #   motor
-pyopenssl==24.2.1
+pyopenssl==24.3.0
     # via iceprod (setup.py)
 python-dateutil==2.9.0.post0
     # via
@@ -106,7 +106,7 @@ requests-futures==1.0.2
     #   wipac-rest-tools
 requests-toolbelt==1.0.0
     # via iceprod (setup.py)
-rpds-py==0.21.0
+rpds-py==0.22.0
     # via
     #   jsonschema
     #   referencing
@@ -117,9 +117,7 @@ setproctitle==1.3.4
 six==1.16.0
     # via python-dateutil
 sniffio==1.3.1
-    # via
-    #   anyio
-    #   httpx
+    # via anyio
 statsd==4.0.1
     # via iceprod (setup.py)
 tornado==6.4.2

diff --git a/tests/credentials/test_service.py b/tests/credentials/test_service.py
@@ -3,6 +3,7 @@
 from unittest.mock import MagicMock
 import motor.motor_asyncio
 import jwt
+import pytest
 
 import iceprod.credentials.service
 
@@ -40,6 +41,7 @@ async def test_credentials_service_refresh_empty(mongo_url, mongo_clear, respx_m
     await rs._run_once()
 
 
+@pytest.mark.respx(using="httpx", assert_all_called=False)
 async def test_credentials_service_refresh_not_exp(mongo_url, mongo_clear, respx_mock):
     db = motor.motor_asyncio.AsyncIOMotorClient(mongo_url)['creds']
     clients = '{}'
@@ -66,6 +68,7 @@ async def test_credentials_service_refresh_not_exp(mongo_url, mongo_clear, respx
     assert rs.last_success_time is not None
 
 
+@pytest.mark.respx(using="httpx")
 async def test_credentials_service_refresh_group(mongo_url, mongo_clear, respx_mock, monkeypatch):
     db = motor.motor_asyncio.AsyncIOMotorClient(mongo_url)['creds']
     clients = json.dumps({
@@ -103,6 +106,7 @@ async def test_credentials_service_refresh_group(mongo_url, mongo_clear, respx_m
     assert ret['expiration'] == now+1000
 
 
+@pytest.mark.respx(using="httpx")
 async def test_credentials_service_refresh_user(mongo_url, mongo_clear, respx_mock, monkeypatch):
     db = motor.motor_asyncio.AsyncIOMotorClient(mongo_url)['creds']
     clients = json.dumps({