update credentials to use new rest-tools syntax (#373)

* update credentials to use new rest-tools syntax * <bot> update requirements-docs.txt * <bot> update requirements-tests.txt * <bot> update requirements.txt * fix flake8 * pass job temp cleaning test --------- Co-authored-by: github-actions <[email protected]>
WIPACrepo · Aug 27, 2024 · c4bcb01 · c4bcb01
1 parent 2fddba2
commit c4bcb01
Show file tree

Hide file tree

Showing 6 changed files with 68 additions and 49 deletions.
diff --git a/iceprod/credentials/server.py b/iceprod/credentials/server.py
@@ -11,7 +11,7 @@
 import motor.motor_asyncio
 import requests.exceptions
 from rest_tools.client import RestClient, ClientCredentialsAuth
-from rest_tools.server import RestServer
+from rest_tools.server import RestServer, ArgumentHandler, ArgumentSource
 from tornado.web import HTTPError
 from tornado.web import RequestHandler as TornadoRequestHandler
 from wipac_dev_tools import from_environment
@@ -59,8 +59,20 @@ async def check_attr_auth(self, arg, val, role):
                 raise HTTPError(500, 'auth could not be completed')
 
     async def create(self, db, base_data):
-        url = self.get_json_body_argument('url', type=str, strict_type=True)
-        credential_type = self.get_json_body_argument('type', type=str, choices=['s3', 'oauth'], strict_type=True)
+        now = time.time()
+        argo = ArgumentHandler(ArgumentSource.JSON_BODY_ARGUMENTS, self)
+        argo.add_argument('url', type=str, required=True)
+        argo.add_argument('type', type=str, choices=['s3', 'oauth'], required=True)
+        argo.add_argument('buckets', type=list, default=[], required=False)
+        argo.add_argument('access_key', type=str, default='', required=False)
+        argo.add_argument('secret_key', type=str, default='', required=False)
+        argo.add_argument('access_token', type=str, default='', required=False)
+        argo.add_argument('refresh_token', type=str, default='', required=False)
+        argo.add_argument('expire_date', type=float, default=now, required=False)
+        argo.add_argument('last_use', type=float, default=now, required=False)
+        args = vars(argo.parse_args())
+        url = args['url']
+        credential_type = args['type']
 
         base_data['url'] = url
         data = base_data.copy()
@@ -69,28 +81,24 @@ async def create(self, db, base_data):
         })
 
         if credential_type == 's3':
-            buckets = self.get_json_body_argument('buckets', type=list, strict_type=True)
-            access_key = self.get_json_body_argument('access_key', type=str, strict_type=True)
-            secret_key = self.get_json_body_argument('secret_key', type=str, strict_type=True)
-            if not buckets:
+            if not args['buckets']:
                 raise HTTPError(400, reason='must specify bucket(s)')
-            data['buckets'] = buckets
-            data['access_key'] = access_key
-            data['secret_key'] = secret_key
+            if not args['access_key']:
+                raise HTTPError(400, reason='must specify access_key')
+            if not args['secret_key']:
+                raise HTTPError(400, reason='must specify secret_key')
 
-        elif credential_type == 'oauth':
-            access_token = self.get_json_body_argument('access_token', default='', type=str, strict_type=True)
-            refresh_token = self.get_json_body_argument('refresh_token', default='', type=str, strict_type=True)
-            now = time.time()
-            exp = self.get_json_body_argument('expire_date', default=now, type=float)
-            last_use = self.get_json_body_argument('last_use', default=now, type=float)
+            data['buckets'] = args['buckets']
+            data['access_key'] = args['access_key']
+            data['secret_key'] = args['secret_key']
 
-            if (not access_token) and not refresh_token:
+        elif credential_type == 'oauth':
+            if (not args['access_token']) and not args['refresh_token']:
                 raise HTTPError(400, reason='must specify either access or refresh tokens')
-            data['access_token'] = access_token
-            data['refresh_token'] = refresh_token
-            data['expiration'] = exp
-            data['last_use'] = last_use
+            data['access_token'] = args['access_token']
+            data['refresh_token'] = args['refresh_token']
+            data['expiration'] = args['expire_date']
+            data['last_use'] = args['last_use']
 
             if 'refresh_token' in data and not data.get('access_token', ''):
                 new_cred = await self.refresh_service.refresh_cred(data)
@@ -108,17 +116,21 @@ async def create(self, db, base_data):
         )
 
     async def patch_cred(self, db, base_data):
-        base_data['url'] = self.get_json_body_argument('url', type=str, strict_type=True)
+        argo = ArgumentHandler(ArgumentSource.JSON_BODY_ARGUMENTS, self)
+        argo.add_argument('url', type=str, required=True)
+        argo.add_argument('buckets', type=list, default=[], required=False)
+        argo.add_argument('access_key', type=str, default='', required=False)
+        argo.add_argument('secret_key', type=str, default='', required=False)
+        argo.add_argument('access_token', type=str, default='', required=False)
+        argo.add_argument('refresh_token', type=str, default='', required=False)
+        argo.add_argument('expiration', type=float, default=0, required=False)
+        argo.add_argument('last_use', type=float, default=0, required=False)
+        args = vars(argo.parse_args())
+        base_data['url'] = args['url']
 
         data = {}
-        buckets = self.get_json_body_argument('buckets', default=[], type=list, strict_type=True)
-        if buckets:
-            data['buckets'] = buckets
-        for key in ('access_key', 'secret_key', 'access_token', 'refresh_token'):
-            if val := self.get_json_body_argument(key, default='', type=str, strict_type=True):
-                data[key] = val
-        for key in ('expiration', 'last_use'):
-            if val := self.get_json_body_argument(key, default=None, type=float):
+        for key in ('buckets', 'access_key', 'secret_key', 'access_token', 'refresh_token', 'expiration', 'last_use'):
+            if val := args[key]:
                 data[key] = val
 
         if 'refresh_token' in data and 'access_token' not in data:
@@ -245,9 +257,12 @@ async def delete(self, groupname):
             raise HTTPError(403, 'unauthorized')
 
         args = {'groupname': groupname}
-        url = self.get_json_body_argument('url', default='', type=str)
-        if url:
-            args['url'] = url
+
+        argo = ArgumentHandler(ArgumentSource.JSON_BODY_ARGUMENTS, self)
+        argo.add_argument('url', type=str, default='', required=False)
+        body_args = argo.parse_args()
+        if body_args.url:
+            args['url'] = body_args.url
 
         await self.db.group_creds.delete_many(args)
         self.write({})
@@ -336,9 +351,12 @@ async def delete(self, username):
             raise HTTPError(403, 'unauthorized')
 
         args = {'username': username}
-        url = self.get_json_body_argument('url', default='', type=str)
-        if url:
-            args['url'] = url
+
+        argo = ArgumentHandler(ArgumentSource.JSON_BODY_ARGUMENTS, self)
+        argo.add_argument('url', type=str, default='', required=False)
+        body_args = argo.parse_args()
+        if body_args.url:
+            args['url'] = body_args.url
 
         await self.db.user_creds.delete_many(args)
         self.write({})

diff --git a/iceprod/server/scheduled_tasks/job_temp_cleaning.py b/iceprod/server/scheduled_tasks/job_temp_cleaning.py
@@ -150,9 +150,8 @@ async def run(rest_client, temp_dir, list_dirs, rmtree, dataset=None, debug=Fals
                     logger.info('cleaning site_temp %r', dagtemp)
                     futures.add(asyncio.create_task(rmtree(dagtemp)))
 
-            while futures:
-                done, futures = await asyncio.wait(futures, return_when=asyncio.FIRST_COMPLETED)
-                for f in futures:
+            if futures:
+                for f in asyncio.as_completed(futures):
                     try:
                         await f
                     except Exception:

diff --git a/requirements-docs.txt b/requirements-docs.txt
@@ -12,9 +12,9 @@ asyncache==0.3.1
     # via iceprod (setup.py)
 babel==2.16.0
     # via sphinx
-boto3==1.35.6
+boto3==1.35.7
     # via iceprod (setup.py)
-botocore==1.35.6
+botocore==1.35.7
     # via
     #   boto3
     #   s3transfer

diff --git a/requirements-tests.txt b/requirements-tests.txt
@@ -10,11 +10,11 @@ asyncache==0.3.1
     # via iceprod (setup.py)
 beautifulsoup4==4.12.3
     # via iceprod (setup.py)
-boto3==1.35.6
+boto3==1.35.7
     # via
     #   iceprod (setup.py)
     #   moto
-botocore==1.35.6
+botocore==1.35.7
     # via
     #   boto3
     #   moto

diff --git a/requirements.txt b/requirements.txt
@@ -8,9 +8,9 @@ anyio==4.4.0
     # via httpx
 asyncache==0.3.1
     # via iceprod (setup.py)
-boto3==1.35.6
+boto3==1.35.7
     # via iceprod (setup.py)
-botocore==1.35.6
+botocore==1.35.7
     # via
     #   boto3
     #   s3transfer

diff --git a/tests/credentials/test_server.py b/tests/credentials/test_server.py
@@ -108,8 +108,9 @@ async def test_credentials_groups_s3(server):
     await client.request('POST', f'/groups/{GROUP}/credentials', data3)
 
     ret = await client.request('GET', f'/groups/{GROUP}/credentials')
-    data3['groupname'] = GROUP
-    assert ret == {data['url']: data3, data2['url']: data2}
+    data3_out = data3.copy()
+    data3_out['groupname'] = GROUP
+    assert ret == {data['url']: data3_out, data2['url']: data2}
 
     await client.request('DELETE', f'/groups/{GROUP}/credentials', {'url': 'http://foo'})
 
@@ -270,8 +271,9 @@ async def test_credentials_users_s3(server):
     await client.request('POST', f'/users/{USER}/credentials', data3)
 
     ret = await client.request('GET', f'/users/{USER}/credentials')
-    data3['username'] = USER
-    assert ret == {data['url']: data3, data2['url']: data2}
+    data3_out = data3.copy()
+    data3_out['username'] = USER
+    assert ret == {data['url']: data3_out, data2['url']: data2}
 
     await client.request('DELETE', f'/users/{USER}/credentials', {'url': 'http://foo'})