diff --git a/iceprod/credentials/server.py b/iceprod/credentials/server.py index 02afdfc3..915711a4 100644 --- a/iceprod/credentials/server.py +++ b/iceprod/credentials/server.py @@ -11,7 +11,7 @@ import motor.motor_asyncio import requests.exceptions from rest_tools.client import RestClient, ClientCredentialsAuth -from rest_tools.server import RestServer +from rest_tools.server import RestServer, ArgumentHandler, ArgumentSource from tornado.web import HTTPError from tornado.web import RequestHandler as TornadoRequestHandler from wipac_dev_tools import from_environment @@ -59,8 +59,20 @@ async def check_attr_auth(self, arg, val, role): raise HTTPError(500, 'auth could not be completed') async def create(self, db, base_data): - url = self.get_json_body_argument('url', type=str, strict_type=True) - credential_type = self.get_json_body_argument('type', type=str, choices=['s3', 'oauth'], strict_type=True) + now = time.time() + argo = ArgumentHandler(ArgumentSource.JSON_BODY_ARGUMENTS, self) + argo.add_argument('url', type=str, required=True) + argo.add_argument('type', type=str, choices=['s3', 'oauth'], required=True) + argo.add_argument('buckets', type=list, default=[], required=False) + argo.add_argument('access_key', type=str, default='', required=False) + argo.add_argument('secret_key', type=str, default='', required=False) + argo.add_argument('access_token', type=str, default='', required=False) + argo.add_argument('refresh_token', type=str, default='', required=False) + argo.add_argument('expire_date', type=float, default=now, required=False) + argo.add_argument('last_use', type=float, default=now, required=False) + args = vars(argo.parse_args()) + url = args['url'] + credential_type = args['type'] base_data['url'] = url data = base_data.copy() @@ -69,28 +81,24 @@ async def create(self, db, base_data): }) if credential_type == 's3': - buckets = self.get_json_body_argument('buckets', type=list, strict_type=True) - access_key = self.get_json_body_argument('access_key', type=str, strict_type=True) - secret_key = self.get_json_body_argument('secret_key', type=str, strict_type=True) - if not buckets: + if not args['buckets']: raise HTTPError(400, reason='must specify bucket(s)') - data['buckets'] = buckets - data['access_key'] = access_key - data['secret_key'] = secret_key + if not args['access_key']: + raise HTTPError(400, reason='must specify access_key') + if not args['secret_key']: + raise HTTPError(400, reason='must specify secret_key') - elif credential_type == 'oauth': - access_token = self.get_json_body_argument('access_token', default='', type=str, strict_type=True) - refresh_token = self.get_json_body_argument('refresh_token', default='', type=str, strict_type=True) - now = time.time() - exp = self.get_json_body_argument('expire_date', default=now, type=float) - last_use = self.get_json_body_argument('last_use', default=now, type=float) + data['buckets'] = args['buckets'] + data['access_key'] = args['access_key'] + data['secret_key'] = args['secret_key'] - if (not access_token) and not refresh_token: + elif credential_type == 'oauth': + if (not args['access_token']) and not args['refresh_token']: raise HTTPError(400, reason='must specify either access or refresh tokens') - data['access_token'] = access_token - data['refresh_token'] = refresh_token - data['expiration'] = exp - data['last_use'] = last_use + data['access_token'] = args['access_token'] + data['refresh_token'] = args['refresh_token'] + data['expiration'] = args['expire_date'] + data['last_use'] = args['last_use'] if 'refresh_token' in data and not data.get('access_token', ''): new_cred = await self.refresh_service.refresh_cred(data) @@ -108,17 +116,21 @@ async def create(self, db, base_data): ) async def patch_cred(self, db, base_data): - base_data['url'] = self.get_json_body_argument('url', type=str, strict_type=True) + argo = ArgumentHandler(ArgumentSource.JSON_BODY_ARGUMENTS, self) + argo.add_argument('url', type=str, required=True) + argo.add_argument('buckets', type=list, default=[], required=False) + argo.add_argument('access_key', type=str, default='', required=False) + argo.add_argument('secret_key', type=str, default='', required=False) + argo.add_argument('access_token', type=str, default='', required=False) + argo.add_argument('refresh_token', type=str, default='', required=False) + argo.add_argument('expiration', type=float, default=0, required=False) + argo.add_argument('last_use', type=float, default=0, required=False) + args = vars(argo.parse_args()) + base_data['url'] = args['url'] data = {} - buckets = self.get_json_body_argument('buckets', default=[], type=list, strict_type=True) - if buckets: - data['buckets'] = buckets - for key in ('access_key', 'secret_key', 'access_token', 'refresh_token'): - if val := self.get_json_body_argument(key, default='', type=str, strict_type=True): - data[key] = val - for key in ('expiration', 'last_use'): - if val := self.get_json_body_argument(key, default=None, type=float): + for key in ('buckets', 'access_key', 'secret_key', 'access_token', 'refresh_token', 'expiration', 'last_use'): + if val := args[key]: data[key] = val if 'refresh_token' in data and 'access_token' not in data: @@ -245,9 +257,12 @@ async def delete(self, groupname): raise HTTPError(403, 'unauthorized') args = {'groupname': groupname} - url = self.get_json_body_argument('url', default='', type=str) - if url: - args['url'] = url + + argo = ArgumentHandler(ArgumentSource.JSON_BODY_ARGUMENTS, self) + argo.add_argument('url', type=str, default='', required=False) + body_args = argo.parse_args() + if body_args.url: + args['url'] = body_args.url await self.db.group_creds.delete_many(args) self.write({}) @@ -336,9 +351,12 @@ async def delete(self, username): raise HTTPError(403, 'unauthorized') args = {'username': username} - url = self.get_json_body_argument('url', default='', type=str) - if url: - args['url'] = url + + argo = ArgumentHandler(ArgumentSource.JSON_BODY_ARGUMENTS, self) + argo.add_argument('url', type=str, default='', required=False) + body_args = argo.parse_args() + if body_args.url: + args['url'] = body_args.url await self.db.user_creds.delete_many(args) self.write({}) diff --git a/iceprod/server/scheduled_tasks/job_temp_cleaning.py b/iceprod/server/scheduled_tasks/job_temp_cleaning.py index 5ef7b188..855eadbd 100644 --- a/iceprod/server/scheduled_tasks/job_temp_cleaning.py +++ b/iceprod/server/scheduled_tasks/job_temp_cleaning.py @@ -150,9 +150,8 @@ async def run(rest_client, temp_dir, list_dirs, rmtree, dataset=None, debug=Fals logger.info('cleaning site_temp %r', dagtemp) futures.add(asyncio.create_task(rmtree(dagtemp))) - while futures: - done, futures = await asyncio.wait(futures, return_when=asyncio.FIRST_COMPLETED) - for f in futures: + if futures: + for f in asyncio.as_completed(futures): try: await f except Exception: diff --git a/requirements-docs.txt b/requirements-docs.txt index 34a106ae..22371a46 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -12,9 +12,9 @@ asyncache==0.3.1 # via iceprod (setup.py) babel==2.16.0 # via sphinx -boto3==1.35.6 +boto3==1.35.7 # via iceprod (setup.py) -botocore==1.35.6 +botocore==1.35.7 # via # boto3 # s3transfer diff --git a/requirements-tests.txt b/requirements-tests.txt index 72c0c77a..92b38598 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,11 +10,11 @@ asyncache==0.3.1 # via iceprod (setup.py) beautifulsoup4==4.12.3 # via iceprod (setup.py) -boto3==1.35.6 +boto3==1.35.7 # via # iceprod (setup.py) # moto -botocore==1.35.6 +botocore==1.35.7 # via # boto3 # moto diff --git a/requirements.txt b/requirements.txt index 290ebe6f..be07d417 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,9 +8,9 @@ anyio==4.4.0 # via httpx asyncache==0.3.1 # via iceprod (setup.py) -boto3==1.35.6 +boto3==1.35.7 # via iceprod (setup.py) -botocore==1.35.6 +botocore==1.35.7 # via # boto3 # s3transfer diff --git a/tests/credentials/test_server.py b/tests/credentials/test_server.py index 495d8542..f006dc11 100644 --- a/tests/credentials/test_server.py +++ b/tests/credentials/test_server.py @@ -108,8 +108,9 @@ async def test_credentials_groups_s3(server): await client.request('POST', f'/groups/{GROUP}/credentials', data3) ret = await client.request('GET', f'/groups/{GROUP}/credentials') - data3['groupname'] = GROUP - assert ret == {data['url']: data3, data2['url']: data2} + data3_out = data3.copy() + data3_out['groupname'] = GROUP + assert ret == {data['url']: data3_out, data2['url']: data2} await client.request('DELETE', f'/groups/{GROUP}/credentials', {'url': 'http://foo'}) @@ -270,8 +271,9 @@ async def test_credentials_users_s3(server): await client.request('POST', f'/users/{USER}/credentials', data3) ret = await client.request('GET', f'/users/{USER}/credentials') - data3['username'] = USER - assert ret == {data['url']: data3, data2['url']: data2} + data3_out = data3.copy() + data3_out['username'] = USER + assert ret == {data['url']: data3_out, data2['url']: data2} await client.request('DELETE', f'/users/{USER}/credentials', {'url': 'http://foo'})