diff --git a/tests/temp_factories.py b/tests/temp_factories.py index d1e8ac6732..914d4acaaa 100644 --- a/tests/temp_factories.py +++ b/tests/temp_factories.py @@ -107,6 +107,8 @@ def _make_rse(self, scheme, protocol_impl, parameters=None, add_rse_kwargs=None, } } } + if scheme == 'srm': + protocol_parameters["extended_attributes"] = {"web_service_path": "/srm/managerv2?SFN=", "space_token": "RUCIODISK"} protocol_parameters.update(parameters or {}) rse_core.add_protocol(rse_id=rse_id, parameter=protocol_parameters, session=session) self.created_rses.add(rse_id) diff --git a/tests/test_api_external_representation.py b/tests/test_api_external_representation.py index 4cc322c478..4fa6e23e3f 100644 --- a/tests/test_api_external_representation.py +++ b/tests/test_api_external_representation.py @@ -36,7 +36,6 @@ from rucio.common.config import config_get_bool from rucio.common.types import InternalScope from rucio.common.utils import api_update_return_dict, generate_uuid -from rucio.core.rse import get_rse_id from rucio.core.vo import add_vo, vo_exists from rucio.daemons.abacus import rse as abacus_rse from rucio.daemons.judge import cleaner @@ -93,34 +92,31 @@ class TestApiExternalRepresentation: @classmethod def setUpClass(cls): # Get test RSEs - cls.rse_name = 'MOCK' - cls.rse_id = get_rse_id(rse=cls.rse_name, vo=cls.vo) - cls.rse2_name = 'MOCK2' - cls.rse2_id = get_rse_id(rse=cls.rse2_name, vo=cls.vo) - cls.rse3_name = rse_name_generator() cls.rse3_id = api_rse.add_rse(cls.rse3_name, 'root', vo=cls.vo2 if cls.vo2 else cls.vo) cls.rse4_name = rse_name_generator() cls.rse4_id = api_rse.add_rse(cls.rse4_name, 'root', vo=cls.vo2 if cls.vo2 else cls.vo) api_rse.add_distance(cls.rse3_name, cls.rse4_name, issuer='root', distance=3, vo=cls.vo2 or cls.vo) - def test_api_update_return_dict(self, account, account_name, scope_name, scope): + def test_api_update_return_dict(self, rse_factory, account, account_name, scope_name, scope): """ API: Test the conversion of dictionaries to external representation """ + rse1, rse1_id = rse_factory.make_rse() + rse2, rse2_id = rse_factory.make_rse() test_dict = {'account': account, 'scope': scope, - 'rse_expression': 'MOCK|MOCK2', - 'rse_id': self.rse_id, - 'src_rse_id': self.rse_id, - 'source_rse_id': self.rse_id, - 'dest_rse_id': self.rse_id, - 'destination_rse_id': self.rse_id} + 'rse_expression': f'{rse1}|{rse2}', + 'rse_id': rse1_id, + 'src_rse_id': rse1_id, + 'source_rse_id': rse1_id, + 'dest_rse_id': rse1_id, + 'destination_rse_id': rse1_id} value = api_update_return_dict(test_dict) - expected = {'account': account_name, 'scope': scope_name, 'rse_expression': 'MOCK|MOCK2', - 'rse_id': self.rse_id, 'rse': self.rse_name, - 'src_rse_id': self.rse_id, 'src_rse': self.rse_name, - 'source_rse_id': self.rse_id, 'source_rse': self.rse_name, - 'dest_rse_id': self.rse_id, 'dest_rse': self.rse_name, - 'destination_rse_id': self.rse_id, 'destination_rse': self.rse_name} + expected = {'account': account_name, 'scope': scope_name, 'rse_expression': f'{rse1}|{rse2}', + 'rse_id': rse1_id, 'rse': rse1, + 'src_rse_id': rse1_id, 'src_rse': rse1, + 'source_rse_id': rse1_id, 'source_rse': rse1, + 'dest_rse_id': rse1_id, 'dest_rse': rse1, + 'destination_rse_id': rse1_id, 'destination_rse': rse1} assert value == expected def test_api_account(self, vo, vo2, account, account_name): @@ -134,20 +130,22 @@ def test_api_account(self, vo, vo2, account, account_name): assert account.internal not in out assert '@' not in ' '.join(out) - def test_api_account_limit(self, vo, vo2, account_name): + def test_api_account_limit(self, rse_factory, vo, vo2, account_name): """ ACCOUNT_LIMIT (API): Test external representation of account limits """ # Add mock account limits - rse_expr = '{}|{}'.format(self.rse_name, self.rse2_name) - api_acc_lim.set_local_account_limit(account_name, self.rse_name, 10000, issuer='root', vo=vo) + rse1, rse1_id = rse_factory.make_rse() + rse2, rse2_id = rse_factory.make_rse() + rse_expr = f'{rse1}|{rse2}' + api_acc_lim.set_local_account_limit(account_name, rse1, 10000, issuer='root', vo=vo) api_acc_lim.set_global_account_limit(account_name, rse_expr, 20000, issuer='root', vo=vo) out = api_acc_lim.get_local_account_limits(account_name, vo=vo) - assert self.rse_name in out - assert self.rse_id not in out + assert rse1 in out + assert rse1_id not in out - out = api_acc_lim.get_local_account_limit(account_name, self.rse_name, vo=vo) - assert self.rse_name in out - assert self.rse_id not in out + out = api_acc_lim.get_local_account_limit(account_name, rse1, vo=vo) + assert rse1 in out + assert rse1_id not in out out = api_acc_lim.get_global_account_limits(account_name, vo=vo) assert rse_expr in out @@ -159,15 +157,15 @@ def test_api_account_limit(self, vo, vo2, account_name): if vo2: assert 'vo={}&({})'.format(vo, rse_expr) not in out - out = api_acc_lim.get_local_account_usage(account_name, self.rse_name, issuer='root', vo=vo) + out = api_acc_lim.get_local_account_usage(account_name, rse1, issuer='root', vo=vo) out = list(out) assert 0 != len(out) - assert self.rse_id in [usage['rse_id'] for usage in out if 'rse_id' in usage] + assert rse1_id in [usage['rse_id'] for usage in out if 'rse_id' in usage] for usage in out: if 'rse_id' in usage: assert 'rse' in usage - if usage['rse_id'] == self.rse_id: - assert self.rse_name == usage["rse"] + if usage['rse_id'] == rse1_id: + assert rse1 == usage["rse"] out = api_acc_lim.get_global_account_usage(account_name, rse_expr, issuer='root', vo=vo) out = list(out) @@ -205,9 +203,11 @@ def test_api_did(self, vo, account_name, scope_name): for user in out: assert user['user'] == account_name - def test_api_exporter(self, vo, vo2): + def test_api_exporter(self, vo, rse_factory, vo2): """ EXPORTER (API): Test external representation of exported data """ + rse1, rse1_id = rse_factory.make_rse() + rse2, rse2_id = rse_factory.make_rse() out = export_data('root', vo=vo2 if vo2 else vo) rses = out['rses'] assert self.rse3_name in rses @@ -221,15 +221,15 @@ def test_api_exporter(self, vo, vo2): # check for interference from other VOs if vo2: - assert self.rse_name not in rses - assert self.rse_id not in rses - assert self.rse2_name not in rses - assert self.rse2_id not in rses + assert rse1 not in rses + assert rse1_id not in rses + assert rse2 not in rses + assert rse2_id not in rses - assert self.rse_name not in distances - assert self.rse_id not in distances - assert self.rse2_name not in distances - assert self.rse2_id not in distances + assert rse1 not in distances + assert rse1_id not in distances + assert rse2 not in distances + assert rse2_id not in distances def test_api_identity(self, vo, vo2, account, account_name): """ IDENTITY (API): Test external representation of identity accounts """ @@ -243,20 +243,26 @@ def test_api_identity(self, vo, vo2, account, account_name): if vo2: assert account.internal not in out - def test_api_replica(self, vo, vo2, account_name, scope_name, scope): + def test_api_replica(self, vo, rse_factory, vo2, account_name, scope_name, scope): """ REPLICA (API): Test external representation of replicas """ did = did_name_generator('file') did_parent = did_name_generator('dataset') - pfn = 'srm://mock2.com:8443/srm/managerv2?SFN=/rucio/tmpdisk/rucio_tests/%s/%s' % (scope_name, generate_uuid()) - add_replicas(self.rse2_name, files=[{'scope': scope_name, 'name': did, 'bytes': 100, 'pfn': pfn}], issuer='root', vo=vo) + rse2, rse2_id = rse_factory.make_rse(scheme='srm', protocol_impl='rucio.rse.protocols.gfal.Default', deterministic=False) + protocols = api_rse.get_rse_protocols(rse2, issuer='root', vo=vo) + pfn = 'srm://%s:%s/srm/managerv2?SFN=%s%s/%s' % (protocols['protocols'][0]['hostname'], + protocols['protocols'][0]['port'], + protocols['protocols'][0]['prefix'], + scope_name, + generate_uuid()) + add_replicas(rse2, files=[{'scope': scope_name, 'name': did, 'bytes': 100, 'pfn': pfn}], issuer='root', vo=vo) add_did(scope_name, did_parent, 'dataset', issuer='root', account=account_name, vo=vo) attachment = {'scope': scope_name, 'name': did_parent, 'dids': [{'scope': scope_name, 'name': did}]} attach_dids_to_dids([attachment], issuer='root', vo=vo) - out = get_did_from_pfns([pfn], self.rse2_name, vo=vo) + out = get_did_from_pfns([pfn], rse2, vo=vo) out = list(out) assert 0 != len(out) did_found = False @@ -281,15 +287,17 @@ def test_api_replica(self, vo, vo2, account_name, scope_name, scope): assert scope.internal not in parent assert parents_found - def test_api_request(self, vo, account_name, scope_name): + def test_api_request(self, vo, rse_factory, account_name, scope_name): """ REQUEST (API): Test external representation of requests """ + rse1, rse1_id = rse_factory.make_rse() + rse2, rse2_id = rse_factory.make_rse() did = did_name_generator('dataset') - add_did(scope_name, did, 'dataset', issuer='root', account=account_name, rse=self.rse_name, vo=vo) + add_did(scope_name, did, 'dataset', issuer='root', account=account_name, rse=rse1, vo=vo) requests = [{ - 'dest_rse_id': self.rse2_id, - 'source_rse_id': self.rse_id, + 'dest_rse_id': rse2_id, + 'source_rse_id': rse1_id, 'request_type': constants.RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': did, @@ -312,16 +320,16 @@ def test_api_request(self, vo, account_name, scope_name): for r in reqs: assert r['scope'] == scope_name assert r['account'] == account_name - assert r['source_rse'] == self.rse_name - assert r['dest_rse'] == self.rse2_name + assert r['source_rse'] == rse1 + assert r['dest_rse'] == rse2 - out = get_request_by_did(scope_name, did, self.rse2_name, issuer='root', vo=vo) + out = get_request_by_did(scope_name, did, rse2, issuer='root', vo=vo) assert out['scope'] == scope_name assert out['account'] == account_name - assert out['dest_rse'] == self.rse2_name - assert out['source_rse'] == self.rse_name + assert out['dest_rse'] == rse2 + assert out['source_rse'] == rse1 - out = list_requests([self.rse_name], [self.rse2_name], [constants.RequestState.QUEUED], issuer='root', vo=vo) + out = list_requests([rse1], [rse2], [constants.RequestState.QUEUED], issuer='root', vo=vo) out = list(out) assert 0 != len(out) assert scope_name in [req['scope'] for req in out] @@ -329,16 +337,18 @@ def test_api_request(self, vo, account_name, scope_name): if req['scope'] == scope_name: assert req['scope'] == scope_name assert req['account'] == account_name - assert req['dest_rse'] == self.rse2_name - assert req['source_rse'] == self.rse_name + assert req['dest_rse'] == rse2 + assert req['source_rse'] == rse1 @pytest.mark.noparallel(reason='runs the reaper on a pre-defined rse, might interfere with other tests') - def test_api_rse(self, vo, vo2, account, account_name, scope_name): + def test_api_rse(self, vo, rse_factory, vo2, account, account_name, scope_name): """ RSE (API): Test external representation of RSEs """ - out = api_rse.get_rse(self.rse_name, vo=vo) - assert out['rse'] == self.rse_name - assert out['id'] == self.rse_id + rse1, rse1_id = rse_factory.make_rse() + rse2, rse2_id = rse_factory.make_rse() + out = api_rse.get_rse(rse1, vo=vo) + assert out['rse'] == rse1 + assert out['id'] == rse1_id out = api_rse.list_rses(vo=vo2 if vo2 else vo) out = list(out) @@ -354,18 +364,18 @@ def test_api_rse(self, vo, vo2, account, account_name, scope_name): assert rse['rse'] == self.rse4_name key = "KEY_" + generate_uuid() - api_rse.add_rse_attribute(self.rse_name, key, 1, issuer='root', vo=vo) + api_rse.add_rse_attribute(rse1, key, 1, issuer='root', vo=vo) out = api_rse.get_rses_with_attribute(key) out = list(out) assert 0 != len(out) for rse in out: - assert rse['rse'] == self.rse_name + assert rse['rse'] == rse1 - out = api_rse.get_rse_protocols(self.rse_name, issuer='root', vo=vo) - assert out['rse'] == self.rse_name + out = api_rse.get_rse_protocols(rse1, issuer='root', vo=vo) + assert out['rse'] == rse1 # add some account and RSE counters - rse_mock, rse_mock_id = self.rse_factory.make_mock_rse() + rse_mock, rse_mock_id = rse_factory.make_mock_rse() account_counter.del_counter(rse_id=rse_mock_id, account=account) account_counter.add_counter(rse_id=rse_mock_id, account=account) account_counter.increase(rse_id=rse_mock_id, account=account, files=1, bytes_=10) @@ -393,11 +403,11 @@ def test_api_rse(self, vo, vo2, account, account_name, scope_name): reaper.run(once=True, include_rses=rse_mock, greedy=True) abacus_rse.run(once=True) - out = api_rse.parse_rse_expression('%s|%s' % (self.rse_name, self.rse2_name), vo=vo) - assert self.rse_name in out - assert self.rse2_name in out - assert self.rse_id not in out - assert self.rse2_id not in out + out = api_rse.parse_rse_expression(f'{rse1}|{rse2}', vo=vo) + assert rse1 in out + assert rse2 in out + assert rse1_id not in out + assert rse2_id not in out def test_api_scope(self, vo, vo2, account_name, scope_name, scope): """ SCOPE (API): Test external representation of scopes """ diff --git a/tests/test_replica.py b/tests/test_replica.py index bea96cadcb..20485eee1f 100644 --- a/tests/test_replica.py +++ b/tests/test_replica.py @@ -803,11 +803,12 @@ def test_client_add_list_replicas(rse_factory, replica_client, mock_scope): assert rse2 in replicas[i]['rses'] -def test_client_add_replica_scope_not_found(replica_client): +def test_client_add_replica_scope_not_found(rse_factory, replica_client): """ REPLICA (CLIENT): Add replica with missing scope """ + rse, _ = rse_factory.make_mock_rse() files = [{'scope': 'nonexistingscope', 'name': did_name_generator('file'), 'bytes': 1, 'adler32': '0cc737eb'}] with pytest.raises(ScopeNotFound): - replica_client.add_replicas(rse='MOCK', files=files) + replica_client.add_replicas(rse=rse, files=files) def test_client_access_denied_on_delete_replicas(rse_factory, mock_scope, replica_client): @@ -1057,18 +1058,21 @@ class TestReplicaMetalink: @pytest.mark.dirty @pytest.mark.noparallel(reason='uses pre-defined RSE') - def test_client_list_replicas_metalink_4(self, did_client, replica_client): + def test_client_list_replicas_metalink_4(self, rse_factory, did_client, replica_client): """ REPLICA (METALINK): List replicas as metalink version 4 """ fname = did_name_generator('file') - rses = ['MOCK', 'MOCK3', 'MOCK4'] + rses = [] + for _ in range(3): + rse, _ = rse_factory.make_rse(scheme='https', protocol_impl='rucio.rse.protocols.gfal.Default') + rses.append(rse) dsn = did_name_generator('dataset') files = [{'scope': 'mock', 'name': fname, 'bytes': 1, 'adler32': '0cc737eb'}] did_client.add_dataset(scope='mock', name=dsn) - did_client.add_files_to_dataset('mock', name=dsn, files=files, rse='MOCK') - for r in rses: - replica_client.add_replicas(r, files) + did_client.add_files_to_dataset('mock', name=dsn, files=files, rse=rses[0]) + for rse in rses: + replica_client.add_replicas(rse, files) ml = xmltodict.parse(replica_client.list_replicas(files, metalink=4, diff --git a/tests/test_rse.py b/tests/test_rse.py index d6286abe1b..d8a7fe52df 100644 --- a/tests/test_rse.py +++ b/tests/test_rse.py @@ -301,12 +301,11 @@ def test_create_rse_success(vo, rest_client, auth_token): assert response.status_code == 409 -@pytest.mark.dirty -@pytest.mark.noparallel(reason='uses pre-defined RSE') -def xtest_tag_rses(rest_client, auth_token): +def xtest_tag_rses(rse_factory, rest_client, auth_token): """ RSE (REST): send a POST to tag a RSE """ headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'} - data = {'rse': 'MOCK'} + rse, _ = rse_factory.make_rse() + data = {'rse': rse} response = rest_client.post('/rses/', headers=headers(auth(auth_token), hdrdict(headers_dict)), json=data) assert response.status_code == 201 @@ -315,11 +314,10 @@ def xtest_tag_rses(rest_client, auth_token): assert response.status_code == 201 -@pytest.mark.dirty -@pytest.mark.noparallel(reason='uses pre-defined RSE') -def xtest_list_rse_tags(rest_client, auth_token): +def xtest_list_rse_tags(rse_factory, rest_client, auth_token): """ RSE (REST): Test the listing of RSE tags """ headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'} + rse, _ = rse_factory.make_rse() data = {'rse': 'MOCK'} response = rest_client.post('/rses/', headers=headers(auth(auth_token), hdrdict(headers_dict)), json=data) assert response.status_code == 201 @@ -332,12 +330,11 @@ def xtest_list_rse_tags(rest_client, auth_token): assert response.status_code == 200 -@pytest.mark.dirty -@pytest.mark.noparallel(reason='uses pre-defined RSE') -def test_get_rse_account_usage(rest_client, auth_token): +def test_get_rse_account_usage(rse_factory, rest_client, auth_token): """ RSE (REST): Test of RSE account usage and limit """ headers_dict = {'X-Rucio-Type': 'user', 'X-Rucio-Account': 'root'} - response = rest_client.get('/rses/MOCK/accounts/usage', headers=headers(auth(auth_token), hdrdict(headers_dict))) + rse, _ = rse_factory.make_rse() + response = rest_client.get(f'/rses/{rse}/accounts/usage', headers=headers(auth(auth_token), hdrdict(headers_dict))) assert response.status_code == 200 @@ -1406,11 +1403,11 @@ def test_set_rse_usage(self, rucio_client, rse_factory): assert usage['files'] == 50 break - @pytest.mark.noparallel(reason='uses pre-defined RSE') - def test_set_rse_limits(self, rucio_client): + def test_set_rse_limits(self, rse_factory, rucio_client): """ RSE (CLIENTS): Test the update of RSE limits.""" - assert rucio_client.set_rse_limits(rse='MOCK', name='MinFreeSpace', value=1000000) - limits = rucio_client.get_rse_limits(rse='MOCK') + rse, _ = rse_factory.make_posix_rse() + assert rucio_client.set_rse_limits(rse=rse, name='MinFreeSpace', value=1000000) + limits = rucio_client.get_rse_limits(rse=rse) assert limits['MinFreeSpace'] == 1000000 def test_rsemgr_possible_protocols(self): diff --git a/tests/test_undertaker.py b/tests/test_undertaker.py index f7c25afb2f..05fb0bc904 100644 --- a/tests/test_undertaker.py +++ b/tests/test_undertaker.py @@ -23,7 +23,7 @@ from rucio.core.account_limit import set_local_account_limit from rucio.core.did import add_dids, attach_dids, list_expired_dids, get_did, set_metadata from rucio.core.replica import add_replicas, get_replica -from rucio.core.rse import get_rse_id, add_rse +from rucio.core.rse import add_rse from rucio.core.rule import add_rules, list_rules from rucio.daemons.judge.cleaner import rule_cleaner from rucio.daemons.undertaker.undertaker import undertaker @@ -37,14 +37,13 @@ @pytest.mark.noparallel(reason='uses pre-defined rses; runs undertaker, which impacts other tests') class TestUndertaker: - def test_undertaker(self, vo, mock_scope, root_account): + def test_undertaker(self, vo, rse_factory, mock_scope, root_account): """ UNDERTAKER (CORE): Test the undertaker. """ jdoe = InternalAccount('jdoe', vo=vo) nbdatasets = 5 nbfiles = 5 - rse = 'MOCK' - rse_id = get_rse_id('MOCK', vo=vo) + rse, rse_id = rse_factory.make_rse() set_local_account_limit(jdoe, rse_id, -1) @@ -84,19 +83,20 @@ def test_undertaker(self, vo, mock_scope, root_account): for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse_id)['tombstone'] is not None - def test_list_expired_dids_with_locked_rules(self, vo, mock_scope, root_account): + def test_list_expired_dids_with_locked_rules(self, rse_factory, vo, mock_scope, root_account): """ UNDERTAKER (CORE): Test that the undertaker does not list expired dids with locked rules""" jdoe = InternalAccount('jdoe', vo=vo) # Add quota - set_local_account_limit(jdoe, get_rse_id('MOCK', vo=vo), -1) + rse, rse_id = rse_factory.make_rse() + set_local_account_limit(jdoe, rse_id, -1) dsn = {'name': did_name_generator('dataset'), 'scope': mock_scope, 'type': 'DATASET', 'lifetime': -1, 'rules': [{'account': jdoe, 'copies': 1, - 'rse_expression': 'MOCK', 'locked': True, + 'rse_expression': rse, 'locked': True, 'grouping': 'DATASET'}]} add_dids(dids=[dsn], account=root_account) @@ -184,7 +184,7 @@ def test_removal_all_replicas2(rse_factory, root_account, mock_scope, core_confi add_rules(dids=dsns1, rules=[{'account': root_account, 'copies': 1, 'rse_expression': rse1, 'grouping': 'DATASET'}]) add_rules(dids=dsns1, rules=[{'account': root_account, 'copies': 1, 'rse_expression': rse2, 'grouping': 'DATASET', 'lifetime': -86400}]) - # Clean the rules on MOCK2. Replicas are tombstoned with non Epoch + # Clean the rules on rse2. Replicas are tombstoned with non Epoch rule_cleaner(once=True) for replica in replicas: assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse2_id)['tombstone'] is not None