From 0d1d47e1e9e4df7428b76499ba3246f5ac5769f3 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Mon, 9 Dec 2019 12:01:30 +0100 Subject: [PATCH 1/2] Reduce the number of public methods of `Computer` front-end class The `aiida.orm.computers.Computer` front-end class had a mixture of properties and methods to get and set various properties. Which one was available depended on how that property was stored in the backend. Those that are stored as database columns were exposed through property getter and setters, whereas properties that are part of the `metadata` JSONB column had individual methods. Besides being inconsistent, having explicit methods for each property, of which there are already quite a few and which might increase in the future, is inconvenient. It also makes it more difficult to programmatically get all the properties that "define" a computer. By defining all properties in a mapping class attribute, one can define all setters and getters dynamically. --- .../tests/cmdline/commands/test_code.py | 16 +- .../tests/cmdline/commands/test_computer.py | 92 +- .../tests/cmdline/params/types/test_code.py | 2 +- .../cmdline/params/types/test_computer.py | 10 +- .../tests/cmdline/utils/test_common.py | 2 +- aiida/backends/tests/common/test_serialize.py | 5 +- aiida/backends/tests/engine/test_manager.py | 2 +- aiida/backends/tests/orm/test_querybuilder.py | 6 +- aiida/backends/tests/test_nodes.py | 4 +- .../tools/importexport/orm/test_computers.py | 16 +- aiida/cmdline/commands/cmd_calcjob.py | 2 +- aiida/cmdline/commands/cmd_code.py | 2 +- aiida/cmdline/commands/cmd_computer.py | 62 +- aiida/cmdline/utils/common.py | 2 +- aiida/engine/daemon/execmanager.py | 14 +- aiida/engine/processes/calcjobs/calcjob.py | 14 +- aiida/engine/processes/calcjobs/manager.py | 2 +- aiida/orm/authinfos.py | 18 +- aiida/orm/computers.py | 935 +++++++++--------- aiida/orm/implementation/computers.py | 8 +- aiida/orm/implementation/django/authinfos.py | 4 +- .../implementation/sqlalchemy/authinfos.py | 4 +- aiida/orm/nodes/data/code.py | 10 +- aiida/orm/nodes/data/remote.py | 12 +- aiida/orm/utils/builders/computer.py | 37 +- aiida/transports/cli.py | 8 +- 26 files changed, 653 insertions(+), 636 deletions(-) diff --git a/aiida/backends/tests/cmdline/commands/test_code.py b/aiida/backends/tests/cmdline/commands/test_code.py index d28246533c..b56a844360 100644 --- a/aiida/backends/tests/cmdline/commands/test_code.py +++ b/aiida/backends/tests/cmdline/commands/test_code.py @@ -53,10 +53,10 @@ def test_interactive_remote(self): os.environ['EDITOR'] = 'sleep 1; vim -cwq' label = 'interactive_remote' user_input = '\n'.join( - [label, 'description', 'arithmetic.add', 'yes', self.comp.name, '/remote/abs/path']) + [label, 'description', 'arithmetic.add', 'yes', self.comp.label, '/remote/abs/path']) result = self.cli_runner.invoke(setup_code, input=user_input) self.assertClickResultNoException(result) - self.assertIsInstance(Code.get_from_string('{}@{}'.format(label, self.comp.name)), Code) + self.assertIsInstance(Code.get_from_string('{}@{}'.format(label, self.comp.label)), Code) def test_interactive_upload(self): from aiida.orm import Code @@ -74,12 +74,12 @@ def test_noninteractive_remote(self): label = 'noninteractive_remote' options = [ '--non-interactive', '--label={}'.format(label), '--description=description', - '--input-plugin=arithmetic.add', '--on-computer', '--computer={}'.format(self.comp.name), + '--input-plugin=arithmetic.add', '--on-computer', '--computer={}'.format(self.comp.label), '--remote-abs-path=/remote/abs/path' ] result = self.cli_runner.invoke(setup_code, options) self.assertClickResultNoException(result) - self.assertIsInstance(Code.get_from_string('{}@{}'.format(label, self.comp.name)), Code) + self.assertIsInstance(Code.get_from_string('{}@{}'.format(label, self.comp.label)), Code) def test_noninteractive_upload(self): from aiida.orm import Code @@ -107,7 +107,7 @@ def test_from_config(self): input_plugin: arithmetic.add computer: {c} remote_abs_path: /remote/abs/path -""".format(l=label, c=self.comp.name)) +""".format(l=label, c=self.comp.label)) handle.flush() result = self.cli_runner.invoke(setup_code, ['--non-interactive', '--config', os.path.realpath(handle.name)]) @@ -118,10 +118,10 @@ def test_mixed(self): from aiida.orm import Code label = 'mixed_remote' options = ['--description=description', '--on-computer', '--remote-abs-path=/remote/abs/path'] - user_input = '\n'.join([label, 'arithmetic.add', self.comp.name]) + user_input = '\n'.join([label, 'arithmetic.add', self.comp.label]) result = self.cli_runner.invoke(setup_code, options, input=user_input) self.assertClickResultNoException(result) - self.assertIsInstance(Code.get_from_string('{}@{}'.format(label, self.comp.name)), Code) + self.assertIsInstance(Code.get_from_string('{}@{}'.format(label, self.comp.label)), Code) class TestVerdiCodeCommands(AiidaTestCase): @@ -220,7 +220,7 @@ def test_code_list(self): code.store() options = [ - '-A', '-a', '-o', '--input-plugin=arithmetic.add', '--computer={}'.format(self.comp.name) + '-A', '-a', '-o', '--input-plugin=arithmetic.add', '--computer={}'.format(self.comp.label) ] result = self.cli_runner.invoke(code_list, options) self.assertIsNone(result.exception, result.output) diff --git a/aiida/backends/tests/cmdline/commands/test_computer.py b/aiida/backends/tests/cmdline/commands/test_computer.py index 5170f42819..7f41341f84 100644 --- a/aiida/backends/tests/cmdline/commands/test_computer.py +++ b/aiida/backends/tests/cmdline/commands/test_computer.py @@ -17,7 +17,7 @@ from aiida import orm from aiida.backends.testbase import AiidaTestCase -from aiida.cmdline.commands.cmd_computer import computer_disable, computer_enable, computer_setup +from aiida.cmdline.commands.cmd_computer import computer_setup from aiida.cmdline.commands.cmd_computer import computer_show, computer_list, computer_rename, computer_delete from aiida.cmdline.commands.cmd_computer import computer_test, computer_configure, computer_duplicate @@ -132,15 +132,15 @@ def test_interactive(self): self.assertEqual(new_computer.description, options_dict['description']) self.assertEqual(new_computer.hostname, options_dict['hostname']) - self.assertEqual(new_computer.get_transport_type(), options_dict['transport']) - self.assertEqual(new_computer.get_scheduler_type(), options_dict['scheduler']) - self.assertEqual(new_computer.get_mpirun_command(), options_dict['mpirun-command'].split()) - self.assertEqual(new_computer.get_shebang(), options_dict['shebang']) - self.assertEqual(new_computer.get_workdir(), options_dict['work-dir']) - self.assertEqual(new_computer.get_default_mpiprocs_per_machine(), int(options_dict['mpiprocs-per-machine'])) + self.assertEqual(new_computer.transport_type, options_dict['transport']) + self.assertEqual(new_computer.scheduler_type, options_dict['scheduler']) + self.assertEqual(new_computer.get_property('mpirun_command'), options_dict['mpirun-command'].split()) + self.assertEqual(new_computer.get_property('shebang'), options_dict['shebang']) + self.assertEqual(new_computer.get_property('work_dir'), options_dict['work-dir']) + self.assertEqual(new_computer.get_property('default_mpiprocs_per_machine'), int(options_dict['mpiprocs-per-machine'])) # For now I'm not writing anything in them - self.assertEqual(new_computer.get_prepend_text(), '') - self.assertEqual(new_computer.get_append_text(), '') + self.assertEqual(new_computer.get_property('prepend_text'), '') + self.assertEqual(new_computer.get_property('append_text'), '') def test_mixed(self): os.environ['VISUAL'] = 'sleep 1; vim -cwq' @@ -170,16 +170,16 @@ def test_mixed(self): self.assertEqual(new_computer.description, options_dict_full['description']) self.assertEqual(new_computer.hostname, options_dict_full['hostname']) - self.assertEqual(new_computer.get_transport_type(), options_dict_full['transport']) - self.assertEqual(new_computer.get_scheduler_type(), options_dict_full['scheduler']) - self.assertEqual(new_computer.get_mpirun_command(), options_dict_full['mpirun-command'].split()) - self.assertEqual(new_computer.get_shebang(), options_dict_full['shebang']) - self.assertEqual(new_computer.get_workdir(), options_dict_full['work-dir']) - self.assertEqual(new_computer.get_default_mpiprocs_per_machine(), + self.assertEqual(new_computer.transport_type, options_dict_full['transport']) + self.assertEqual(new_computer.scheduler_type, options_dict_full['scheduler']) + self.assertEqual(new_computer.get_property('mpirun_command'), options_dict_full['mpirun-command'].split()) + self.assertEqual(new_computer.get_property('shebang'), options_dict_full['shebang']) + self.assertEqual(new_computer.get_property('work_dir'), options_dict_full['work-dir']) + self.assertEqual(new_computer.get_property('default_mpiprocs_per_machine'), int(options_dict_full['mpiprocs-per-machine'])) # For now I'm not writing anything in them - self.assertEqual(new_computer.get_prepend_text(), options_dict_full['prepend-text']) - self.assertEqual(new_computer.get_append_text(), options_dict_full['append-text']) + self.assertEqual(new_computer.get_property('prepend_text'), options_dict_full['prepend-text']) + self.assertEqual(new_computer.get_property('append_text'), options_dict_full['append-text']) def test_noninteractive(self): """ @@ -196,14 +196,14 @@ def test_noninteractive(self): self.assertEqual(new_computer.description, options_dict['description']) self.assertEqual(new_computer.hostname, options_dict['hostname']) - self.assertEqual(new_computer.get_transport_type(), options_dict['transport']) - self.assertEqual(new_computer.get_scheduler_type(), options_dict['scheduler']) - self.assertEqual(new_computer.get_mpirun_command(), options_dict['mpirun-command'].split()) - self.assertEqual(new_computer.get_shebang(), options_dict['shebang']) - self.assertEqual(new_computer.get_workdir(), options_dict['work-dir']) - self.assertEqual(new_computer.get_default_mpiprocs_per_machine(), int(options_dict['mpiprocs-per-machine'])) - self.assertEqual(new_computer.get_prepend_text(), options_dict['prepend-text']) - self.assertEqual(new_computer.get_append_text(), options_dict['append-text']) + self.assertEqual(new_computer.transport_type, options_dict['transport']) + self.assertEqual(new_computer.scheduler_type, options_dict['scheduler']) + self.assertEqual(new_computer.get_property('mpirun_command'), options_dict['mpirun-command'].split()) + self.assertEqual(new_computer.get_property('shebang'), options_dict['shebang']) + self.assertEqual(new_computer.get_property('work_dir'), options_dict['work-dir']) + self.assertEqual(new_computer.get_property('default_mpiprocs_per_machine'), int(options_dict['mpiprocs-per-machine'])) + self.assertEqual(new_computer.get_property('prepend_text'), options_dict['prepend-text']) + self.assertEqual(new_computer.get_property('append_text'), options_dict['append-text']) # Test that I cannot generate twice a computer with the same label result = self.cli_runner.invoke(computer_setup, options) @@ -223,7 +223,7 @@ def test_noninteractive_optional_default_mpiprocs(self): new_computer = orm.Computer.objects.get(name=options_dict['label']) self.assertIsInstance(new_computer, orm.Computer) - self.assertIsNone(new_computer.get_default_mpiprocs_per_machine()) + self.assertIsNone(new_computer.get_property('default_mpiprocs_per_machine')) def test_noninteractive_optional_default_mpiprocs_2(self): """ @@ -238,7 +238,7 @@ def test_noninteractive_optional_default_mpiprocs_2(self): new_computer = orm.Computer.objects.get(name=options_dict['label']) self.assertIsInstance(new_computer, orm.Computer) - self.assertIsNone(new_computer.get_default_mpiprocs_per_machine()) + self.assertIsNone(new_computer.get_property('default_mpiprocs_per_machine')) def test_noninteractive_optional_default_mpiprocs_3(self): """ @@ -500,7 +500,7 @@ def setUpClass(cls, *args, **kwargs): transport_type='local', scheduler_type='direct', workdir='/tmp/aiida') - cls.comp.set_default_mpiprocs_per_machine(1) + cls.comp.set_property('default_mpiprocs_per_machine', 1) cls.comp.store() def setUp(self): @@ -656,15 +656,15 @@ def test_computer_duplicate_interactive(self): new_computer = orm.Computer.objects.get(name=label) self.assertEqual(self.comp.description, new_computer.description) - self.assertEqual(self.comp.get_hostname(), new_computer.get_hostname()) - self.assertEqual(self.comp.get_transport_type(), new_computer.get_transport_type()) - self.assertEqual(self.comp.get_scheduler_type(), new_computer.get_scheduler_type()) - self.assertEqual(self.comp.get_shebang(), new_computer.get_shebang()) - self.assertEqual(self.comp.get_workdir(), new_computer.get_workdir()) - self.assertEqual(self.comp.get_mpirun_command(), new_computer.get_mpirun_command()) - self.assertEqual(self.comp.get_default_mpiprocs_per_machine(), new_computer.get_default_mpiprocs_per_machine()) - self.assertEqual(self.comp.get_prepend_text(), new_computer.get_prepend_text()) - self.assertEqual(self.comp.get_append_text(), new_computer.get_append_text()) + self.assertEqual(self.comp.hostname, new_computer.hostname) + self.assertEqual(self.comp.transport_type, new_computer.transport_type) + self.assertEqual(self.comp.scheduler_type, new_computer.scheduler_type) + self.assertEqual(self.comp.get_property('shebang'), new_computer.get_property('shebang')) + self.assertEqual(self.comp.get_property('work_dir'), new_computer.get_property('work_dir')) + self.assertEqual(self.comp.get_property('mpirun_command'), new_computer.get_property('mpirun_command')) + self.assertEqual(self.comp.get_property('default_mpiprocs_per_machine'), new_computer.get_property('default_mpiprocs_per_machine')) + self.assertEqual(self.comp.get_property('prepend_text'), new_computer.get_property('prepend_text')) + self.assertEqual(self.comp.get_property('append_text'), new_computer.get_property('append_text')) def test_computer_duplicate_non_interactive(self): label = 'computer_duplicate_noninteractive' @@ -674,12 +674,12 @@ def test_computer_duplicate_non_interactive(self): new_computer = orm.Computer.objects.get(name=label) self.assertEqual(self.comp.description, new_computer.description) - self.assertEqual(self.comp.get_hostname(), new_computer.get_hostname()) - self.assertEqual(self.comp.get_transport_type(), new_computer.get_transport_type()) - self.assertEqual(self.comp.get_scheduler_type(), new_computer.get_scheduler_type()) - self.assertEqual(self.comp.get_shebang(), new_computer.get_shebang()) - self.assertEqual(self.comp.get_workdir(), new_computer.get_workdir()) - self.assertEqual(self.comp.get_mpirun_command(), new_computer.get_mpirun_command()) - self.assertEqual(self.comp.get_default_mpiprocs_per_machine(), new_computer.get_default_mpiprocs_per_machine()) - self.assertEqual(self.comp.get_prepend_text(), new_computer.get_prepend_text()) - self.assertEqual(self.comp.get_append_text(), new_computer.get_append_text()) + self.assertEqual(self.comp.hostname, new_computer.hostname) + self.assertEqual(self.comp.transport_type, new_computer.transport_type) + self.assertEqual(self.comp.scheduler_type, new_computer.scheduler_type) + self.assertEqual(self.comp.get_property('shebang'), new_computer.get_property('shebang')) + self.assertEqual(self.comp.get_property('work_dir'), new_computer.get_property('work_dir')) + self.assertEqual(self.comp.get_property('mpirun_command'), new_computer.get_property('mpirun_command')) + self.assertEqual(self.comp.get_property('default_mpiprocs_per_machine'), new_computer.get_property('default_mpiprocs_per_machine')) + self.assertEqual(self.comp.get_property('prepend_text'), new_computer.get_property('prepend_text')) + self.assertEqual(self.comp.get_property('append_text'), new_computer.get_property('append_text')) diff --git a/aiida/backends/tests/cmdline/params/types/test_code.py b/aiida/backends/tests/cmdline/params/types/test_code.py index f96d9ecf05..8a13644d4e 100644 --- a/aiida/backends/tests/cmdline/params/types/test_code.py +++ b/aiida/backends/tests/cmdline/params/types/test_code.py @@ -70,7 +70,7 @@ def test_get_by_fullname(self): """ Verify that using the LABEL@machinename will retrieve the correct entity """ - identifier = '{}@{}'.format(self.entity_01.label, self.computer.name) # pylint: disable=no-member + identifier = '{}@{}'.format(self.entity_01.label, self.computer.label) # pylint: disable=no-member result = self.param_base.convert(identifier, None, None) self.assertEqual(result.uuid, self.entity_01.uuid) diff --git a/aiida/backends/tests/cmdline/params/types/test_computer.py b/aiida/backends/tests/cmdline/params/types/test_computer.py index b6523a52c8..f6fb7e5955 100644 --- a/aiida/backends/tests/cmdline/params/types/test_computer.py +++ b/aiida/backends/tests/cmdline/params/types/test_computer.py @@ -60,7 +60,7 @@ def test_get_by_label(self): """ Verify that using the LABEL will retrieve the correct entity """ - identifier = '{}'.format(self.entity_01.name) + identifier = '{}'.format(self.entity_01.label) result = self.param.convert(identifier, None, None) self.assertEqual(result.uuid, self.entity_01.uuid) @@ -71,11 +71,11 @@ def test_ambiguous_label_pk(self): Verify that using an ambiguous identifier gives precedence to the ID interpretation Appending the special ambiguity breaker character will force the identifier to be treated as a LABEL """ - identifier = '{}'.format(self.entity_02.name) + identifier = '{}'.format(self.entity_02.label) result = self.param.convert(identifier, None, None) self.assertEqual(result.uuid, self.entity_01.uuid) - identifier = '{}{}'.format(self.entity_02.name, OrmEntityLoader.label_ambiguity_breaker) + identifier = '{}{}'.format(self.entity_02.label, OrmEntityLoader.label_ambiguity_breaker) result = self.param.convert(identifier, None, None) self.assertEqual(result.uuid, self.entity_02.uuid) @@ -86,10 +86,10 @@ def test_ambiguous_label_uuid(self): Verify that using an ambiguous identifier gives precedence to the UUID interpretation Appending the special ambiguity breaker character will force the identifier to be treated as a LABEL """ - identifier = '{}'.format(self.entity_03.name) + identifier = '{}'.format(self.entity_03.label) result = self.param.convert(identifier, None, None) self.assertEqual(result.uuid, self.entity_01.uuid) - identifier = '{}{}'.format(self.entity_03.name, OrmEntityLoader.label_ambiguity_breaker) + identifier = '{}{}'.format(self.entity_03.label, OrmEntityLoader.label_ambiguity_breaker) result = self.param.convert(identifier, None, None) self.assertEqual(result.uuid, self.entity_03.uuid) diff --git a/aiida/backends/tests/cmdline/utils/test_common.py b/aiida/backends/tests/cmdline/utils/test_common.py index dea9f7ac1c..e306664bbd 100644 --- a/aiida/backends/tests/cmdline/utils/test_common.py +++ b/aiida/backends/tests/cmdline/utils/test_common.py @@ -21,7 +21,7 @@ def test_get_node_summary(self): """Test the `get_node_summary` utility.""" from aiida.cmdline.utils.common import get_node_summary - computer_label = self.computer.name # pylint: disable=no-member + computer_label = self.computer.label # pylint: disable=no-member code = orm.Code( input_plugin_name='arithmetic.add', diff --git a/aiida/backends/tests/common/test_serialize.py b/aiida/backends/tests/common/test_serialize.py index 1a83dea592..0ab689264c 100644 --- a/aiida/backends/tests/common/test_serialize.py +++ b/aiida/backends/tests/common/test_serialize.py @@ -73,9 +73,8 @@ def test_serialize_computer_round_trip(self): computer = self.computer deserialized = serialize.deserialize(serialize.serialize(computer)) - # pylint: disable=no-member - self.assertEqual(computer.uuid, deserialized.uuid) - self.assertEqual(computer.name, deserialized.name) + self.assertEqual(computer.uuid, deserialized.uuid) # pylint: disable=no-member + self.assertEqual(computer.label, deserialized.label) # pylint: disable=no-member def test_serialize_unstored_node(self): """Test that you can't serialize an unstored node""" diff --git a/aiida/backends/tests/engine/test_manager.py b/aiida/backends/tests/engine/test_manager.py index 4e2748e901..f55e46e0fe 100644 --- a/aiida/backends/tests/engine/test_manager.py +++ b/aiida/backends/tests/engine/test_manager.py @@ -65,7 +65,7 @@ def tearDown(self): def test_get_minimum_update_interval(self): """Test the `JobsList.get_minimum_update_interval` method.""" - minimum_poll_interval = self.auth_info.computer.get_minimum_job_poll_interval() + minimum_poll_interval = self.auth_info.computer.get_property('minimum_job_poll_interval') self.assertEqual(self.jobs_list.get_minimum_update_interval(), minimum_poll_interval) def test_last_updated(self): diff --git a/aiida/backends/tests/orm/test_querybuilder.py b/aiida/backends/tests/orm/test_querybuilder.py index 3c7d12c7a5..ec0b190398 100644 --- a/aiida/backends/tests/orm/test_querybuilder.py +++ b/aiida/backends/tests/orm/test_querybuilder.py @@ -1355,11 +1355,11 @@ def test_statistics_default_class(self): # pylint: disable=no-member expected_dict = { 'description': self.computer.description, - 'scheduler_type': self.computer.get_scheduler_type(), + 'scheduler_type': self.computer.scheduler_type, 'hostname': self.computer.hostname, 'uuid': self.computer.uuid, - 'name': self.computer.name, - 'transport_type': self.computer.get_transport_type(), + 'name': self.computer.label, + 'transport_type': self.computer.transport_type, 'id': self.computer.id, 'metadata': self.computer.get_metadata(), } diff --git a/aiida/backends/tests/test_nodes.py b/aiida/backends/tests/test_nodes.py index 742beef3f4..6526f49fc8 100644 --- a/aiida/backends/tests/test_nodes.py +++ b/aiida/backends/tests/test_nodes.py @@ -1034,7 +1034,7 @@ def test_code_loading_from_string(self): self.assertEquals(q_code_1.get_remote_exec_path(), code1.get_remote_exec_path()) # Test that the code2 can be loaded correctly with its label - q_code_2 = orm.Code.get_from_string(code2.label + '@' + self.computer.get_name()) + q_code_2 = orm.Code.get_from_string(code2.label + '@' + self.computer.label) self.assertEquals(q_code_2.id, code2.id) self.assertEquals(q_code_2.label, code2.label) self.assertEquals(q_code_2.get_remote_exec_path(), code2.get_remote_exec_path()) @@ -1087,7 +1087,7 @@ def test_code_loading_using_get(self): self.assertEquals(q_code_1.get_remote_exec_path(), code1.get_remote_exec_path()) # Test that the code2 can be loaded correctly with its label and computername - q_code_2 = orm.Code.get(label=code2.label, machinename=self.computer.get_name()) + q_code_2 = orm.Code.get(label=code2.label, machinename=self.computer.label) self.assertEquals(q_code_2.id, code2.id) self.assertEquals(q_code_2.label, code2.label) self.assertEquals(q_code_2.get_remote_exec_path(), code2.get_remote_exec_path()) diff --git a/aiida/backends/tests/tools/importexport/orm/test_computers.py b/aiida/backends/tests/tools/importexport/orm/test_computers.py index 80613acb8c..5f20565927 100644 --- a/aiida/backends/tests/tools/importexport/orm/test_computers.py +++ b/aiida/backends/tests/tools/importexport/orm/test_computers.py @@ -59,7 +59,7 @@ def test_same_computer_import(self, temp_dir): calc2.seal() # Store locally the computer name - comp_name = str(comp.name) + comp_name = str(comp.label) comp_uuid = str(comp.uuid) # Export the first job calculation @@ -148,14 +148,14 @@ def test_same_computer_different_name_import(self, temp_dir): calc1.seal() # Store locally the computer name - comp1_name = str(comp1.name) + comp1_name = str(comp1.label) # Export the first job calculation filename1 = os.path.join(temp_dir, 'export1.tar.gz') export([calc1], outfile=filename1, silent=True) # Rename the computer - comp1.set_name(comp1_name + '_updated') + comp1.label = comp1_name + '_updated' # Store a second calculation calc2_label = 'calc2' @@ -222,7 +222,7 @@ def test_different_computer_same_name_import(self, temp_dir): # Set the computer name comp1_name = 'localhost_1' - self.computer.set_name(comp1_name) + self.computer.label = comp1_name # Store a calculation calc1_label = 'calc1' @@ -242,7 +242,7 @@ def test_different_computer_same_name_import(self, temp_dir): self.insert_data() # Set the computer name to the same name as before - self.computer.set_name(comp1_name) + self.computer.label = comp1_name # Store a second calculation calc2_label = 'calc2' @@ -262,7 +262,7 @@ def test_different_computer_same_name_import(self, temp_dir): self.insert_data() # Set the computer name to the same name as before - self.computer.set_name(comp1_name) + self.computer.label = comp1_name # Store a third calculation calc3_label = 'calc3' @@ -318,7 +318,7 @@ def test_import_of_computer_json_params(self, temp_dir): # Set the computer name comp1_name = 'localhost_1' comp1_metadata = {'workdir': '/tmp/aiida'} - self.computer.set_name(comp1_name) + self.computer.label = comp1_name self.computer.set_metadata(comp1_metadata) # Store a calculation @@ -367,7 +367,7 @@ def test_import_of_django_sqla_export_file(self): builder = orm.QueryBuilder() builder.append( orm.Computer, project=['metadata'], tag='comp', filters={'name': { - '!==': self.computer.name + '!==': self.computer.label }} ) self.assertEqual(builder.count(), 1, 'Expected only one computer') diff --git a/aiida/cmdline/commands/cmd_calcjob.py b/aiida/cmdline/commands/cmd_calcjob.py index 2dd1043867..028f4b544c 100644 --- a/aiida/cmdline/commands/cmd_calcjob.py +++ b/aiida/cmdline/commands/cmd_calcjob.py @@ -257,4 +257,4 @@ def calcjob_cleanworkdir(calcjobs, past_days, older_than, computers, force): clean_remote(transport, path) counter += 1 - echo.echo_success('{} remote folders cleaned on {}'.format(counter, computer.name)) + echo.echo_success('{} remote folders cleaned on {}'.format(counter, computer.label)) diff --git a/aiida/cmdline/commands/cmd_code.py b/aiida/cmdline/commands/cmd_code.py index 1c2d6eef73..d7f5d112c6 100644 --- a/aiida/cmdline/commands/cmd_code.py +++ b/aiida/cmdline/commands/cmd_code.py @@ -245,7 +245,7 @@ def code_list(computer, input_plugin, all_entries, all_users, show_owner): qb_computer_filters = dict() if computer is not None: - qb_computer_filters['name'] = computer.name + qb_computer_filters['name'] = computer.lbael qb_code_filters = dict() if input_plugin is not None: diff --git a/aiida/cmdline/commands/cmd_computer.py b/aiida/cmdline/commands/cmd_computer.py index 397dc93659..2659d7845f 100644 --- a/aiida/cmdline/commands/cmd_computer.py +++ b/aiida/cmdline/commands/cmd_computer.py @@ -138,7 +138,7 @@ def _computer_create_temp_file(transport, scheduler, authinfo): # pylint: disab import os file_content = "Test from 'verdi computer test' on {}".format(datetime.datetime.now().isoformat()) - workdir = authinfo.get_workdir().format(username=transport.whoami()) + workdir = authinfo.get_property('work_dir').format(username=transport.whoami()) try: transport.chdir(workdir) @@ -246,8 +246,8 @@ def computer_setup(ctx, non_interactive, **kwargs): kwargs['prepend_text'] = pre kwargs['append_text'] = post - kwargs['transport'] = kwargs['transport'].name - kwargs['scheduler'] = kwargs['scheduler'].name + kwargs['transport'] = kwargs.pop('transport').name + kwargs['scheduler'] = kwargs.pop('scheduler').name computer_builder = ComputerBuilder(**kwargs) try: @@ -260,10 +260,10 @@ def computer_setup(ctx, non_interactive, **kwargs): except ValidationError as err: echo.echo_critical('unable to store the computer: {}. Exiting...'.format(err)) else: - echo.echo_success('Computer<{}> {} created'.format(computer.pk, computer.name)) + echo.echo_success('Computer<{}> {} created'.format(computer.pk, computer.label)) echo.echo_info('Note: before the computer can be used, it has to be configured with the command:') - echo.echo_info(' verdi computer configure {} {}'.format(computer.get_transport_type(), computer.name)) + echo.echo_info(' verdi computer configure {} {}'.format(computer.transport_type, computer.label)) @verdi_computer.command('duplicate') @@ -299,8 +299,8 @@ def computer_duplicate(ctx, computer, non_interactive, **kwargs): kwargs['prepend_text'] = pre kwargs['append_text'] = post - kwargs['transport'] = kwargs['transport'].name - kwargs['scheduler'] = kwargs['scheduler'].name + kwargs['transport'] = kwargs.pop('transport').name + kwargs['scheduler'] = kwargs.pop('scheduler').name computer_builder = ctx.computer_builder for key, value in kwargs.items(): @@ -312,20 +312,20 @@ def computer_duplicate(ctx, computer, non_interactive, **kwargs): except (ComputerBuilder.ComputerValidationError, ValidationError) as e: echo.echo_critical('{}: {}'.format(type(e).__name__, e)) else: - echo.echo_success('stored computer {}<{}>'.format(computer.name, computer.pk)) + echo.echo_success('stored computer {}<{}>'.format(computer.label, computer.pk)) try: computer.store() except ValidationError as err: echo.echo_critical('unable to store the computer: {}. Exiting...'.format(err)) else: - echo.echo_success('Computer<{}> {} created'.format(computer.pk, computer.name)) + echo.echo_success('Computer<{}> {} created'.format(computer.pk, computer.label)) is_configured = computer.is_user_configured(orm.User.objects.get_default()) if not is_configured: echo.echo_info('Note: before the computer can be used, it has to be configured with the command:') - echo.echo_info(' verdi computer configure {} {}'.format(computer.get_transport_type(), computer.name)) + echo.echo_info(' verdi computer configure {} {}'.format(computer.transport_type, computer.label)) @verdi_computer.command('enable') @@ -340,15 +340,15 @@ def computer_enable(computer, user): authinfo = computer.get_authinfo(user) except NotExistent: echo.echo_critical( - "User with email '{}' is not configured for computer '{}' yet.".format(user.email, computer.name) + "User with email '{}' is not configured for computer '{}' yet.".format(user.email, computer.label) ) if not authinfo.enabled: authinfo.enabled = True - echo.echo_info("Computer '{}' enabled for user {}.".format(computer.name, user.get_full_name())) + echo.echo_info("Computer '{}' enabled for user {}.".format(computer.label, user.get_full_name())) else: echo.echo_info( - "Computer '{}' was already enabled for user {} {}.".format(computer.name, user.first_name, user.last_name) + "Computer '{}' was already enabled for user {} {}.".format(computer.label, user.first_name, user.last_name) ) @@ -366,15 +366,17 @@ def computer_disable(computer, user): authinfo = computer.get_authinfo(user) except NotExistent: echo.echo_critical( - "User with email '{}' is not configured for computer '{}' yet.".format(user.email, computer.name) + "User with email '{}' is not configured for computer '{}' yet.".format(user.email, computer.label) ) if authinfo.enabled: authinfo.enabled = False - echo.echo_info("Computer '{}' disabled for user {}.".format(computer.name, user.get_full_name())) + echo.echo_info("Computer '{}' disabled for user {}.".format(computer.label, user.get_full_name())) else: echo.echo_info( - "Computer '{}' was already disabled for user {} {}.".format(computer.name, user.first_name, user.last_name) + "Computer '{}' was already disabled for user {} {}.".format( + computer.label, user.first_name, user.last_name + ) ) @@ -396,10 +398,10 @@ def computer_list(all_entries, raw): if not computers: echo.echo_info("No computers configured yet. Use 'verdi computer setup'") - sort = lambda computer: computer.name + sort = lambda computer: computer.label highlight = lambda comp: comp.is_user_configured(user) and comp.is_user_enabled(user) hide = lambda comp: not (comp.is_user_configured(user) and comp.is_user_enabled(user)) and not all_entries - echo.echo_formatted_list(computers, ['name'], sort=sort, highlight=highlight, hide=hide) + echo.echo_formatted_list(computers, ['label'], sort=sort, highlight=highlight, hide=hide) @verdi_computer.command('show') @@ -416,25 +418,15 @@ def computer_show(computer): @with_dbenv() def computer_rename(computer, new_name): """Rename a computer.""" - from aiida.common.exceptions import UniquenessError - - old_name = computer.get_name() + old_name = computer.label if old_name == new_name: echo.echo_critical('The old and new names are the same.') try: - computer.set_name(new_name) - computer.store() + computer.label = new_name except ValidationError as error: echo.echo_critical('Invalid input! {}'.format(error)) - except UniquenessError as error: - echo.echo_critical( - 'Uniqueness error encountered! Probably a ' - "computer with name '{}' already exists" - ''.format(new_name) - ) - echo.echo_critical('(Message was: {})'.format(error)) echo.echo_success("Computer '{}' renamed to '{}'".format(old_name, new_name)) @@ -468,15 +460,15 @@ def computer_test(user, print_traceback, computer): if user is None: user = orm.User.objects.get_default() - echo.echo_info('Testing computer<{}> for user<{}>...'.format(computer.name, user.email)) + echo.echo_info('Testing computer<{}> for user<{}>...'.format(computer.label, user.email)) try: authinfo = computer.get_authinfo(user) except NotExistent: - echo.echo_critical('Computer<{}> is not yet configured for user<{}>'.format(computer.name, user.email)) + echo.echo_critical('Computer<{}> is not yet configured for user<{}>'.format(computer.label, user.email)) if not authinfo.enabled: - echo.echo_warning('Computer<{}> is disabled for user<{}>'.format(computer.name, user.email)) + echo.echo_warning('Computer<{}> is disabled for user<{}>'.format(computer.label, user.email)) click.confirm('Do you really want to test it?', abort=True) scheduler = authinfo.computer.get_scheduler() @@ -564,7 +556,7 @@ def computer_delete(computer): from aiida.common.exceptions import InvalidOperation from aiida import orm - compname = computer.name + compname = computer.label try: orm.Computer.objects.delete(computer.id) @@ -595,7 +587,7 @@ def computer_config_show(computer, user, defaults, as_option_string): transport_cls = computer.get_transport_class() option_list = [ - param for param in transport_cli.create_configure_cmd(computer.get_transport_type()).params + param for param in transport_cli.create_configure_cmd(computer.transport_type).params if isinstance(param, click.core.Option) ] option_list = [option for option in option_list if option.name in transport_cls.get_valid_auth_params()] diff --git a/aiida/cmdline/utils/common.py b/aiida/cmdline/utils/common.py index 6fc7f20c05..f94c6b0549 100644 --- a/aiida/cmdline/utils/common.py +++ b/aiida/cmdline/utils/common.py @@ -123,7 +123,7 @@ def get_node_summary(node): pass else: if computer is not None: - table.append(['computer', '[{}] {}'.format(node.computer.pk, node.computer.name)]) + table.append(['computer', '[{}] {}'.format(node.computer.pk, node.computer.label)]) return tabulate(table, headers=table_headers) diff --git a/aiida/engine/daemon/execmanager.py b/aiida/engine/daemon/execmanager.py index e23edc49e4..3a678c6257 100644 --- a/aiida/engine/daemon/execmanager.py +++ b/aiida/engine/daemon/execmanager.py @@ -74,11 +74,11 @@ def upload_calculation(node, transport, calc_info, script_filename, inputs=None, remote_user = transport.whoami() # TODO Doc: {username} field # TODO: if something is changed here, fix also 'verdi computer test' - remote_working_directory = computer.get_workdir().format(username=remote_user) + remote_working_directory = computer.get_property('work_dir').format(username=remote_user) if not remote_working_directory.strip(): raise exceptions.ConfigurationError( "[submission of calculation {}] No remote_working_directory configured for computer '{}'".format( - node.pk, computer.name)) + node.pk, computer.label)) # If it already exists, no exception is raised try: @@ -95,7 +95,7 @@ def upload_calculation(node, transport, calc_info, script_filename, inputs=None, '[submission of calculation {}] ' 'Unable to create the remote directory {} on ' "computer '{}': {}".format( - node.pk, remote_working_directory, computer.name, exc)) + node.pk, remote_working_directory, computer.label, exc)) # Store remotely with sharding (here is where we choose # the folder structure of remote jobs; then I store this # in the calculation properties using _set_remote_dir @@ -202,20 +202,20 @@ def find_data_node(inputs, uuid): with open(os.path.join(workdir, '_aiida_remote_copy_list.txt'), 'w') as handle: for remote_computer_uuid, remote_abs_path, dest_rel_path in remote_copy_list: handle.write('would have copied {} to {} in working directory on remote {}'.format( - remote_abs_path, dest_rel_path, computer.name)) + remote_abs_path, dest_rel_path, computer.label)) if remote_symlink_list: with open(os.path.join(workdir, '_aiida_remote_symlink_list.txt'), 'w') as handle: for remote_computer_uuid, remote_abs_path, dest_rel_path in remote_symlink_list: handle.write('would have created symlinks from {} to {} in working directory on remote {}'.format( - remote_abs_path, dest_rel_path, computer.name)) + remote_abs_path, dest_rel_path, computer.label)) else: for (remote_computer_uuid, remote_abs_path, dest_rel_path) in remote_copy_list: if remote_computer_uuid == computer.uuid: logger.debug('[submission of calculation {}] copying {} remotely, directly on the machine {}'.format( - node.pk, dest_rel_path, computer.name)) + node.pk, dest_rel_path, computer.label)) try: transport.copy(remote_abs_path, dest_rel_path) except (IOError, OSError): @@ -230,7 +230,7 @@ def find_data_node(inputs, uuid): for (remote_computer_uuid, remote_abs_path, dest_rel_path) in remote_symlink_list: if remote_computer_uuid == computer.uuid: logger.debug('[submission of calculation {}] copying {} remotely, directly on the machine {}'.format( - node.pk, dest_rel_path, computer.name)) + node.pk, dest_rel_path, computer.label)) try: transport.symlink(remote_abs_path, dest_rel_path) except (IOError, OSError): diff --git a/aiida/engine/processes/calcjobs/calcjob.py b/aiida/engine/processes/calcjobs/calcjob.py index 469e3121c5..32be5aaef4 100644 --- a/aiida/engine/processes/calcjobs/calcjob.py +++ b/aiida/engine/processes/calcjobs/calcjob.py @@ -74,7 +74,7 @@ def validate_calc_job(inputs): raise exceptions.InputValidationError('invalid parser specified: {}'.format(exception)) scheduler = computer.get_scheduler() # pylint: disable=no-member - def_cpus_machine = computer.get_default_mpiprocs_per_machine() # pylint: disable=no-member + def_cpus_machine = computer.get_property('default_mpiprocs_per_machine') # pylint: disable=no-member if def_cpus_machine is not None: resources['default_mpiprocs_per_machine'] = def_cpus_machine @@ -244,7 +244,7 @@ def run(self): if not code.can_run_on(computer): raise InputValidationError( 'The selected code {} for calculation {} cannot run on computer {}'.format( - code.pk, self.node.pk, computer.name)) + code.pk, self.node.pk, computer.label)) # After this call, no modifications to the folder should be done self.node.put_object_from_tree(folder.abspath, force=True) @@ -328,7 +328,7 @@ def presubmit(self, folder): # I create the job template to pass to the scheduler job_tmpl = JobTemplate() - job_tmpl.shebang = computer.get_shebang() + job_tmpl.shebang = computer.get_property('shebang') job_tmpl.submit_as_hold = False job_tmpl.rerunnable = False job_tmpl.job_environment = {} @@ -373,19 +373,19 @@ def presubmit(self, folder): # - most importantly, skips the cases in which one of the methods # would return None, in which case the join method would raise # an exception - prepend_texts = [computer.get_prepend_text()] + \ + prepend_texts = [computer.get_property('prepend_text')] + \ [code.get_prepend_text() for code in codes] + \ [calcinfo.prepend_text, self.node.get_option('prepend_text')] job_tmpl.prepend_text = '\n\n'.join(prepend_text for prepend_text in prepend_texts if prepend_text) append_texts = [self.node.get_option('append_text'), calcinfo.append_text] + \ [code.get_append_text() for code in codes] + \ - [computer.get_append_text()] + [computer.get_property('append_text')] job_tmpl.append_text = '\n\n'.join(append_text for append_text in append_texts if append_text) # Set resources, also with get_default_mpiprocs_per_machine resources = self.node.get_option('resources') - def_cpus_machine = computer.get_default_mpiprocs_per_machine() + def_cpus_machine = computer.get_property('default_mpiprocs_per_machine') if def_cpus_machine is not None: resources['default_mpiprocs_per_machine'] = def_cpus_machine job_tmpl.job_resource = scheduler.create_job_resource(**resources) @@ -394,7 +394,7 @@ def presubmit(self, folder): for key, value in job_tmpl.job_resource.items(): subst_dict[key] = value - mpi_args = [arg.format(**subst_dict) for arg in computer.get_mpirun_command()] + mpi_args = [arg.format(**subst_dict) for arg in computer.get_property('mpirun_command')] extra_mpirun_params = self.node.get_option('mpirun_extra_params') # same for all codes in the same calc # set the codes_info diff --git a/aiida/engine/processes/calcjobs/manager.py b/aiida/engine/processes/calcjobs/manager.py index 68b55518db..28ab8ac21f 100644 --- a/aiida/engine/processes/calcjobs/manager.py +++ b/aiida/engine/processes/calcjobs/manager.py @@ -73,7 +73,7 @@ def get_minimum_update_interval(self): :return: the minimum interval :rtype: float """ - return self._authinfo.computer.get_minimum_job_poll_interval() + return self._authinfo.computer.get_property('minimum_job_poll_interval') @property def last_updated(self): diff --git a/aiida/orm/authinfos.py b/aiida/orm/authinfos.py index d1bf0e8dcb..0448d3250d 100644 --- a/aiida/orm/authinfos.py +++ b/aiida/orm/authinfos.py @@ -9,8 +9,6 @@ ########################################################################### """Module for the `AuthInfo` ORM class.""" -from aiida.common import exceptions -from aiida.plugins import TransportFactory from aiida.manage.manager import get_manager from . import entities from . import users @@ -50,9 +48,9 @@ def __init__(self, computer, user, backend=None): def __str__(self): if self.enabled: - return 'AuthInfo for {} on {}'.format(self.user.email, self.computer.name) + return 'AuthInfo for {} on {}'.format(self.user.email, self.computer.label) - return 'AuthInfo for {} on {} [DISABLED]'.format(self.user.email, self.computer.name) + return 'AuthInfo for {} on {} [DISABLED]'.format(self.user.email, self.computer.label) @property def enabled(self): @@ -130,7 +128,7 @@ def get_workdir(self): try: return self.get_metadata()[self.PROPERTY_WORKDIR] except KeyError: - return self.computer.get_workdir() + return self.computer.get_property('work_dir') def get_transport(self): """Return a fully configured transport that can be used to connect to the computer set for this instance. @@ -138,13 +136,5 @@ def get_transport(self): :rtype: :class:`aiida.transports.Transport` """ computer = self.computer - transport_type = computer.get_transport_type() - - try: - transport_class = TransportFactory(transport_type) - except exceptions.EntryPointError as exception: - raise exceptions.ConfigurationError( - 'transport type `{}` could not be loaded: {}'.format(transport_type, exception) - ) - + transport_class = computer.get_transport_class() return transport_class(machine=computer.hostname, **self.get_auth_params()) diff --git a/aiida/orm/computers.py b/aiida/orm/computers.py index 2832733fa0..c1d909e771 100644 --- a/aiida/orm/computers.py +++ b/aiida/orm/computers.py @@ -8,7 +8,6 @@ # For further information please visit http://www.aiida.net # ########################################################################### """Module for Computer entities""" - import logging import os import warnings @@ -25,28 +24,94 @@ __all__ = ('Computer',) -class Computer(entities.Entity): - """ - Base class to map a node in the DB + its permanent repository counterpart. +def validate_work_dir(_, value): + """Validate the transport string.""" + if not value.strip(): + raise exceptions.ValidationError('No work_dir specified') + + try: + convertedwd = value.format(username='test') + except KeyError as exc: + raise exceptions.ValidationError('In work_dir there is an unknown replacement field {}'.format(exc.args[0])) + except ValueError as exc: + raise exceptions.ValidationError("Error in the string: '{}'".format(exc)) + + if not os.path.isabs(convertedwd): + raise exceptions.ValidationError('The work_dir must be an absolute path') + + +def validate_mpirun_command(computer, value): + """Validate the mpirun_command variable, MUST be called after properly checking for a valid scheduler.""" + if not isinstance(value, (tuple, list)) or not all(isinstance(i, str) for i in value): + raise exceptions.ValidationError('the mpirun_command must be a list of strings') + + try: + job_resource_keys = computer.get_scheduler().job_resource_class.get_valid_keys() + except exceptions.EntryPointError: + raise exceptions.ValidationError('Unable to load the scheduler for this computer') + + subst = {i: 'value' for i in job_resource_keys} + subst['tot_num_mpiprocs'] = 'value' - Stores attributes starting with an underscore. + try: + for arg in value: + arg.format(**subst) + except KeyError as exc: + raise exceptions.ValidationError('In workdir there is an unknown replacement field {}'.format(exc.args[0])) + except ValueError as exc: + raise exceptions.ValidationError("Error in the string: '{}'".format(exc)) - Caches files and attributes before the first save, and saves everything only on store(). - After the call to store(), attributes cannot be changed. - Only after storing (or upon loading from uuid) metadata can be modified - and in this case they are directly set on the db. +def validate_default_mpiprocs_per_machine(_, value): # pylint: disable=invalid-name + """Validate the default number of CPUs per machine (node).""" + if value is None: + return - In the plugin, also set the _plugin_type_string, to be set in the DB in the 'type' field. - """ + if not isinstance(value, int) or value <= 0: + raise exceptions.ValidationError( + 'Invalid value for default_mpiprocs_per_machine, must be a positive integer, or an empty string if you do ' + 'not want to provide a default value.' + ) + + +def validate_shebang(_, value): + """Validate the shebang string.""" + if not isinstance(value, str): + raise ValueError('{} is invalid. Input has to be a string'.format(value)) + if not value.startswith('#!'): + raise ValueError('{} is invalid. A shebang line has to start with #!'.format(value)) + + +class Computer(entities.Entity): + """Class to represent a computer on which calculations are run.""" # pylint: disable=too-many-public-methods _logger = logging.getLogger(__name__) - PROPERTY_MINIMUM_SCHEDULER_POLL_INTERVAL = 'minimum_scheduler_poll_interval' # pylint: disable=invalid-name - PROPERTY_MINIMUM_SCHEDULER_POLL_INTERVAL__DEFAULT = 10. # pylint: disable=invalid-name - PROPERTY_WORKDIR = 'workdir' + PROPERTY_WORK_DIR = 'workdir' PROPERTY_SHEBANG = 'shebang' + PROPERTY_MPI_RUN_COMMAND = 'mpirun_command' + PROPERTY_MPI_PROCES_PER_MACHINE = 'default_mpiprocs_per_machine' + PROPERTY_PREPEND_TEXT = 'prepend_text' + PROPERTY_APPEND_TEXT = 'append_text' + PROPERTY_MINIMUM_SCHEDULER_POLL_INTERVAL = 'minimum_scheduler_poll_interval' # pylint: disable=invalid-name + + _property_mapping = { + 'label': ('label', None, None, None), + 'description': ('description', None, None, None), + 'hostname': ('hostname', None, None, None), + 'transport_type': ('transport', None, None, None), + 'scheduler_type': ('scheduler', None, None, None), + 'shebang': ('shebang', PROPERTY_SHEBANG, '#!/bin/bash', validate_shebang), + 'work_dir': ('work_dir', PROPERTY_WORK_DIR, '/scratch/{username}/aiida_run/', validate_work_dir), + 'mpirun_command': + ('mpirun_command', PROPERTY_MPI_RUN_COMMAND, ['mpirun', '-np', '{tot_num_mpiprocs}'], validate_mpirun_command), + 'default_mpiprocs_per_machine': + ('default_mpiprocs_per_machine', PROPERTY_MPI_PROCES_PER_MACHINE, None, validate_default_mpiprocs_per_machine), + 'prepend_text': ('prepend_text', PROPERTY_PREPEND_TEXT, '', None), + 'append_text': ('append_text', PROPERTY_APPEND_TEXT, '', None), + 'minimum_job_poll_interval': ('minimum_job_poll_interval', PROPERTY_MINIMUM_SCHEDULER_POLL_INTERVAL, 10, None), + } class Collection(entities.Collection): """The collection of Computer entries.""" @@ -62,7 +127,7 @@ def delete(self, id): # pylint: disable=redefined-builtin,invalid-name def __init__( self, name, hostname, description='', transport_type='', scheduler_type='', workdir=None, backend=None ): - """Construct a new computer + """Construct a new computer. :type name: str :type hostname: str @@ -84,383 +149,181 @@ def __init__( scheduler_type=scheduler_type ) super().__init__(model) + if workdir is not None: - self.set_workdir(workdir) + self.set_property('work_dir', workdir) def __repr__(self): return '<{}: {}>'.format(self.__class__.__name__, str(self)) def __str__(self): - return '{} ({}), pk: {}'.format(self.name, self.hostname, self.pk) - - @property - def full_text_info(self): - """ - Return a (multiline) string with a human-readable detailed information on this computer. - - :rtype: str - """ - ret_lines = [] - ret_lines.append('Computer name: {}'.format(self.name)) - ret_lines.append(' * PK: {}'.format(self.pk)) - ret_lines.append(' * UUID: {}'.format(self.uuid)) - ret_lines.append(' * Description: {}'.format(self.description)) - ret_lines.append(' * Hostname: {}'.format(self.hostname)) - ret_lines.append(' * Transport type: {}'.format(self.get_transport_type())) - ret_lines.append(' * Scheduler type: {}'.format(self.get_scheduler_type())) - ret_lines.append(' * Work directory: {}'.format(self.get_workdir())) - ret_lines.append(' * Shebang: {}'.format(self.get_shebang())) - ret_lines.append(' * mpirun command: {}'.format(' '.join(self.get_mpirun_command()))) - def_cpus_machine = self.get_default_mpiprocs_per_machine() - if def_cpus_machine is not None: - ret_lines.append(' * Default number of cpus per machine: {}'.format(def_cpus_machine)) - # pylint: disable=fixme - # TODO: Put back following line when we port Node to new backend system - # ret_lines.append(" * Used by: {} nodes".format(len(self._dbcomputer.dbnodes.all()))) - - ret_lines.append(' * prepend text:') - if self.get_prepend_text().strip(): - for line in self.get_prepend_text().split('\n'): - ret_lines.append(' {}'.format(line)) - else: - ret_lines.append(' # No prepend text.') - ret_lines.append(' * append text:') - if self.get_append_text().strip(): - for line in self.get_append_text().split('\n'): - ret_lines.append(' {}'.format(line)) - else: - ret_lines.append(' # No append text.') + return '{} ({}), pk: {}'.format(self.label, self.hostname, self.pk) - return '\n'.join(ret_lines) + def copy(self): + """Return a copy of the current object to work with, not stored yet.""" + return Computer.from_backend_entity(self._backend_entity.copy()) @property def logger(self): return self._logger - # region validation - - @classmethod - def _name_validator(cls, name): - """ - Validates the name. - """ - if not name.strip(): - raise exceptions.ValidationError('No name specified') - - @classmethod - def _hostname_validator(cls, hostname): - """ - Validates the hostname. - """ - if not hostname.strip(): - raise exceptions.ValidationError('No hostname specified') - - @classmethod - def _description_validator(cls, description): - """ - Validates the description. - """ - # The description is always valid - - @classmethod - def _transport_type_validator(cls, transport_type): - """ - Validates the transport string. - """ - if transport_type not in transports.Transport.get_valid_transports(): - raise exceptions.ValidationError('The specified transport is not a valid one') - - @classmethod - def _scheduler_type_validator(cls, scheduler_type): - """ - Validates the transport string. - """ - if scheduler_type not in schedulers.Scheduler.get_valid_schedulers(): - raise exceptions.ValidationError('The specified scheduler is not a valid one') - - @classmethod - def _prepend_text_validator(cls, prepend_text): - """ - Validates the prepend text string. - """ - # no validation done - - @classmethod - def _append_text_validator(cls, append_text): - """ - Validates the append text string. - """ - # no validation done + @property + def label(self): + """Return the computer label. - @classmethod - def _workdir_validator(cls, workdir): + :return: the label """ - Validates the transport string. - """ - if not workdir.strip(): - raise exceptions.ValidationError('No workdir specified') - - try: - convertedwd = workdir.format(username='test') - except KeyError as exc: - raise exceptions.ValidationError('In workdir there is an unknown replacement field {}'.format(exc.args[0])) - except ValueError as exc: - raise exceptions.ValidationError("Error in the string: '{}'".format(exc)) + return self._backend_entity.name - if not os.path.isabs(convertedwd): - raise exceptions.ValidationError('The workdir must be an absolute path') + @label.setter + def label(self, value): + """Set the computer label. - def _mpirun_command_validator(self, mpirun_cmd): + :param value: the label """ - Validates the mpirun_command variable. MUST be called after properly - checking for a valid scheduler. - """ - if not isinstance(mpirun_cmd, (tuple, list)) or not all(isinstance(i, str) for i in mpirun_cmd): - raise exceptions.ValidationError('the mpirun_command must be a list of strings') - - try: - job_resource_keys = self.get_scheduler().job_resource_class.get_valid_keys() - except exceptions.EntryPointError: - raise exceptions.ValidationError('Unable to load the scheduler for this computer') + if not value.strip(): + raise exceptions.ValidationError('invalid label') - subst = {i: 'value' for i in job_resource_keys} - subst['tot_num_mpiprocs'] = 'value' + self._backend_entity.set_name(value) - try: - for arg in mpirun_cmd: - arg.format(**subst) - except KeyError as exc: - raise exceptions.ValidationError('In workdir there is an unknown replacement field {}'.format(exc.args[0])) - except ValueError as exc: - raise exceptions.ValidationError("Error in the string: '{}'".format(exc)) + @property + def description(self): + """Return the description of the computer. - def validate(self): + :return: the description + :rtype: str """ - Check if the attributes and files retrieved from the DB are valid. - Raise a ValidationError if something is wrong. + return self._backend_entity.description - Must be able to work even before storing: therefore, use the get_attr and similar methods - that automatically read either from the DB or from the internal attribute cache. + @description.setter + def description(self, value): + """Set the computer description. - For the base class, this is always valid. Subclasses will reimplement this. - In the subclass, always call the super().validate() method first! + :param value: the description """ - if not self.get_name().strip(): - raise exceptions.ValidationError('No name specified') - - self._hostname_validator(self.get_hostname()) - self._description_validator(self.get_description()) - self._transport_type_validator(self.get_transport_type()) - self._scheduler_type_validator(self.get_scheduler_type()) - self._workdir_validator(self.get_workdir()) + self._backend_entity.set_description(value) - try: - mpirun_cmd = self.get_mpirun_command() - except exceptions.DbContentError: - raise exceptions.ValidationError('Error in the DB content of the metadata') + @property + def hostname(self): + return self._backend_entity.hostname - # To be called AFTER the validation of the scheduler - self._mpirun_command_validator(mpirun_cmd) + @hostname.setter + def hostname(self, value): + """Set the computer hostname. - @classmethod - def _default_mpiprocs_per_machine_validator(cls, def_cpus_per_machine): + :param value: the hostname """ - Validates the default number of CPUs per machine (node) - """ - if def_cpus_per_machine is None: - return - - if not isinstance(def_cpus_per_machine, int) or def_cpus_per_machine <= 0: - raise exceptions.ValidationError( - 'Invalid value for default_mpiprocs_per_machine, ' - 'must be a positive integer, or an empty ' - 'string if you do not want to provide a ' - 'default value.' - ) + if not value.strip(): + raise exceptions.ValidationError('invalid hostname') - # endregion + self._backend_entity.set_hostname(value) - def copy(self): - """ - Return a copy of the current object to work with, not stored yet. - """ - return Computer.from_backend_entity(self._backend_entity.copy()) + @property + def scheduler_type(self): + return self._backend_entity.get_scheduler_type() - def store(self): - """ - Store the computer in the DB. + @scheduler_type.setter + def scheduler_type(self, value): + """Set the computer scheduler type. - Differently from Nodes, a computer can be re-stored if its properties - are to be changed (e.g. a new mpirun command, etc.) + :param value: the scheduler type """ - self.validate() - return super().store() + if value not in schedulers.Scheduler.get_valid_schedulers(): + raise exceptions.ValidationError('the specified scheduler is invalid') - @property - def name(self): - return self._backend_entity.name + self._backend_entity.set_scheduler_type(value) @property - def label(self): - """ - The computer label - """ - return self.name - - @label.setter - def label(self, value): - """ - Set the computer label (i.e., name) - """ - self.set_name(value) + def transport_type(self): + return self._backend_entity.get_transport_type() - @property - def description(self): - """ - Get a description of the computer + @transport_type.setter + def transport_type(self, value): + """Set the computer transport type. - :return: the description - :rtype: str + :param value: the transport type """ - return self._backend_entity.description + if value not in transports.Transport.get_valid_transports(): + raise exceptions.ValidationError('the specified transport is invalid') - @property - def hostname(self): - return self._backend_entity.hostname + self._backend_entity.set_transport_type(value) def get_metadata(self): return self._backend_entity.get_metadata() def set_metadata(self, metadata): - """ - Set the metadata. - - .. note: You still need to call the .store() method to actually save - data to the database! (The store method can be called multiple - times, differently from AiiDA Node objects). - """ + """Set the metadata.""" self._backend_entity.set_metadata(metadata) - def delete_property(self, name, raise_exception=True): - """ - Delete a property from this computer - - :param name: the name of the property - :type name: str + def get_properties(self): + """Return a dictionary of all computer properties. - :param raise_exception: if True raise if the property does not exist, otherwise return None - :type raise_exception: bool + :return: mapping of properties """ - olddata = self.get_metadata() - try: - del olddata[name] - self.set_metadata(olddata) - except KeyError: - if raise_exception: - raise AttributeError("'{}' property not found".format(name)) + properties = {} - def set_property(self, name, value): - """ - Set a property on this computer + for name, property_tuple in self._property_mapping.items(): + if property_tuple[1] is None: + properties[property_tuple[0]] = getattr(self, name) + else: + prop = self.get_property(property_tuple[1], property_tuple[2]) + if name == 'mpirun_command': + prop = ' '.join(prop) + properties[property_tuple[0]] = prop - :param name: the property name - :param value: the new value - """ - metadata = self.get_metadata() or {} - metadata[name] = value - self.set_metadata(metadata) + return properties - def get_property(self, name, *args): - """ - Get a property of this computer + def get_property(self, name, default=None): + """Return a property of this computer. :param name: the property name :type name: str - :param args: additional arguments + :param default: optional default :return: the property value """ - if len(args) > 1: - raise TypeError('get_property expected at most 2 arguments') - olddata = self.get_metadata() try: - return olddata[name] + return self.get_metadata()[name] except KeyError: - if not args: - raise AttributeError("'{}' property not found".format(name)) - return args[0] - - def get_prepend_text(self): - return self.get_property('prepend_text', '') - - def set_prepend_text(self, val): - self.set_property('prepend_text', str(val)) - - def get_append_text(self): - return self.get_property('append_text', '') - - def set_append_text(self, val): - self.set_property('append_text', str(val)) - - def get_mpirun_command(self): - """ - Return the mpirun command. Must be a list of strings, that will be - then joined with spaces when submitting. - - I also provide a sensible default that may be ok in many cases. - """ - return self.get_property('mpirun_command', ['mpirun', '-np', '{tot_num_mpiprocs}']) + if default is None: + return self._property_mapping[name][2] + return default - def set_mpirun_command(self, val): - """ - Set the mpirun command. It must be a list of strings (you can use - string.split() if you have a single, space-separated string). - """ - if not isinstance(val, (tuple, list)) or not all(isinstance(i, str) for i in val): - raise TypeError('the mpirun_command must be a list of strings') - self.set_property('mpirun_command', val) + def set_property(self, name, value): + """Set a property for this computer. - def get_default_mpiprocs_per_machine(self): - """ - Return the default number of CPUs per machine (node) for this computer, - or None if it was not set. + :param name: the property name + :param value: the new value """ - return self.get_property('default_mpiprocs_per_machine', None) + try: + _, _, _, validator = self._property_mapping[name] + except KeyError: + raise AttributeError("unknown property '{}'".format(name)) - def set_default_mpiprocs_per_machine(self, def_cpus_per_machine): - """ - Set the default number of CPUs per machine (node) for this computer. - Accepts None if you do not want to set this value. - """ - if def_cpus_per_machine is None: - self.delete_property('default_mpiprocs_per_machine', raise_exception=False) - else: - if not isinstance(def_cpus_per_machine, int): - raise TypeError('def_cpus_per_machine must be an integer (or None)') - self.set_property('default_mpiprocs_per_machine', def_cpus_per_machine) + if validator: + validator(self, value) - def get_minimum_job_poll_interval(self): - """ - Get the minimum interval between subsequent requests to update the list - of jobs currently running on this computer. + metadata = self.get_metadata() or {} + metadata[name] = value + self.set_metadata(metadata) - :return: The minimum interval (in seconds) - :rtype: float - """ - return self.get_property( - self.PROPERTY_MINIMUM_SCHEDULER_POLL_INTERVAL, self.PROPERTY_MINIMUM_SCHEDULER_POLL_INTERVAL__DEFAULT - ) + def delete_property(self, name, raise_exception=True): + """Delete a property from this computer. - def set_minimum_job_poll_interval(self, interval): - """ - Set the minimum interval between subsequent requests to update the list - of jobs currently running on this computer. + :param name: the name of the property + :type name: str - :param interval: The minimum interval in seconds - :type interval: float + :param raise_exception: if True raise if the property does not exist, otherwise return None + :type raise_exception: bool """ - self.set_property(self.PROPERTY_MINIMUM_SCHEDULER_POLL_INTERVAL, interval) + metadata = self.get_metadata() + try: + del metadata[name] + except KeyError: + if raise_exception: + raise AttributeError("unknown property '{}'".format(name)) + self.set_metadata(metadata) def get_transport(self, user=None): """ @@ -487,48 +350,199 @@ def get_transport(self, user=None): authinfo = authinfos.AuthInfo.objects(self.backend).get(dbcomputer=self, aiidauser=user) return authinfo.get_transport() - def get_workdir(self): - """ - Get the working directory for this computer - :return: The currently configured working directory - :rtype: str - """ - return self.get_property(self.PROPERTY_WORKDIR, '/scratch/{username}/aiida_run/') + def get_authinfo(self, user): + """Return the `aiida.orm.authinfo.AuthInfo` instance for the given user on this computer if it exists. - def set_workdir(self, val): - self.set_property(self.PROPERTY_WORKDIR, val) + :param user: a User instance. + :return: a AuthInfo instance + :raise aiida.common.NotExistent: if the computer is not configured for the given user. + """ + from . import authinfos + return authinfos.AuthInfo.objects(self.backend).get(dbcomputer_id=self.id, aiidauser_id=user.id) - def get_shebang(self): - return self.get_property(self.PROPERTY_SHEBANG, '#!/bin/bash') + def is_user_configured(self, user): + """Return whether the computer is configured for the given user. - def set_shebang(self, val): + :param user: the user to check + :return: True if configured, False otherwise + :rtype: bool """ - :param str val: A valid shebang line + try: + self.get_authinfo(user) + return True + except exceptions.NotExistent: + return False + + def is_user_enabled(self, user): + """Return whether the computer is enabled or the given user. + + :param user: the user to check + :return: True if enabled, False otherwise + :rtype: bool """ - if not isinstance(val, str): - raise ValueError('{} is invalid. Input has to be a string'.format(val)) - if not val.startswith('#!'): - raise ValueError('{} is invalid. A shebang line has to start with #!'.format(val)) - metadata = self.get_metadata() - metadata['shebang'] = val - self.set_metadata(metadata) + try: + authinfo = self.get_authinfo(user) + return authinfo.enabled + except exceptions.NotExistent: + # Return False if the user is not configured (in a sense it is disabled for that user) + return False + + def get_transport_class(self): + """Return the transport class for this computer. + + :return: the transport class + :rtype: :class:`aiida.transports.Transport` + """ + return TransportFactory(self.transport_type) + + def get_scheduler(self): + """Return a scheduler instance for this computer. + + :return: the scheduler instance + :rtype: :class:`aiida.schedulers.Scheduler` + """ + return SchedulerFactory(self.scheduler_type)() + + def configure(self, user=None, **kwargs): + """ + Configure a computer for a user with valid auth params passed via kwargs + + :param user: the user to configure the computer for + :kwargs: the configuration keywords with corresponding values + :return: the authinfo object for the configured user + :rtype: :class:`aiida.orm.AuthInfo` + """ + from . import authinfos + + transport_cls = self.get_transport_class() + user = user or users.User.objects(self.backend).get_default() + valid_keys = set(transport_cls.get_valid_auth_params()) + + if not set(kwargs.keys()).issubset(valid_keys): + invalid_keys = [key for key in kwargs if key not in valid_keys] + raise ValueError( + '{transport}: received invalid authentication parameter(s) "{invalid}"'.format( + transport=transport_cls, invalid=invalid_keys + ) + ) + + try: + authinfo = self.get_authinfo(user) + except exceptions.NotExistent: + authinfo = authinfos.AuthInfo(self, user) + + auth_params = authinfo.get_auth_params() + + if valid_keys: + auth_params.update(kwargs) + authinfo.set_auth_params(auth_params) + authinfo.store() + + return authinfo + + def get_configuration(self, user=None): + """ + Get the configuration of computer for the given user as a dictionary + + :param user: the user to to get the configuration for. Uses default user if `None` + :type user: :class:`aiida.orm.User` + """ + + backend = self.backend + user = user or users.User.objects(self.backend).get_default() + + config = {} + try: + authinfo = backend.authinfos.get(self, user) + config = authinfo.get_auth_params() + except exceptions.NotExistent: + pass + + return config + + @property + def full_text_info(self): + """ + Return a (multiline) string with a human-readable detailed information on this computer. + + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`. + + :rtype: str + """ + ret_lines = [] + ret_lines.append('Computer name: {}'.format(self.name)) + ret_lines.append(' * PK: {}'.format(self.pk)) + ret_lines.append(' * UUID: {}'.format(self.uuid)) + ret_lines.append(' * Description: {}'.format(self.description)) + ret_lines.append(' * Hostname: {}'.format(self.hostname)) + ret_lines.append(' * Transport type: {}'.format(self.transport)) + ret_lines.append(' * Scheduler type: {}'.format(self.scheduler)) + ret_lines.append(' * Work directory: {}'.format(self.get_property('work_dir'))) + ret_lines.append(' * Shebang: {}'.format(self.get_property('shebang'))) + ret_lines.append(' * mpirun command: {}'.format(' '.join(self.get_property('mpirun_command')))) + def_cpus_machine = self.get_property('default_mpiprocs_per_machine') + if def_cpus_machine is not None: + ret_lines.append(' * Default number of cpus per machine: {}'.format(def_cpus_machine)) + # pylint: disable=fixme + # TODO: Put back following line when we port Node to new backend system + # ret_lines.append(" * Used by: {} nodes".format(len(self._dbcomputer.dbnodes.all()))) + + ret_lines.append(' * prepend text:') + if self.get_property('prepend_text').strip(): + for line in self.get_property('prepend_text').split('\n'): + ret_lines.append(' {}'.format(line)) + else: + ret_lines.append(' # No prepend text.') + ret_lines.append(' * append text:') + if self.get_property('append_text').strip(): + for line in self.get_property('append_text').split('\n'): + ret_lines.append(' {}'.format(line)) + else: + ret_lines.append(' # No append text.') + + return '\n'.join(ret_lines) + + @property + def name(self): + """ + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.label` property instead. + """ + return self._backend_entity.name def get_name(self): + """ + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.label` property instead. + """ return self._backend_entity.get_name() def set_name(self, val): + """ + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.label` property instead. + """ self._backend_entity.set_name(val) def get_hostname(self): """ Get this computer hostname + + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.hostname` property instead. + :rtype: str """ - return self._backend_entity.get_hostname() + return self._backend_entity.hostname def set_hostname(self, val): """ Set the hostname of this computer + + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.hostname` property instead. + :param val: The new hostname :type val: str """ @@ -538,6 +552,9 @@ def get_description(self): """ Get the description for this computer + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.description` property instead. + :return: the description :rtype: str """ @@ -546,67 +563,55 @@ def set_description(self, val): """ Set the description for this computer + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.description` property instead. + :param val: the new description :type val: str """ self._backend_entity.set_description(val) - def get_authinfo(self, user): - """ - Return the aiida.orm.authinfo.AuthInfo instance for the - given user on this computer, if the computer - is configured for the given user. - - :param user: a User instance. - :return: a AuthInfo instance - :raise aiida.common.NotExistent: if the computer is not configured for the given - user. + def get_mpirun_command(self): """ - from . import authinfos - - return authinfos.AuthInfo.objects(self.backend).get(dbcomputer_id=self.id, aiidauser_id=user.id) + Return the mpirun command. Must be a list of strings, that will be + then joined with spaces when submitting. - def is_user_configured(self, user): - """ - Is the user configured on this computer? + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.get_property('mpirun_command')` instead. - :param user: the user to check - :return: True if configured, False otherwise - :rtype: bool + I also provide a sensible default that may be ok in many cases. """ - try: - self.get_authinfo(user) - return True - except exceptions.NotExistent: - return False + return self.get_property('mpirun_command', ['mpirun', '-np', '{tot_num_mpiprocs}']) - def is_user_enabled(self, user): + def set_mpirun_command(self, val): """ - Is the given user enabled to run on this computer? + Set the mpirun command. It must be a list of strings (you can use + string.split() if you have a single, space-separated string). - :param user: the user to check - :return: True if enabled, False otherwise - :rtype: bool + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.set_property('mpirun_command', value)` instead. """ - try: - authinfo = self.get_authinfo(user) - return authinfo.enabled - except exceptions.NotExistent: - # Return False if the user is not configured (in a sense, - # it is disabled for that user) - return False + if not isinstance(val, (tuple, list)) or not all(isinstance(i, str) for i in val): + raise TypeError('the mpirun_command must be a list of strings') + self.set_property('mpirun_command', val) def get_scheduler_type(self): """ Get the scheduler type for this computer + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.scheduler` property instead. + :return: the scheduler type :rtype: str """ - return self._backend_entity.get_scheduler_type() + return self._backend_entity.scheduler def set_scheduler_type(self, scheduler_type): """ + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.scheduler` property instead. + :param scheduler_type: the new scheduler type """ self._scheduler_type_validator(scheduler_type) @@ -616,109 +621,147 @@ def get_transport_type(self): """ Get the current transport type for this computer + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.transport` property instead. + :return: the transport type :rtype: str """ - return self._backend_entity.get_transport_type() + return self._backend_entity.transport def set_transport_type(self, transport_type): """ Set the transport type for this computer + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.transport` property instead. + :param transport_type: the new transport type :type transport_type: str """ self._backend_entity.set_transport_type(transport_type) - def get_transport_class(self): + def get_default_mpiprocs_per_machine(self): """ - Get the transport class for this computer. Can be used to instantiate a transport instance. + Return the default number of CPUs per machine (node) for this computer, + or None if it was not set. + + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.get_property('default_mpiprocs_per_machine')` instead. - :return: the transport class """ - try: - return TransportFactory(self.get_transport_type()) - except exceptions.EntryPointError as exception: - raise exceptions.ConfigurationError( - 'No transport found for {} [type {}], message: {}'.format( - self.name, self.get_transport_type(), exception - ) - ) + return self.get_property('default_mpiprocs_per_machine', None) - def get_scheduler(self): + def set_default_mpiprocs_per_machine(self, def_cpus_per_machine): """ - Get a scheduler instance for this computer + Set the default number of CPUs per machine (node) for this computer. + Accepts None if you do not want to set this value. + + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.set_property('default_mpiprocs_per_machine', val)` property instead. - :return: the scheduler instance - :rtype: :class:`aiida.schedulers.Scheduler` """ - try: - scheduler_class = SchedulerFactory(self.get_scheduler_type()) - # I call the init without any parameter - return scheduler_class() - except exceptions.EntryPointError as exception: - raise exceptions.ConfigurationError( - 'No scheduler found for {} [type {}], message: {}'.format( - self.name, self.get_scheduler_type(), exception - ) - ) + if def_cpus_per_machine is None: + self.delete_property('default_mpiprocs_per_machine', raise_exception=False) + else: + if not isinstance(def_cpus_per_machine, int): + raise TypeError('def_cpus_per_machine must be an integer (or None)') + self.set_property('default_mpiprocs_per_machine', def_cpus_per_machine) - def configure(self, user=None, **kwargs): + def get_minimum_job_poll_interval(self): """ - Configure a computer for a user with valid auth params passed via kwargs + Get the minimum interval between subsequent requests to update the list + of jobs currently running on this computer. - :param user: the user to configure the computer for - :kwargs: the configuration keywords with corresponding values - :return: the authinfo object for the configured user - :rtype: :class:`aiida.orm.AuthInfo` + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.get_property('minimum_job_poll_interval')` instead. + + :return: The minimum interval (in seconds) + :rtype: float """ - from . import authinfos + return self.get_property(self.PROPERTY_MINIMUM_SCHEDULER_POLL_INTERVAL, 10) - transport_cls = self.get_transport_class() - user = user or users.User.objects(self.backend).get_default() - valid_keys = set(transport_cls.get_valid_auth_params()) + def set_minimum_job_poll_interval(self, interval): + """ + Set the minimum interval between subsequent requests to update the list + of jobs currently running on this computer. - if not set(kwargs.keys()).issubset(valid_keys): - invalid_keys = [key for key in kwargs if key not in valid_keys] - raise ValueError( - '{transport}: received invalid authentication parameter(s) "{invalid}"'.format( - transport=transport_cls, invalid=invalid_keys - ) - ) + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.set_property('minimum_job_poll_interval', val)` property instead. - try: - authinfo = self.get_authinfo(user) - except exceptions.NotExistent: - authinfo = authinfos.AuthInfo(self, user) + :param interval: The minimum interval in seconds + :type interval: float + """ + self.set_property(self.PROPERTY_MINIMUM_SCHEDULER_POLL_INTERVAL, interval) - auth_params = authinfo.get_auth_params() + def get_workdir(self): + """ + Get the working directory for this computer - if valid_keys: - auth_params.update(kwargs) - authinfo.set_auth_params(auth_params) - authinfo.store() + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.get_property('work_dir')` instead. - return authinfo + :return: The currently configured working directory + :rtype: str + """ + return self.get_property(self.PROPERTY_WORK_DIR, '/scratch/{username}/aiida_run/') - def get_configuration(self, user=None): + def set_workdir(self, val): """ - Get the configuration of computer for the given user as a dictionary + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.set_property('work_dir', val)` property instead. + """ + self.set_property(self.PROPERTY_WORK_DIR, val) - :param user: the user to to get the configuration for. Uses default user if `None` - :type user: :class:`aiida.orm.User` + def get_shebang(self): """ + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.get_property('shebang')` instead. + """ + return self.get_property(self.PROPERTY_SHEBANG, '#!/bin/bash') - backend = self.backend - user = user or users.User.objects(self.backend).get_default() + def set_shebang(self, val): + """ + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.set_property('shebang', val)` property instead. - config = {} - try: - authinfo = backend.authinfos.get(self, user) - config = authinfo.get_auth_params() - except exceptions.NotExistent: - pass + :param str val: A valid shebang line + """ + if not isinstance(val, str): + raise ValueError('{} is invalid. Input has to be a string'.format(val)) + if not val.startswith('#!'): + raise ValueError('{} is invalid. A shebang line has to start with #!'.format(val)) + metadata = self.get_metadata() + metadata['shebang'] = val + self.set_metadata(metadata) - return config + def get_prepend_text(self): + """ + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.get_property('prepend_text')` instead. + """ + return self.get_property('prepend_text', '') + + def set_prepend_text(self, val): + """ + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.set_property('prepend_text', val)` property instead. + """ + self.set_property('prepend_text', str(val)) + + def get_append_text(self): + """ + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.get_property('append_text')` instead. + """ + return self.get_property('append_text', '') + + def set_append_text(self, val): + """ + .. deprecated:: 1.0.0 + Will be removed in `v2.0.0`: use `self.set_property('append_text', val)` property instead. + """ + self.set_property('append_text', str(val)) @staticmethod def get_schema(): diff --git a/aiida/orm/implementation/computers.py b/aiida/orm/implementation/computers.py index b90cd41681..14aa64cf5e 100644 --- a/aiida/orm/implementation/computers.py +++ b/aiida/orm/implementation/computers.py @@ -62,13 +62,7 @@ def get_metadata(self): @abc.abstractmethod def set_metadata(self, metadata): - """ - Set the metadata. - - .. note: You still need to call the .store() method to actually save - data to the database! (The store method can be called multiple - times, differently from AiiDA Node objects). - """ + """Set the metadata.""" @abc.abstractmethod def get_name(self): diff --git a/aiida/orm/implementation/django/authinfos.py b/aiida/orm/implementation/django/authinfos.py index 87e0ff6910..e20059baf1 100644 --- a/aiida/orm/implementation/django/authinfos.py +++ b/aiida/orm/implementation/django/authinfos.py @@ -152,11 +152,11 @@ def get(self, computer, user): authinfo = DbAuthInfo.objects.get(dbcomputer=computer.id, aiidauser=user.id) except ObjectDoesNotExist: raise exceptions.NotExistent( - 'User<{}> has no configuration for Computer<{}>'.format(user.email, computer.name) + 'User<{}> has no configuration for Computer<{}>'.format(user.email, computer.label) ) except MultipleObjectsReturned: raise exceptions.MultipleObjectsError( - 'User<{}> has multiple configurations for Computer<{}>'.format(user.email, computer.name) + 'User<{}> has multiple configurations for Computer<{}>'.format(user.email, computer.label) ) else: return self.from_dbmodel(authinfo) diff --git a/aiida/orm/implementation/sqlalchemy/authinfos.py b/aiida/orm/implementation/sqlalchemy/authinfos.py index c68b0e03f0..67391260bd 100644 --- a/aiida/orm/implementation/sqlalchemy/authinfos.py +++ b/aiida/orm/implementation/sqlalchemy/authinfos.py @@ -151,11 +151,11 @@ def get(self, computer, user): authinfo = session.query(DbAuthInfo).filter_by(dbcomputer_id=computer.id, aiidauser_id=user.id).one() except NoResultFound: raise exceptions.NotExistent( - 'User<{}> has no configuration for Computer<{}>'.format(user.email, computer.name) + 'User<{}> has no configuration for Computer<{}>'.format(user.email, computer.label) ) except MultipleResultsFound: raise exceptions.MultipleObjectsError( - 'User<{}> has multiple configurations for Computer<{}>'.format(user.email, computer.name) + 'User<{}> has multiple configurations for Computer<{}>'.format(user.email, computer.label) ) else: return self.from_dbmodel(authinfo) diff --git a/aiida/orm/nodes/data/code.py b/aiida/orm/nodes/data/code.py index f4fb1b40b3..6dc52f6475 100644 --- a/aiida/orm/nodes/data/code.py +++ b/aiida/orm/nodes/data/code.py @@ -106,7 +106,7 @@ def get_computer_name(self): computer_str = 'repository' else: if self.computer is not None: - computer_str = self.computer.name + computer_str = self.computer.label else: computer_str = '[unknown]' @@ -502,16 +502,16 @@ def get_full_text_info(self, verbose=False): result.append(['Remote machine', self.get_remote_computer().name]) result.append(['Remote absolute path', self.get_remote_exec_path()]) - if self.get_prepend_text().strip(): + if self.get_property('prepend_text').strip(): result.append(['Prepend text', '']) - for line in self.get_prepend_text().split('\n'): + for line in self.get_property('prepend_text').split('\n'): result.append(['', line]) else: result.append(['Prepend text', 'No prepend text']) - if self.get_append_text().strip(): + if self.get_property('append_text').strip(): result.append(['Append text', '']) - for line in self.get_append_text().split('\n'): + for line in self.get_property('append_text').split('\n'): result.append(['', line]) else: result.append(['Append text', 'No append text']) diff --git a/aiida/orm/nodes/data/remote.py b/aiida/orm/nodes/data/remote.py index 03a089e515..2239b51ca4 100644 --- a/aiida/orm/nodes/data/remote.py +++ b/aiida/orm/nodes/data/remote.py @@ -28,7 +28,7 @@ def __init__(self, remote_path=None, **kwargs): self.set_remote_path(remote_path) def get_computer_name(self): - return self.computer.name + return self.computer.label def get_remote_path(self): return self.get_attribute('remote_path') @@ -71,7 +71,7 @@ def getfile(self, relpath, destpath): except IOError as e: if e.errno == 2: # file not existing raise IOError('The required remote file {} on {} does not exist or has been deleted.'.format( - full_path, self.computer.name + full_path, self.computer.label )) else: raise @@ -96,7 +96,7 @@ def listdir(self, relpath='.'): if e.errno == 2 or e.errno == 20: # directory not existing or not a directory exc = IOError( 'The required remote folder {} on {} does not exist, is not a directory or has been deleted.'.format( - full_path, self.computer.name + full_path, self.computer.label )) exc.errno = e.errno raise exc @@ -109,7 +109,7 @@ def listdir(self, relpath='.'): if e.errno == 2 or e.errno == 20: # directory not existing or not a directory exc = IOError( 'The required remote folder {} on {} does not exist, is not a directory or has been deleted.'.format( - full_path, self.computer.name + full_path, self.computer.label )) exc.errno = e.errno raise exc @@ -134,7 +134,7 @@ def listdir_withattributes(self, path='.'): if e.errno == 2 or e.errno == 20: # directory not existing or not a directory exc = IOError( 'The required remote folder {} on {} does not exist, is not a directory or has been deleted.'.format( - full_path, self.computer.name + full_path, self.computer.label )) exc.errno = e.errno raise exc @@ -147,7 +147,7 @@ def listdir_withattributes(self, path='.'): if e.errno == 2 or e.errno == 20: # directory not existing or not a directory exc = IOError( 'The required remote folder {} on {} does not exist, is not a directory or has been deleted.'.format( - full_path, self.computer.name + full_path, self.computer.label )) exc.errno = e.errno raise exc diff --git a/aiida/orm/utils/builders/computer.py b/aiida/orm/utils/builders/computer.py index 04fe730cf2..a21f2515d0 100644 --- a/aiida/orm/utils/builders/computer.py +++ b/aiida/orm/utils/builders/computer.py @@ -39,15 +39,15 @@ def get_computer_spec(computer): spec = {} spec['label'] = computer.label spec['description'] = computer.description - spec['hostname'] = computer.get_hostname() - spec['scheduler'] = computer.get_scheduler_type() - spec['transport'] = computer.get_transport_type() - spec['prepend_text'] = computer.get_prepend_text() - spec['append_text'] = computer.get_append_text() - spec['work_dir'] = computer.get_workdir() - spec['shebang'] = computer.get_shebang() - spec['mpirun_command'] = ' '.join(computer.get_mpirun_command()) - spec['mpiprocs_per_machine'] = computer.get_default_mpiprocs_per_machine() + spec['hostname'] = computer.hostname + spec['scheduler'] = computer.scheduler_type + spec['transport'] = computer.transport_type + spec['prepend_text'] = computer.get_property('prepend_text') + spec['append_text'] = computer.get_property('append_text') + spec['work_dir'] = computer.get_property('work_dir') + spec['shebang'] = computer.get_property('shebang') + spec['mpirun_command'] = ' '.join(computer.get_property('mpirun_command')) + spec['mpiprocs_per_machine'] = computer.get_property('default_mpiprocs_per_machine') return spec @@ -77,13 +77,13 @@ def new(self): computer = Computer(name=self._get_and_count('label', used), hostname=self._get_and_count('hostname', used)) - computer.set_description(self._get_and_count('description', used)) - computer.set_scheduler_type(self._get_and_count('scheduler', used)) - computer.set_transport_type(self._get_and_count('transport', used)) - computer.set_prepend_text(self._get_and_count('prepend_text', used)) - computer.set_append_text(self._get_and_count('append_text', used)) - computer.set_workdir(self._get_and_count('work_dir', used)) - computer.set_shebang(self._get_and_count('shebang', used)) + computer.description = self._get_and_count('description', used) + computer.scheduler_type = self._get_and_count('scheduler', used) + computer.transport_type = self._get_and_count('transport', used) + computer.set_property('prepend_text', self._get_and_count('prepend_text', used)) + computer.set_property('append_text', self._get_and_count('append_text', used)) + computer.set_property('work_dir', self._get_and_count('work_dir', used)) + computer.set_property('shebang', self._get_and_count('shebang', used)) mpiprocs_per_machine = self._get_and_count('mpiprocs_per_machine', used) # In the command line, 0 means unspecified @@ -102,13 +102,12 @@ def new(self): 'Invalid value provided for mpiprocs_per_machine, ' 'must be positive' ) - computer.set_default_mpiprocs_per_machine(mpiprocs_per_machine) + computer.set_property('default_mpiprocs_per_machine', mpiprocs_per_machine) mpirun_command_internal = self._get_and_count('mpirun_command', used).strip().split(' ') if mpirun_command_internal == ['']: mpirun_command_internal = [] - computer._mpirun_command_validator(mpirun_command_internal) # pylint: disable=protected-access - computer.set_mpirun_command(mpirun_command_internal) + computer.set_property('mpirun_command', mpirun_command_internal) # Complain if there are keys that are passed but not used if passed_keys - used: diff --git a/aiida/transports/cli.py b/aiida/transports/cli.py index ecce23c8c0..16b65092ab 100644 --- a/aiida/transports/cli.py +++ b/aiida/transports/cli.py @@ -26,10 +26,10 @@ # pylint: disable=unused-argument def match_comp_transport(ctx, param, computer, transport_type): """Check the computer argument against the transport type.""" - if computer.get_transport_type() != transport_type: + if computer.transport_type != transport_type: echo.echo_critical( 'Computer {} has transport of type "{}", not {}!'.format( - computer.name, computer.get_transport_type(), transport_type + computer.label, computer.transport_type, transport_type ) ) return computer @@ -42,12 +42,12 @@ def configure_computer_main(computer, user, **kwargs): user = user or orm.User.objects.get_default() - echo.echo_info('Configuring computer {} for user {}.'.format(computer.name, user.email)) + echo.echo_info('Configuring computer {} for user {}.'.format(computer.label, user.email)) if user.email != get_manager().get_profile().default_user: echo.echo_info('Configuring different user, defaults may not be appropriate.') computer.configure(user=user, **kwargs) - echo.echo_success('{} successfully configured for {}'.format(computer.name, user.email)) + echo.echo_success('{} successfully configured for {}'.format(computer.label, user.email)) def common_params(command_func): From 547965c6797dc629f82f260ce0f8893ea930efe2 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Mon, 9 Dec 2019 12:15:16 +0100 Subject: [PATCH 2/2] Add `--format` option to `verdi computer show` This allows to, in addition to the default table display, dump the contents of a computer in yaml format which can in turn be used for the `--config` option of `verdi computer setup`. This makes the sharing of a particular computer that has been setup very easy. Sharing yaml config files is more apt than AiiDA export archives which was the only possible way of sharing `Computer` instances to date. --- .../tests/cmdline/commands/test_computer.py | 19 +++++++++++++++++++ aiida/cmdline/commands/cmd_computer.py | 15 +++++++++++++-- aiida/cmdline/params/options/__init__.py | 18 ++++++++++-------- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/aiida/backends/tests/cmdline/commands/test_computer.py b/aiida/backends/tests/cmdline/commands/test_computer.py index 7f41341f84..f3e5dbfe06 100644 --- a/aiida/backends/tests/cmdline/commands/test_computer.py +++ b/aiida/backends/tests/cmdline/commands/test_computer.py @@ -318,6 +318,25 @@ def test_noninteractive_from_config(self): self.assertClickResultNoException(result) self.assertIsInstance(orm.Computer.objects.get(name=label), orm.Computer) + def test_show_yaml(self): + """Test round-trip of `verdi computer show --format yaml` with `verdi computer setup`.""" + self.computer.set_property('default_mpiprocs_per_machine', 1) + options = ['--format', 'yaml', '--', self.computer.pk] + result = self.cli_runner.invoke(computer_show, options) + self.assertClickResultNoException(result) + + # Rename the existing computer otherwise the setup command will fail + self.computer.label = self.computer.label + '_backup' + + # Create a temporary file with the yaml formatted output and use it to setup a computer + yaml_config = result.stdout_bytes + with tempfile.NamedTemporaryFile() as handle: + handle.write(yaml_config) + handle.flush() + handle.seek(0) + result = self.cli_runner.invoke(computer_setup, ['--non-interactive', '--config', handle.name]) + self.assertClickResultNoException(result) + class TestVerdiComputerConfigure(AiidaTestCase): diff --git a/aiida/cmdline/commands/cmd_computer.py b/aiida/cmdline/commands/cmd_computer.py index 2659d7845f..235eaf6ecd 100644 --- a/aiida/cmdline/commands/cmd_computer.py +++ b/aiida/cmdline/commands/cmd_computer.py @@ -406,10 +406,21 @@ def computer_list(all_entries, raw): @verdi_computer.command('show') @arguments.COMPUTER() +@options.FORMAT(type=click.Choice(['table', 'yaml']), default='table', help='Format of the output.') @with_dbenv() -def computer_show(computer): +def computer_show(computer, fmt): """Show detailed information for a computer.""" - echo.echo(computer.full_text_info) + import tabulate + import yaml + + if fmt == 'yaml': + echo.echo(yaml.dump(computer.get_properties())) + else: + properties = computer.get_properties() + properties['pk'] = computer.pk + properties['uuid'] = computer.uuid + data = sorted([(k.lower(), v) for k, v in properties.items()]) + echo.echo(tabulate.tabulate(data)) @verdi_computer.command('rename') diff --git a/aiida/cmdline/params/options/__init__.py b/aiida/cmdline/params/options/__init__.py index c67e520bb8..648e640beb 100644 --- a/aiida/cmdline/params/options/__init__.py +++ b/aiida/cmdline/params/options/__init__.py @@ -20,14 +20,14 @@ __all__ = ( 'graph_traversal_rules', 'PROFILE', 'CALCULATION', 'CALCULATIONS', 'CODE', 'CODES', 'COMPUTER', 'COMPUTERS', - 'DATUM', 'DATA', 'GROUP', 'GROUPS', 'NODE', 'NODES', 'FORCE', 'SILENT', 'VISUALIZATION_FORMAT', 'INPUT_FORMAT', - 'EXPORT_FORMAT', 'ARCHIVE_FORMAT', 'NON_INTERACTIVE', 'DRY_RUN', 'USER_EMAIL', 'USER_FIRST_NAME', 'USER_LAST_NAME', - 'USER_INSTITUTION', 'BACKEND', 'DB_HOST', 'DB_PORT', 'DB_USERNAME', 'DB_PASSWORD', 'DB_NAME', 'REPOSITORY_PATH', - 'PROFILE_ONLY_CONFIG', 'PROFILE_SET_DEFAULT', 'PREPEND_TEXT', 'APPEND_TEXT', 'LABEL', 'DESCRIPTION', 'INPUT_PLUGIN', - 'CALC_JOB_STATE', 'PROCESS_STATE', 'EXIT_STATUS', 'FAILED', 'LIMIT', 'PROJECT', 'ORDER_BY', 'PAST_DAYS', - 'OLDER_THAN', 'ALL', 'ALL_STATES', 'ALL_USERS', 'GROUP_CLEAR', 'RAW', 'HOSTNAME', 'TRANSPORT', 'SCHEDULER', 'USER', - 'PORT', 'FREQUENCY', 'VERBOSE', 'TIMEOUT', 'FORMULA_MODE', 'TRAJECTORY_INDEX', 'WITH_ELEMENTS', - 'WITH_ELEMENTS_EXCLUSIVE' + 'DATUM', 'DATA', 'GROUP', 'GROUPS', 'NODE', 'NODES', 'FORCE', 'SILENT', 'FORMAT', 'VISUALIZATION_FORMAT', + 'INPUT_FORMAT', 'EXPORT_FORMAT', 'ARCHIVE_FORMAT', 'NON_INTERACTIVE', 'DRY_RUN', 'USER_EMAIL', 'USER_FIRST_NAME', + 'USER_LAST_NAME', 'USER_INSTITUTION', 'BACKEND', 'DB_HOST', 'DB_PORT', 'DB_USERNAME', 'DB_PASSWORD', 'DB_NAME', + 'REPOSITORY_PATH', 'PROFILE_ONLY_CONFIG', 'PROFILE_SET_DEFAULT', 'PREPEND_TEXT', 'APPEND_TEXT', 'LABEL', + 'DESCRIPTION', 'INPUT_PLUGIN', 'CALC_JOB_STATE', 'PROCESS_STATE', 'EXIT_STATUS', 'FAILED', 'LIMIT', 'PROJECT', + 'ORDER_BY', 'PAST_DAYS', 'OLDER_THAN', 'ALL', 'ALL_STATES', 'ALL_USERS', 'GROUP_CLEAR', 'RAW', 'HOSTNAME', + 'TRANSPORT', 'SCHEDULER', 'USER', 'PORT', 'FREQUENCY', 'VERBOSE', 'TIMEOUT', 'FORMULA_MODE', 'TRAJECTORY_INDEX', + 'WITH_ELEMENTS', 'WITH_ELEMENTS_EXCLUSIVE' ) TRAVERSAL_RULE_HELP_STRING = { @@ -184,6 +184,8 @@ def decorator(command): SILENT = OverridableOption('-s', '--silent', is_flag=True, default=False, help='Suppress any output printed to stdout.') +FORMAT = OverridableOption('-F', '--format', 'fmt', show_default=True, help='The desired format.') + VISUALIZATION_FORMAT = OverridableOption( '-F', '--format', 'fmt', show_default=True, help='Format of the visualized output.' )