diff --git a/starcluster/config.py b/starcluster/config.py index e15c866e3..67504ea99 100644 --- a/starcluster/config.py +++ b/starcluster/config.py @@ -531,8 +531,13 @@ def _load_sections(self, section_prefix, section_settings, sections_store = AttributeDict() for sec in sections: name = self._get_section_name(sec) - sections_store[name] = self._load_section(sec, section_settings, - filter_settings) + sections_store[name] = AttributeDict() + self._load_settings(sec, section_settings, sections_store[name], filter_settings) + for sec in sections: + name = self._get_section_name(sec) + self._load_extends_settings(name, sections_store) + self._load_defaults(section_settings, sections_store[name]) + self._check_required(name, section_settings, sections_store[name]) return sections_store def _load_cluster_sections(self, cluster_sections): diff --git a/starcluster/static.py b/starcluster/static.py index 72bb4e896..2d4513477 100644 --- a/starcluster/static.py +++ b/starcluster/static.py @@ -271,6 +271,7 @@ def create_sc_config_dirs(): PLUGIN_SETTINGS = { 'setup_class': (str, True, None, None, None), + 'extends': (str, False, None, None, None), } PERMISSION_SETTINGS = { @@ -287,6 +288,7 @@ def create_sc_config_dirs(): # 'source_group_owner': (int, False, None), } +# func, required, default, options, callback CLUSTER_SETTINGS = { 'spot_bid': (float, False, None, None, None), 'cluster_size': (int, True, None, None, None), diff --git a/starcluster/templates/config.py b/starcluster/templates/config.py index 885d741c3..772ac0b99 100644 --- a/starcluster/templates/config.py +++ b/starcluster/templates/config.py @@ -317,6 +317,18 @@ # # Set a custom packer. Must be one of 'json', 'pickle', or 'msgpack' # # This is optional. # PACKER = pickle +# # MASTER_ENGINES = 0 # Avoid calculations on master if you are worried about memory/resources +# +# # [plugin ipcluster-joblib] +# # EXTENDS = ipcluster +# # NODE_ENGINES = 1 +# +# # [plugin ipclusterrestart] +# # SETUP_CLASS = starcluster.plugins.ipcluster.IPClusterRestartEngines +# +# # [plugin ipclusterrestart-joblib] +# # SETUP_CLASS = starcluster.plugins.ipcluster.IPClusterRestartEngines +# # EXTENDS = ipcluster-joblib # # Use this plugin to create a cluster SSH "dashboard" using tmux. The plugin # creates a tmux session on the master node that automatically connects to all diff --git a/starcluster/tests/templates/config.py b/starcluster/tests/templates/config.py index 33b29852f..1aaec6dae 100644 --- a/starcluster/tests/templates/config.py +++ b/starcluster/tests/templates/config.py @@ -52,6 +52,7 @@ 'p3_param1': 'bon', 'p3_param2': 'jour', 'p3_param3': 'monsignour', + 'p4_param1': 'oui', 's1_protocol': 'udp', 's1_from_port': 20, 's1_to_port': 20, @@ -71,7 +72,7 @@ 'c1_master_type': 'm1.small', 'c1_node_type': 'm1.small', 'c1_vols': 'v1,v2,v3', - 'c1_plugs': 'p1,p2,p3', + 'c1_plugs': 'p1,p2,p3,p4', 'c1_zone': 'us-east-1c', 'c2_extends': 'c1', 'c2_keyname': 'k2', @@ -150,6 +151,10 @@ MY_OTHER_ARG = %(p3_param2)s MY_OTHER_OTHER_ARG = %(p3_param3)s +[plugin p4] +EXTENDS = p3 +MY_OTHER_OTHER_ARG = %(p4_param1)s + [permission s1] protocol = %(s1_protocol)s from_port = %(s1_from_port)s diff --git a/starcluster/tests/test_config.py b/starcluster/tests/test_config.py index 784d9c8e8..192e7346e 100644 --- a/starcluster/tests/test_config.py +++ b/starcluster/tests/test_config.py @@ -169,15 +169,17 @@ def test_order_invariance(self): def test_plugins(self): c1 = self.config.get_cluster_template('c1') plugs = c1.plugins - assert len(plugs) == 3 + assert len(plugs) == 4 # test that order is preserved - p1, p2, p3 = plugs + p1, p2, p3, p4 = plugs p1_name = p1.__name__ p1_class = utils.get_fq_class_name(p1) p2_name = p2.__name__ p2_class = utils.get_fq_class_name(p2) p3_name = p3.__name__ p3_class = utils.get_fq_class_name(p3) + p4_name = p4.__name__ + p4_class = utils.get_fq_class_name(p4) assert p1_name == 'p1' assert p1_class == 'starcluster.tests.mytestplugin.SetupClass' assert p1.my_arg == '23' @@ -193,6 +195,13 @@ def test_plugins(self): assert p3.my_arg == 'bon' assert p3.my_other_arg == 'jour' assert p3.my_other_other_arg == 'monsignour' + # Test that p4 inherits from p3 + assert p4_name == 'p4' + assert p4_class == setup_class3 + assert p4.my_arg == 'bon' + assert p4.my_other_arg == 'jour' + # And test that the p4 argument overides the p3 default + assert p4.my_other_other_arg == 'oui' def test_plugin_not_defined(self): try: