From 14e50090a296ebadbb8badc6925435f6080d1a83 Mon Sep 17 00:00:00 2001 From: Miki Bonacci <46074008+mikibonacci@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:20:58 +0200 Subject: [PATCH 01/19] Add backward compatibility of the clean_button and exception catching (#847) --- src/aiidalab_qe/app/result/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aiidalab_qe/app/result/__init__.py b/src/aiidalab_qe/app/result/__init__.py index 72742b0d8..8b83364ce 100644 --- a/src/aiidalab_qe/app/result/__init__.py +++ b/src/aiidalab_qe/app/result/__init__.py @@ -158,9 +158,9 @@ def _update_clean_scratch_button_layout(self): if isinstance(called_descendant, orm.CalcJobNode): try: cleaned_bool.append( - called_descendant.outputs.remote_folder.is_cleaned + called_descendant.outputs.remote_folder.is_empty ) - except (OSError, KeyError): + except Exception: pass self.clean_scratch_button.disabled = all(cleaned_bool) @@ -184,7 +184,7 @@ def _on_click_clean_scratch_button(self, _=None): if isinstance(called_descendant, orm.CalcJobNode): try: called_descendant.outputs.remote_folder._clean() - except (OSError, KeyError): + except Exception: pass # update the kill button layout From 6101655c4fd194270cd2b9c487509900f408a42c Mon Sep 17 00:00:00 2001 From: Miki Bonacci <46074008+mikibonacci@users.noreply.github.com> Date: Tue, 8 Oct 2024 15:28:57 +0200 Subject: [PATCH 02/19] Features/warning long calc (#840) This fixes https://github.com/aiidalab/aiidalab-qe/issues/816 * Displaying the warning message if resources are not enough. The check and the message were already there, but not triggered/shown. * Modification for the warning message. * Adding also volume check, for low dimensional system can be useful. * Refining warning messages - not on localhost but big system and CPUs<4 - same as above, but on localhost - on localhost and more than 1 CPUs (we may not have enough slots) * adding self.input_structure check * Warning messages modified as suggested * Adding tests for warning messages --------- Co-authored-by: AndresOrtegaGuerrero <34098967+AndresOrtegaGuerrero@users.noreply.github.com> --- src/aiidalab_qe/app/submission/__init__.py | 86 ++++++++++++++++------ tests/conftest.py | 8 ++ tests/test_submit_qe_workchain.py | 56 ++++++++++++++ 3 files changed, 126 insertions(+), 24 deletions(-) diff --git a/src/aiidalab_qe/app/submission/__init__.py b/src/aiidalab_qe/app/submission/__init__.py index a7787341a..2cf0eda95 100644 --- a/src/aiidalab_qe/app/submission/__init__.py +++ b/src/aiidalab_qe/app/submission/__init__.py @@ -7,7 +7,6 @@ import ipywidgets as ipw import traitlets as tl -from IPython.display import display from aiida import orm from aiida.common import NotExistent @@ -53,6 +52,7 @@ class SubmitQeAppWorkChainStep(ipw.VBox, WizardAppWidgetStep): # Warn the user if they are trying to run calculations for a large # structure on localhost. RUN_ON_LOCALHOST_NUM_SITES_WARN_THRESHOLD = 10 + RUN_ON_LOCALHOST_VOLUME_WARN_THRESHOLD = 1000 # \AA^3 # Put a limit on how many MPI tasks you want to run per k-pool by default MAX_MPI_PER_POOL = 20 @@ -65,8 +65,8 @@ class SubmitQeAppWorkChainStep(ipw.VBox, WizardAppWidgetStep): external_submission_blockers = tl.List(tl.Unicode()) def __init__(self, qe_auto_setup=True, **kwargs): - self.message_area = ipw.Output() self._submission_blocker_messages = ipw.HTML() + self._submission_warning_messages = ipw.HTML() self.pw_code = PwCodeResourceSetupWidget( description="pw.x:", default_calc_job_plugin="quantumespresso.pw" @@ -129,10 +129,10 @@ def __init__(self, qe_auto_setup=True, **kwargs): super().__init__( children=[ *self.code_children, - self.message_area, self.sssp_installation_status, self.qe_setup_status, self._submission_blocker_messages, + self._submission_warning_messages, self.process_label_help, self.process_label, self.process_description, @@ -143,6 +143,10 @@ def __init__(self, qe_auto_setup=True, **kwargs): # set default codes self.set_selected_codes(DEFAULT_PARAMETERS["codes"]) + # observe these two for the resource checking: + self.pw_code.num_cpus.observe(self._check_resources, "value") + self.pw_code.num_nodes.observe(self._check_resources, "value") + @tl.observe("internal_submission_blockers", "external_submission_blockers") def _observe_submission_blockers(self, _change): """Observe the submission blockers and update the message area.""" @@ -222,48 +226,82 @@ def _auto_select_code(self, change): _ALERT_MESSAGE = """
× - × {message}
""" def _show_alert_message(self, message, alert_class="info"): - with self.message_area: - display( - ipw.HTML( - self._ALERT_MESSAGE.format(alert_class=alert_class, message=message) - ) - ) + self._submission_warning_messages.value = self._ALERT_MESSAGE.format( + alert_class=alert_class, message=message + ) - def _check_resources(self): + @tl.observe("input_structure") + def _check_resources(self, _change=None): """Check whether the currently selected resources will be sufficient and warn if not.""" - if not self.pw_code.value: + if not self.pw_code.value or not self.input_structure: return # No code selected, nothing to do. - num_cpus = self.resources_config.num_cpus.value + num_cpus = self.pw_code.num_cpus.value * self.pw_code.num_nodes.value on_localhost = ( orm.load_node(self.pw_code.value).computer.hostname == "localhost" ) - if self.pw_code.value and on_localhost and num_cpus > 1: + num_sites = len(self.input_structure.sites) + volume = self.input_structure.get_cell_volume() + + if ( + self.input_structure + and not on_localhost + and ( + num_sites > self.RUN_ON_LOCALHOST_NUM_SITES_WARN_THRESHOLD + or volume > self.RUN_ON_LOCALHOST_VOLUME_WARN_THRESHOLD + ) + and num_cpus < 4 + ): + # Warning-1 self._show_alert_message( - "The selected code would be executed on the local host, but " - "the number of CPUs is larger than one. Please review " - "the configuration and consider to select a code that runs " - "on a larger system if necessary.", + f"⚠ Warning: The selected structure has a large number of atoms ({num_sites}) " + f"or a significant cell volume ({int(volume)} Å3), making it computationally demanding " + "to run at the localhost. Consider the following: " + "", alert_class="warning", ) elif ( self.input_structure and on_localhost - and len(self.input_structure.sites) - > self.RUN_ON_LOCALHOST_NUM_SITES_WARN_THRESHOLD + and ( + num_sites > self.RUN_ON_LOCALHOST_NUM_SITES_WARN_THRESHOLD + or volume > self.RUN_ON_LOCALHOST_VOLUME_WARN_THRESHOLD + ) + and num_cpus < 4 ): + # Warning-2 + self._show_alert_message( + f"⚠ Warning: The selected structure has a large number of atoms ({num_sites}) " + f"or a significant cell volume ({int(volume)} Å3), making it computationally demanding " + "to run in a reasonable amount of time. Consider the following: " + "", + alert_class="warning", + ) + elif on_localhost and num_cpus > 1: + # Warning-3 self._show_alert_message( - "The selected code would be executed on the local host, but the " - "number of sites of the selected structure is relatively large. " - "Consider to select a code that runs on a larger system if " - "necessary.", + "⚠ Warning: the selected pw.x code will run on the local host, but " + "the number of CPUs is larger than one. Please be sure that your local " + "environment has enough free CPUs for the calculation. Consider the following: " + "", alert_class="warning", ) + else: + self._submission_warning_messages.value = "" @tl.observe("state") def _observe_state(self, change): diff --git a/tests/conftest.py b/tests/conftest.py index 844021450..c05067b76 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -105,6 +105,14 @@ def _generate_structure_data(name="silicon", pbc=(True, True, True)): structure.append_atom(position=(0.0, 0.0, 1.0), symbols="O") structure.append_atom(position=(0.0, 1.0, 0.0), symbols="H") + elif name == "H2O-larger": + # just a larger supercell. To test the warning messages + cell = [[20.0, 0.0, 0.0], [0.0, 20.0, 0.0], [0.0, 0.0, 20.0]] + structure = orm.StructureData(cell=cell) + structure.append_atom(position=(0.0, 0.0, 0.0), symbols="H") + structure.append_atom(position=(0.0, 0.0, 1.0), symbols="O") + structure.append_atom(position=(0.0, 1.0, 0.0), symbols="H") + structure.pbc = pbc return structure diff --git a/tests/test_submit_qe_workchain.py b/tests/test_submit_qe_workchain.py index 181554392..6f54d1ee1 100644 --- a/tests/test_submit_qe_workchain.py +++ b/tests/test_submit_qe_workchain.py @@ -132,6 +132,62 @@ def test_create_builder_advanced_settings( ) +@pytest.mark.usefixtures("sssp") +def test_warning_messages( + generate_structure_data, + submit_app_generator, +): + """ "Test the creation of the workchain builder. + + metal, non-magnetic + """ + + app = submit_app_generator(properties=["bands", "pdos"]) + submit_step = app.submit_step + submit_step.codes["pw"].num_cpus.value = 1 + submit_step._check_resources() + # no warning: + assert submit_step._submission_warning_messages.value == "" + + # now we increase the resources, so we should have the Warning-3 + submit_step.codes["pw"].num_cpus.value = 8 + submit_step._check_resources() + message = ( + "⚠ Warning: the selected pw.x code will run on the local host, but " + "the number of CPUs is larger than one. Please be sure that your local " + "environment has enough free CPUs for the calculation. Consider the following: " + "" + ) + assert ( + submit_step._submission_warning_messages.value + == submit_step._ALERT_MESSAGE.format(alert_class="warning", message=message) + ) + + # now we use a large structure, so we should have the Warning-1 (and 2 if not on localhost) + structure = generate_structure_data("H2O-larger") + submit_step.input_structure = structure + num_sites, volume = len(structure.sites), structure.get_cell_volume() + submit_step.codes["pw"].num_cpus.value = 1 + submit_step._check_resources() + message = ( + f"⚠ Warning: The selected structure has a large number of atoms ({num_sites}) " + f"or a significant cell volume ({int(volume)} Å3), making it computationally demanding " + "to run in a reasonable amount of time. Consider the following: " + "" + ) + assert ( + submit_step._submission_warning_messages.value + == submit_step._ALERT_MESSAGE.format(alert_class="warning", message=message) + ) + + def builder_to_readable_dict(builder): """transverse the builder and return a dictionary with readable values.""" from aiida import orm From ad726337c3f2e1fa24fcc46fb8458f04726ca016 Mon Sep 17 00:00:00 2001 From: Xing Wang Date: Tue, 8 Oct 2024 16:01:43 +0200 Subject: [PATCH 03/19] Show result tabs immediately when the process is finished. (#844) Use the process monitor with the `on_sealed` callbacks to update the workchain viewer when the process is finished. Since the callback functions are run in another thread, we should avoid using the AiiDA node directly inside the callback. We load the AiiDA node using the process uuid instead. --- .../app/result/workchain_viewer.py | 38 +++++++++++-------- tests/test_plugins_electronic_structure.py | 5 +++ tests/test_result.py | 4 ++ 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/aiidalab_qe/app/result/workchain_viewer.py b/src/aiidalab_qe/app/result/workchain_viewer.py index e0de4de48..dff95c155 100644 --- a/src/aiidalab_qe/app/result/workchain_viewer.py +++ b/src/aiidalab_qe/app/result/workchain_viewer.py @@ -25,13 +25,14 @@ @register_viewer_widget("process.workflow.workchain.WorkChainNode.") class WorkChainViewer(ipw.VBox): _results_shown = tl.Set() + process_uuid = tl.Unicode(allow_none=True) def __init__(self, node, **kwargs): if node.process_label != "QeAppWorkChain": super().__init__() return - self.node = node + self.process_uuid = node.uuid # In the new version of the plugin, the ui_parameters are stored as a yaml string # which is then converted to a dictionary ui_parameters = node.base.extras.get("ui_parameters", {}) @@ -41,12 +42,12 @@ def __init__(self, node, **kwargs): self.title = ipw.HTML( f"""
-

QE App Workflow (pk: {self.node.pk}) — - {self.node.inputs.structure.get_formula()} +

QE App Workflow (pk: {node.pk}) — + {node.inputs.structure.get_formula()}

""" ) - self.workflows_summary = SummaryView(self.node) + self.workflows_summary = SummaryView(node) self.summary_tab = ipw.VBox(children=[self.workflows_summary]) # Only the summary tab is shown by default @@ -59,7 +60,7 @@ def __init__(self, node, **kwargs): self.results = {} entries = get_entry_items("aiidalab_qe.properties", "result") for identifier, entry_point in entries.items(): - result = entry_point(self.node) + result = entry_point(node) self.results[identifier] = result self.results[identifier].identifier = identifier @@ -83,29 +84,36 @@ def toggle_camera(): toggle_camera() self.result_tabs.observe(on_selected_index_change, "selected_index") - self._update_view() super().__init__( children=[self.title, self.result_tabs], **kwargs, ) - self._process_monitor = ProcessMonitor( - process=self.node, - callbacks=[ + self.process_monitor = ProcessMonitor( + timeout=1.0, + on_sealed=[ self._update_view, ], ) + ipw.dlink((self, "process_uuid"), (self.process_monitor, "value")) + + @property + def node(self): + """Load the workchain node using the process_uuid. + Because the workchain node is used in another thread inside the process monitor, + we need to load the node from the database, instead of passing the node object. + Otherwise, we will get a "Instance is not persistent" error. + """ + return orm.load_node(self.process_uuid) def _update_view(self): with self.hold_trait_notifications(): - if self.node.is_finished: + node = self.node + if node.is_finished: self._show_workflow_output() # if the structure is present in the workchain, # the structure tab will be added. - if ( - "structure" not in self._results_shown - and "structure" in self.node.outputs - ): + if "structure" not in self._results_shown and "structure" in node.outputs: self._show_structure() self.result_tabs.children += (self.structure_tab,) # index of the last tab @@ -119,7 +127,7 @@ def _update_view(self): if result.identifier not in self._results_shown: # check if the all required results are in the outputs results_ready = [ - label in self.node.outputs for label in result.workchain_labels + label in node.outputs for label in result.workchain_labels ] if all(results_ready): result._update_view() diff --git a/tests/test_plugins_electronic_structure.py b/tests/test_plugins_electronic_structure.py index c64c4e778..f25351563 100644 --- a/tests/test_plugins_electronic_structure.py +++ b/tests/test_plugins_electronic_structure.py @@ -1,5 +1,7 @@ def test_electronic_structure(generate_qeapp_workchain): """Test the electronic structure tab.""" + import time + import plotly.graph_objects as go from aiida import engine @@ -10,7 +12,10 @@ def test_electronic_structure(generate_qeapp_workchain): wkchain = generate_qeapp_workchain() wkchain.node.set_exit_status(0) wkchain.node.set_process_state(engine.ProcessState.FINISHED) + wkchain.node.seal() wcv = WorkChainViewer(wkchain.node) + # wait for the tabs to be updated by the process monitor + time.sleep(3) # find the tab with the identifier "electronic_structure" # the built-in summary and structure tabs is not a plugin panel, # thus don't have identifiers diff --git a/tests/test_result.py b/tests/test_result.py index cbeef97ed..60b1ba34a 100644 --- a/tests/test_result.py +++ b/tests/test_result.py @@ -28,10 +28,14 @@ def test_kill_and_clean_buttons(app_to_submit, generate_qeapp_workchain): @pytest.mark.usefixtures("sssp") def test_workchainview(generate_qeapp_workchain): """Test the result tabs are properly updated""" + import time + from aiidalab_qe.app.result.workchain_viewer import WorkChainViewer wkchain = generate_qeapp_workchain() + wkchain.node.seal() wcv = WorkChainViewer(wkchain.node) + time.sleep(3) assert len(wcv.result_tabs.children) == 5 assert wcv.result_tabs._titles["0"] == "Workflow Summary" assert wcv.result_tabs._titles["1"] == "Final Geometry" From a7721426fa62fe031ed0af79f59e2e4760d61d14 Mon Sep 17 00:00:00 2001 From: Xing Wang Date: Wed, 9 Oct 2024 12:02:47 +0200 Subject: [PATCH 04/19] Replace workchain selector with the `Job History` button (#849) Merge the job list page into the app by adding a `Job History` button on the top so that the user can switch between the calculation and the job list page. * Add job list into the app wrapper * Remove work_chain_selector * Remove the header of the wizard app, and add a button to start a new calculation by opening a new qe app page. * lazy loading of job list --- qe.ipynb | 11 ++++---- src/aiidalab_qe/app/main.py | 33 +++++++++++++++--------- src/aiidalab_qe/app/utils/search_jobs.py | 24 +++++++++++------ src/aiidalab_qe/app/wrapper.py | 32 +++++++++++++++++++++++ tests/test_app.py | 4 +-- 5 files changed, 77 insertions(+), 27 deletions(-) diff --git a/qe.ipynb b/qe.ipynb index 58e173278..0ec99c175 100644 --- a/qe.ipynb +++ b/qe.ipynb @@ -89,13 +89,14 @@ "url = urlparse.urlsplit(jupyter_notebook_url) # noqa F821\n", "query = urlparse.parse_qs(url.query)\n", "\n", - "app_with_work_chain_selector = App(qe_auto_setup=True)\n", - "# if a pk is provided in the query string, set it as the value of the work_chain_selector\n", + "app = App(qe_auto_setup=True)\n", + "# if a pk is provided in the query string, set it as the process of the app\n", "if \"pk\" in query:\n", - " pk = int(query[\"pk\"][0])\n", - " app_with_work_chain_selector.work_chain_selector.value = pk\n", + " pk = query[\"pk\"][0]\n", + " app.process = pk\n", "\n", - "view.main.children = [app_with_work_chain_selector]" + "view.main.children = [app]\n", + "view.app = app" ] }, { diff --git a/src/aiidalab_qe/app/main.py b/src/aiidalab_qe/app/main.py index ab3d3d875..471cba3f0 100644 --- a/src/aiidalab_qe/app/main.py +++ b/src/aiidalab_qe/app/main.py @@ -4,19 +4,23 @@ """ import ipywidgets as ipw +import traitlets as tl +from IPython.display import Javascript, display from aiida.orm import load_node from aiidalab_qe.app.configuration import ConfigureQeAppWorkChainStep from aiidalab_qe.app.result import ViewQeAppWorkChainStatusAndResultsStep from aiidalab_qe.app.structure import StructureSelectionStep from aiidalab_qe.app.submission import SubmitQeAppWorkChainStep -from aiidalab_qe.common import QeAppWorkChainSelector from aiidalab_widgets_base import WizardAppWidget, WizardAppWidgetStep class App(ipw.VBox): """The main widget that combines all the application steps together.""" + # The PK or UUID of the work chain node. + process = tl.Union([tl.Unicode(), tl.Int()], allow_none=True) + def __init__(self, qe_auto_setup=True): # Create the application steps self.structure_step = StructureSelectionStep(auto_advance=True) @@ -64,23 +68,27 @@ def __init__(self, qe_auto_setup=True): ("Status & Results", self.results_step), ] ) + # hide the header + self._wizard_app_widget.children[0].layout.display = "none" self._wizard_app_widget.observe(self._observe_selected_index, "selected_index") - # Add process selection header - self.work_chain_selector = QeAppWorkChainSelector( - layout=ipw.Layout(width="auto") + # Add a button to start a new calculation + self.new_work_chains_button = ipw.Button( + description="Start New Calculation", + tooltip="Open a new page to start a separate calculation", + button_style="success", + icon="plus-circle", + layout=ipw.Layout(width="30%"), ) - self.work_chain_selector.observe(self._observe_process_selection, "value") - ipw.dlink( - (self.submit_step, "process"), - (self.work_chain_selector, "value"), - transform=lambda node: None if node is None else node.pk, - ) + def on_button_click(_): + display(Javascript("window.open('./qe.ipynb', '_blank')")) + + self.new_work_chains_button.on_click(on_button_click) super().__init__( children=[ - self.work_chain_selector, + self.new_work_chains_button, self._wizard_app_widget, ] ) @@ -119,7 +127,8 @@ def _observe_selected_index(self, change): ) self.submit_step.external_submission_blockers = blockers - def _observe_process_selection(self, change): + @tl.observe("process") + def _observe_process(self, change): from aiida.orm.utils.serialize import deserialize_unsafe if change["old"] == change["new"]: diff --git a/src/aiidalab_qe/app/utils/search_jobs.py b/src/aiidalab_qe/app/utils/search_jobs.py index d13271010..4288630a8 100644 --- a/src/aiidalab_qe/app/utils/search_jobs.py +++ b/src/aiidalab_qe/app/utils/search_jobs.py @@ -1,18 +1,20 @@ -import ipywidgets as ipw -import pandas as pd -from IPython.display import display - -from aiida.orm import QueryBuilder -from aiidalab_qe.workflows import QeAppWorkChain - - class QueryInterface: def __init__(self): + pass + + def setup_table(self): + import ipywidgets as ipw + self.df = self.load_data() self.table = ipw.HTML() self.setup_widgets() def load_data(self): + import pandas as pd + + from aiida.orm import QueryBuilder + from aiidalab_qe.workflows import QeAppWorkChain + projections = [ "id", "extras.structure", @@ -70,6 +72,8 @@ def load_data(self): ] def setup_widgets(self): + import ipywidgets as ipw + self.css_style = """