From ccf6a3c888258a68faa4aac1caea1c20a6eccbb5 Mon Sep 17 00:00:00 2001 From: Benedikt Daurer Date: Fri, 10 Mar 2023 16:36:43 +0000 Subject: [PATCH] Add new cloud-compatible CLI --- .dockerignore | 2 +- .github/workflows/containers.yml | 58 -------- .github/workflows/test.yml | 4 +- ptypy/accelerate/cuda_cupy/dependencies.yml | 1 + ptypy/io/cli.py | 138 ++++++++++++++++++++ pyproject.toml | 2 + 6 files changed, 144 insertions(+), 61 deletions(-) delete mode 100644 .github/workflows/containers.yml create mode 100644 ptypy/io/cli.py diff --git a/.dockerignore b/.dockerignore index 5e3ed6b58..ce3161d70 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,4 +4,4 @@ doc tutorial extra archive -*.md \ No newline at end of file +*.md diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml deleted file mode 100644 index 19e25ae5a..000000000 --- a/.github/workflows/containers.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: "Tests with Containers" - -on: - push: - branches: - - master - pull_request: - branches: - - master - - dev - - hotfixes - -jobs: - build-test: - runs-on: ubuntu-latest - strategy: - max-parallel: 10 - fail-fast: true - matrix: - python-version: ['3.7','3.8','3.9','3.10'] - platform: ['core', 'full'] # ['pycuda', 'cupy'] - name: Testing ${{ matrix.platform }}/python=${{ matrix.python-version }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build Image - uses: docker/build-push-action@v4 - with: - context: . - tags: ptypy_${{ matrix.platform }}_py${{ matrix.python-version }}_dev:latest - load: true - cache-from: type=gha - cache-to: type=gha,mode=max - target: runtime - build-args: | - PYTHON_VERSION=${{ matrix.python-version }} - PLATFORM=${{ matrix.platform }} - - name: Run test inside container - run: docker run ptypy_${{ matrix.platform }}_py${{ matrix.python-version }}_dev:latest pytest - - flake8-lint: - runs-on: ubuntu-latest - name: Lint - steps: - - name: Check out source repository - uses: actions/checkout@v3 - - name: Set up Python environment - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: flake8 Lint - uses: py-actions/flake8@v2 - with: - max-line-length: "120" - exclude: "./archive,./benchmark" - args: "--count --select=E9,F63,F7,F82 --show-source --statistics" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e68e79c3d..26d005c6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,8 +9,8 @@ on: pull_request: branches: - master -# - dev -# - hotfixes + - dev + - hotfixes # Also trigger on page_build, as well as release created events page_build: release: diff --git a/ptypy/accelerate/cuda_cupy/dependencies.yml b/ptypy/accelerate/cuda_cupy/dependencies.yml index 9b803c65f..27ab76b2f 100644 --- a/ptypy/accelerate/cuda_cupy/dependencies.yml +++ b/ptypy/accelerate/cuda_cupy/dependencies.yml @@ -11,6 +11,7 @@ dependencies: - mpi4py - pillow - pyfftw + - pyyaml - pip - compilers - cupy diff --git a/ptypy/io/cli.py b/ptypy/io/cli.py new file mode 100644 index 000000000..3cc35a081 --- /dev/null +++ b/ptypy/io/cli.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +""" +Wrapper to store nearly anything in an hdf5 file. + +This file is part of the PTYPY package. + + :copyright: Copyright 2014 by the PTYPY team, see AUTHORS. + :license: see LICENSE for details. +""" +import argparse +import json +import re +from collections.abc import MutableMapping + +from ptypy.core import Ptycho +from ptypy.utils.verbose import log +from ptypy import utils as u +from ptypy import load_ptyscan_module, load_gpu_engines + +def ptypy_run(): + u.verbose.set_level("info") + opt = parse() + run(opt) + +def parse() -> dict: + parser = argparse.ArgumentParser(description='A cloud-compatible command-line interface for ptypy') + config_group = parser.add_mutually_exclusive_group(required=True) + config_group.add_argument("-f", "--file", type=str, + help="Provide parameter configuration as a JSON or YAML file.") + config_group.add_argument("-j", "--json", type=str, + help="Provide parameter configuration as a JSON string.") + parser.add_argument('--output-folder', '-o', type=str, + help="The path we want the outputs to exist in (will get created).") + parser.add_argument('--ptypy-level', '-l', default=5, type=str, + help="The level we want to run to ptypy to.") + parser.add_argument('--identifier', '-i', type=str, default=None, + help="This is the same as p.run.") + parser.add_argument('--plot', '-p', action="store_true", + help="Turn on plotting") + parser.add_argument('--ptyscan-modules', '-s', nargs="+", default=None, + help="A list of ptyscan modules to be loaded") + parser.add_argument('--backends', '-b', nargs="+", default=None, + help="A list of CPU/GPU backends to be loaded") + return parser.parse_args() + +def run(args): + + # Load PtyScan modules + if args.ptyscan_modules is not None: + for ptyscan in args.ptyscan_modules: + load_ptyscan_module(ptyscan) + + # Load CPU/GPU backends + if args.backends is not None: + for backend in args.backends: + load_gpu_engines(backend) + + # Load parameter tree from file or JSON string + if args.file: + p = create_parameter_tree(load_config_as_dict_from_file(args.file)) + if args.json: + p = create_parameter_tree(json.loads(args.json)) + p.run = args.identifier + + # TODO + if args.plot: + log("info", "Turning the plotting on.") + #parameters.io.autoplot = u.Param(active=True) + pass + else: + log("info", "Turning the plotting off.") + p.io.autoplot = u.Param(active=False) + + # + if args.output_folder is not None: + p.io.home = get_output_file_name(args) + p.io.rfile = "%s.ptyr" % get_output_file_name(args) + #parameters.io.autosave = u.Param(active=True) + #log(3, "Autosave is on, with io going in {}, and the final reconstruction into {}".format(parameters.io.home, + # parameters.io.rfile)) + else: + p.io.rfile = None + p.io.autosave = u.Param(active=False) + log("info", "Autosave is off. No output will be saved.") + + # Substitute %(run) with in ptyscan + substitute_id_in_ptyscan(p) + + # Run PtyPy to given level + P = Ptycho(p, level=args.ptypy_level) + + return P + + +def load_config_as_dict_from_file(config_file) -> dict: + if config_file.endswith((".json", ".jsn")): + param_dict = u.param_from_json(config_file) + elif config_file.endswith((".yaml", "yml")): + param_dict = u.param_from_yaml(config_file) + else: + raise FileExistsError(f"Cannot parse {config_file}, expecting a JSON or YAML config file") + return param_dict + +def create_parameter_tree(params) -> u.Param: + parameters_to_run = u.Param() + if params['base_file'] is not None: + log(3, "Basing this scan off of the scan in {}".format(params['base_file'])) + previous_scan = Ptycho.load_run(params['base_file'], False) # load in the run but without the data + previous_parameters = previous_scan.p + parameters_to_run.update(previous_parameters) + if params['parameter_tree'] is not None: + parameters_to_run.update(params['parameter_tree'], Convert=True) + return parameters_to_run + +def substitute_id_in_ptyscan(params): + def _substitute(d, p): + for k, v in d.items(): + if isinstance(v, MutableMapping): + _substitute(v, p) + elif isinstance(v, str) and re.search('%\(\w+\)s', v) is not None: + # only perform substitution when the format %(...)s is present + d[k] = v % p + for scan_key, scan in params.scans.items(): + data_entry = scan.data + _substitute(data_entry, params) + +def get_output_file_name(args): + from datetime import datetime + now = datetime.now() + if args.identifier is not None: + output_path = "{}/scan_{}_{}".format(args.output_folder, args.identifier, now.strftime("%Y%m%d%H%M%S")) + else: + output_path = "{}/scan_{}".format(args.output_folder, now.strftime("%Y%m%d%H%M%S")) + log("info", "Output is going in: {}".format(output_path)) + return output_path + +if __name__ == "__main__": + ptypy_run() diff --git a/pyproject.toml b/pyproject.toml index 635431745..c6ed4de7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,8 @@ dependencies = [ [project.optional-dependencies] full = ["mpi4py","matplotlib","pyzmq","pillow", "pyfftw"] +[project.scripts] +"ptypy.cli" = "ptypy.io.cli:ptypy_run" #[project.scripts] #"ptypy.plot" = 'scripts/ptypy.plot'