diff --git a/README.md b/README.md index 36c49230..2e0e011c 100644 --- a/README.md +++ b/README.md @@ -80,12 +80,22 @@ Related Links How to get started with pyEMU ============================= -pyEMU is available through pyPI: +pyEMU is available through pyPI and conda. To install pyEMU type: -`>>>pip install pyemu` + >>>conda install -c conda-forge pyemu + +or + + >>>pip install pyemu pyEMU needs `numpy` and `pandas`. For plotting, `matplotloib`, `pyshp`, and `flopy` to take advantage of the auto interface construction +After pyEMU is installed, the PEST++ software suite can be installed for your operating system using the command: + + get-pestpp :pyemu + +See [documentation](get_pestpp.md) for more information. + Found a bug? Got a smart idea? Contributions welcome. ==================================================== Feel free to raise and issue or submit a pull request. diff --git a/autotest/conftest.py b/autotest/conftest.py index c47e3120..581840ba 100644 --- a/autotest/conftest.py +++ b/autotest/conftest.py @@ -1,6 +1,9 @@ +from pathlib import Path import pytest from pst_from_tests import freybergmf6_2_pstfrom +pytest_plugins = ["modflow_devtools.fixtures"] + collect_ignore = [ # "utils_tests.py", @@ -16,5 +19,3 @@ # "mat_tests.py", # "da_tests.py" ] - - diff --git a/autotest/get_pestpp_tests.py b/autotest/get_pestpp_tests.py new file mode 100644 index 00000000..a3d1b330 --- /dev/null +++ b/autotest/get_pestpp_tests.py @@ -0,0 +1,276 @@ +"""Test get-pestpp utility.""" +import os +import platform +import sys +from os.path import expandvars +from pathlib import Path +from platform import system +from urllib.error import HTTPError + +import pytest +from flaky import flaky +from modflow_devtools.markers import requires_github +from modflow_devtools.misc import run_py_script +from pyemu.utils import get_pestpp +from pyemu.utils.get_pestpp import get_release, get_releases, select_bindir + +rate_limit_msg = "rate limit exceeded" +get_pestpp_script = ( + Path(__file__).parent.parent / "pyemu" / "utils" / "get_pestpp.py" +) +bindir_options = { + "pyemu": Path(expandvars(r"%LOCALAPPDATA%\pyemu")) / "bin" + if system() == "Windows" + else Path.home() / ".local" / "share" / "pyemu" / "bin", + "python": Path(sys.prefix) + / ("Scripts" if system() == "Windows" else "bin"), + "home": Path.home() / ".local" / "bin", +} +owner_options = [ + "usgs", +] +repo_options = { + "pestpp": [ + "pestpp-da", + "pestpp-glm", + "pestpp-ies", + "pestpp-mou", + "pestpp-opt", + "pestpp-sen", + "pestpp-sqp", + "pestpp-swp", + ], +} + +if system() == "Windows": + bindir_options["windowsapps"] = Path( + expandvars(r"%LOCALAPPDATA%\Microsoft\WindowsApps") + ) +else: + bindir_options["system"] = Path("/usr") / "local" / "bin" + + +@pytest.fixture +def downloads_dir(tmp_path_factory): + downloads_dir = tmp_path_factory.mktemp("Downloads") + return downloads_dir + + +@pytest.fixture(autouse=True) +def create_home_local_bin(): + # make sure $HOME/.local/bin exists for :home option + home_local = Path.home() / ".local" / "bin" + home_local.mkdir(parents=True, exist_ok=True) + + +def run_get_pestpp_script(*args): + return run_py_script(get_pestpp_script, *args, verbose=True) + + +def append_ext(path: str): + if system() == "Windows": + return f"{path}{'.exe'}" + elif system() == "Darwin": + return f"{path}{''}" + elif system() == "Linux": + return f"{path}{''}" + + +@pytest.mark.parametrize("per_page", [-1, 0, 101, 1000]) +def test_get_releases_bad_page_size(per_page): + with pytest.raises(ValueError): + get_releases(repo="pestpp", per_page=per_page) + + +@flaky +@requires_github +@pytest.mark.parametrize("repo", repo_options.keys()) +def test_get_releases(repo): + releases = get_releases(repo=repo) + assert "latest" in releases + + +@flaky +@requires_github +@pytest.mark.parametrize("repo", repo_options.keys()) +def test_get_release(repo): + tag = "latest" + release = get_release(repo=repo, tag=tag) + assets = release["assets"] + release_tag_name = release["tag_name"] + + expected_assets = [ + f"pestpp-{release_tag_name}-linux.tar.gz", + f"pestpp-{release_tag_name}-imac.tar.gz", + f"pestpp-{release_tag_name}-iwin.zip", + ] + expected_ostags = [a.replace(".zip", "") for a in expected_assets] + expected_ostags = [a.replace("tar.gz", "") for a in expected_assets] + actual_assets = [asset["name"] for asset in assets] + + if repo == "pestpp": + # can remove if modflow6 releases follow asset name conventions followed in executables and nightly build repos + assert {a.rpartition("_")[2] for a in actual_assets} >= { + a for a in expected_assets if not a.startswith("win") + } + else: + for ostag in expected_ostags: + assert any( + ostag in a for a in actual_assets + ), f"dist not found for {ostag}" + + +@pytest.mark.parametrize("bindir", bindir_options.keys()) +def test_select_bindir(bindir, function_tmpdir): + expected_path = bindir_options[bindir] + if not os.access(expected_path, os.W_OK): + pytest.skip(f"{expected_path} is not writable") + selected = select_bindir(f":{bindir}") + + if system() != "Darwin": + assert selected == expected_path + else: + # for some reason sys.prefix can return different python + # installs when invoked here and get_modflow.py on macOS + # https://github.com/modflowpy/flopy/actions/runs/3331965840/jobs/5512345032#step:8:1835 + # + # work around by just comparing the end of the bin path + # should be .../Python.framework/Versions//bin + assert selected.parts[-4:] == expected_path.parts[-4:] + + +def test_script_help(): + assert get_pestpp_script.exists() + stdout, stderr, returncode = run_get_pestpp_script("-h") + assert "usage" in stdout + assert len(stderr) == 0 + assert returncode == 0 + + +@flaky +@requires_github +def test_script_invalid_options(function_tmpdir, downloads_dir): + # try with bindir that doesn't exist + bindir = function_tmpdir / "bin1" + assert not bindir.exists() + stdout, stderr, returncode = run_get_pestpp_script(bindir) + if rate_limit_msg in stderr: + pytest.skip(f"GitHub {rate_limit_msg}") + assert "does not exist" in stderr + assert returncode == 1 + + # attempt to fetch a non-existing release-id + bindir.mkdir() + assert bindir.exists() + stdout, stderr, returncode = run_get_pestpp_script( + bindir, "--release-id", "1.9", "--downloads-dir", downloads_dir + ) + if rate_limit_msg in stderr: + pytest.skip(f"GitHub {rate_limit_msg}") + assert "Release 1.9 not found" in stderr + assert returncode == 1 + + # try to select an invalid --subset + bindir = function_tmpdir / "bin2" + bindir.mkdir() + stdout, stderr, returncode = run_get_pestpp_script( + bindir, "--subset", "pestpp-opt,mpx", "--downloads-dir", downloads_dir + ) + if rate_limit_msg in stderr: + pytest.skip(f"GitHub {rate_limit_msg}") + assert "subset item not found: mpx" in stderr + assert returncode == 1 + + +@flaky +@requires_github +def test_script_valid_options(function_tmpdir, downloads_dir): + # fetch latest + bindir = function_tmpdir / "bin1" + bindir.mkdir() + stdout, stderr, returncode = run_get_pestpp_script( + bindir, "--downloads-dir", downloads_dir + ) + if rate_limit_msg in stderr: + pytest.skip(f"GitHub {rate_limit_msg}") + assert len(stderr) == returncode == 0 + files = [item.name for item in bindir.iterdir() if item.is_file()] + assert len(files) == 8 + + # valid subset + bindir = function_tmpdir / "bin2" + bindir.mkdir() + stdout, stderr, returncode = run_get_pestpp_script( + bindir, + "--subset", + "pestpp-da,pestpp-swp,pestpp-ies", + "--downloads-dir", + downloads_dir, + ) + if rate_limit_msg in stderr: + pytest.skip(f"GitHub {rate_limit_msg}") + assert len(stderr) == returncode == 0 + files = [item.stem for item in bindir.iterdir() if item.is_file()] + assert sorted(files) == ["pestpp-da", "pestpp-ies", "pestpp-swp"] + + # similar as before, but also specify a ostag + bindir = function_tmpdir / "bin3" + bindir.mkdir() + stdout, stderr, returncode = run_get_pestpp_script( + bindir, + "--subset", + "pestpp-ies", + "--release-id", + "5.2.6", + "--ostag", + "win", + "--downloads-dir", + downloads_dir, + ) + if rate_limit_msg in stderr: + pytest.skip(f"GitHub {rate_limit_msg}") + assert len(stderr) == returncode == 0 + files = [item.name for item in bindir.iterdir() if item.is_file()] + assert sorted(files) == ["pestpp-ies.exe"] + + +@flaky +@requires_github +@pytest.mark.parametrize("owner", owner_options) +@pytest.mark.parametrize("repo", repo_options.keys()) +def test_script(function_tmpdir, owner, repo, downloads_dir): + bindir = str(function_tmpdir) + stdout, stderr, returncode = run_get_pestpp_script( + bindir, + "--owner", + owner, + "--repo", + repo, + "--downloads-dir", + downloads_dir, + ) + if rate_limit_msg in stderr: + pytest.skip(f"GitHub {rate_limit_msg}") + + paths = list(function_tmpdir.glob("*")) + names = [p.name for p in paths] + expected_names = [append_ext(p) for p in repo_options[repo]] + assert set(names) >= set(expected_names) + + +@flaky +@requires_github +@pytest.mark.parametrize("owner", owner_options) +@pytest.mark.parametrize("repo", repo_options.keys()) +def test_python_api(function_tmpdir, owner, repo, downloads_dir): + bindir = str(function_tmpdir) + try: + get_pestpp(bindir, owner=owner, repo=repo, downloads_dir=downloads_dir) + except HTTPError as err: + if err.code == 403: + pytest.skip(f"GitHub {rate_limit_msg}") + + paths = list(function_tmpdir.glob("*")) + names = [p.name for p in paths] + expected_names = [append_ext(p) for p in repo_options[repo]] + assert set(names) >= set(expected_names) diff --git a/etc/environment.yml b/etc/environment.yml index 37e238d7..1efd1df8 100644 --- a/etc/environment.yml +++ b/etc/environment.yml @@ -5,16 +5,18 @@ dependencies: # required - python>=3.8 - numpy>=1.15.0 - - pandas + - pandas<2.1.0 # optional - matplotlib>=1.4.0 - pyshp - jinja2 # tests - coveralls + - flaky - pytest - pytest-cov - pytest-xdist - nbmake - shapely - pyproj + - modflow-devtools diff --git a/get_pestpp.md b/get_pestpp.md new file mode 100644 index 00000000..199ef7da --- /dev/null +++ b/get_pestpp.md @@ -0,0 +1,89 @@ +# Install the PEST++ software suite + + + + + +- [Command-line interface](#command-line-interface) + - [Using the `get-pestpp` command](#using-the-get-pestpp-command) + - [Using `get_pestpp.py` as a script](#using-get_pestpppy-as-a-script) +- [pyEMU module](#pyEMU-module) +- [Where to install?](#where-to-install) +- [Selecting a distribution](#selecting-a-distribution) + + + +pyEMU includes a `get-pestpp` utility to install the PEST++ software suite for Windows, Mac or Linux. If pyEMU is installed, the utility is available in the Python environment as a `get-pestpp` command. The script `pyEMU/utils/get_pestpp.py` has no dependencies and can be invoked independently. + +The utility uses the [GitHub releases API](https://docs.github.com/en/rest/releases) to download versioned archives containing PEST++ executables. The utility is able to match the binary archive to the operating system and extract the console programs to a user-defined directory. A prompt can also be used to help the user choose where to install programs. + +## Command-line interface + +### Using the `get-pestpp` command + +When pyEMU is installed, a `get-pestpp` (or `get-pestpp.exe` for Windows) program is installed, which is usually installed to the PATH (depending on the Python setup). From a console: + +```console +$ get-pestpp --help +usage: get-pestpp [-h] +... +``` + +### Using `get_pestpp.py` as a script + +The script requires Python 3.6 or later and does not have any dependencies, not even pyEMU. It can be downloaded separately and used the same as the console program, except with a different invocation. For example: + +```console +$ wget https://raw.githubusercontent.com/modflowpy/pyEMU/develop/pyEMU/utils/get_pestpp.py +$ python3 get_pestpp.py --help +usage: get_pestpp.py [-h] +... +``` + +## pyEMU module + +The same functionality of the command-line interface is available from the pyEMU module, as demonstrated below: + +```python +from pathlib import Path +import pyemu + +bindir = Path("/tmp/bin") +bindir.mkdir(exist_ok=True) +pymu.utils.get_pestpp(bindir) +list(bindir.iterdir()) + +# Or use an auto-select option +pyemu.utils.get_pestpp(":pyemu") +``` + +## Where to install? + +A required `bindir` parameter must be supplied to the utility, which specifies where to install the programs. This can be any existing directory, usually which is on the users' PATH environment variable. + +To assist the user, special values can be specified starting with the colon character. Use a single `:` to interactively select an option of paths. + +Other auto-select options are only available if the current user can write files (some may require `sudo` for Linux or macOS): + - `:prev` - if this utility was run by pyEMU more than once, the first option will be the previously used `bindir` path selection + - `:pyemu` - special option that will create and install programs for pyEMU + - `:python` - use Python's bin (or Scripts) directory + - `:home` - use `$HOME/.local/bin` + - `:system` - use `/usr/local/bin` + - `:windowsapps` - use `%LOCALAPPDATA%\Microsoft\WindowsApps` + +## Selecting a distribution + +By default the distribution from the [`usgs/pestpp` repository](https://github.com/usgs/pestpp) is installed. This includes the MODFLOW 6 binary `mf6` and over 20 other related programs. The utility can also install from forks of the main [PEST++ 6 repo](https://github.com/usgs/pestpp) or other repo distributions, which contain only: + +- `pestpp-da` +- `pestpp-glm` +- `pestpp-ies` +- `pestpp-mou` +- `pestpp-opt` +- `pestpp-sen` +- `pestpp-sqp` +- `pestpp-swp` + +To select a distribution, specify a repository name with the `--repo` command line option or the `repo` function argument. + +The repository owner can also be configured with the `--owner` option. This can be useful for installing from unreleased PEST++ software suite feature branches still in development — the only compatibility requirement is that release assets be named identically to those on the official repositories. diff --git a/pyemu/__init__.py b/pyemu/__init__.py index 387e72a1..2e1b6b22 100644 --- a/pyemu/__init__.py +++ b/pyemu/__init__.py @@ -7,28 +7,19 @@ high-dimensional ensemble generation. """ -from .la import LinearAnalysis -from .sc import Schur -from .ev import ErrVar -from .en import Ensemble, ParameterEnsemble, ObservationEnsemble from .eds import EnDS - +from .en import Ensemble, ObservationEnsemble, ParameterEnsemble +from .ev import ErrVar +from .la import LinearAnalysis +from .logger import Logger # from .mc import MonteCarlo # from .inf import Influence -from .mat import Matrix, Jco, Cov -from .pst import Pst, pst_utils -from .utils import ( - helpers, - gw_utils, - optimization, - geostats, - pp_utils, - os_utils, - smp_utils, - metrics, -) +from .mat import Cov, Jco, Matrix from .plot import plot_utils -from .logger import Logger +from .pst import Pst, pst_utils +from .sc import Schur +from .utils import (geostats, gw_utils, helpers, metrics, optimization, + os_utils, pp_utils, smp_utils) #from .prototypes import * try: diff --git a/pyemu/utils/__init__.py b/pyemu/utils/__init__.py index 3f5206c3..8486b78b 100644 --- a/pyemu/utils/__init__.py +++ b/pyemu/utils/__init__.py @@ -4,11 +4,14 @@ functionality dedicated to wrapping MODFLOW models into the PEST(++) model independent framework """ -from .helpers import * +from . import get_pestpp as get_pestpp_module from .geostats import * -from .pp_utils import * from .gw_utils import * +from .helpers import * +from .metrics import * from .os_utils import * -from .smp_utils import * +from .pp_utils import * from .pst_from import * -from .metrics import * +from .smp_utils import * + +get_pestpp = get_pestpp_module.run_main diff --git a/pyemu/utils/get_pestpp.py b/pyemu/utils/get_pestpp.py new file mode 100755 index 00000000..69f46f64 --- /dev/null +++ b/pyemu/utils/get_pestpp.py @@ -0,0 +1,790 @@ +#!/usr/bin/env python3 +"""Download and install the PEST++ software suite. + +This script originates from pyemu: https://github.com/pypest/pyemu +This file can be downloaded and run independently outside pyemu. +It requires Python 3.6 or later, and has no dependencies. + +See https://developer.github.com/v3/repos/releases/ for GitHub Releases API. +""" +import json +import os +import shutil +import sys +import tarfile +import tempfile +import urllib +import urllib.request +import warnings +import zipfile +from importlib.util import find_spec +from pathlib import Path + +__all__ = ["run_main"] +__license__ = "CC0" + +from typing import Dict, List, Tuple + +default_owner = "usgs" +default_repo = "pestpp" +# key is the repo name, value is the renamed file prefix for the download +renamed_prefix = { + "pestpp": "pestpp", +} +available_repos = list(renamed_prefix.keys()) +available_ostags = ["linux", "mac", "win"] +max_http_tries = 3 + +# Check if this is running from pyemu +within_pyemu = False +spec = find_spec("pyemu") +if spec is not None: + within_pyemu = ( + Path(spec.origin).resolve().parent in Path(__file__).resolve().parents + ) +del spec + +# local pyemu install location (selected with :pyemu) +pyemu_appdata_path = ( + Path(os.path.expandvars(r"%LOCALAPPDATA%\pyemu")) + if sys.platform.startswith("win") + else Path.home() / ".local" / "share" / "pyemu" +) + + +def get_ostag() -> str: + """Determine operating system tag from sys.platform.""" + if sys.platform.startswith("linux"): + return "linux" + elif sys.platform.startswith("win"): + return "win" + elif sys.platform.startswith("darwin"): + return "mac" + raise ValueError(f"platform {sys.platform!r} not supported") + + +def get_suffixes(ostag) -> Tuple[str, str]: + if ostag in ["win"]: + return ".exe", ".dll" + elif ostag == "linux": + return "", ".so" + elif ostag == "mac": + return "", ".dylib" + else: + raise KeyError( + f"unrecognized ostag {ostag!r}; choose one of {available_ostags}" + ) + + +def get_request(url, params={}): + """Get urllib.request.Request, with parameters and headers. + + This bears GITHUB_TOKEN if it is set as an environment variable. + """ + if isinstance(params, dict): + if len(params) > 0: + url += "?" + urllib.parse.urlencode(params) + else: + raise TypeError("data must be a dict") + headers = {} + github_token = os.environ.get("GITHUB_TOKEN") + if github_token: + headers["Authorization"] = f"Bearer {github_token}" + return urllib.request.Request(url, headers=headers) + + +def get_releases( + owner=None, repo=None, quiet=False, per_page=None +) -> List[str]: + """Get list of available releases.""" + owner = default_owner if owner is None else owner + repo = default_repo if repo is None else repo + req_url = f"https://api.github.com/repos/{owner}/{repo}/releases" + + params = {} + if per_page is not None: + if per_page < 1 or per_page > 100: + raise ValueError("per_page must be between 1 and 100") + params["per_page"] = per_page + + request = get_request(req_url, params=params) + num_tries = 0 + while True: + num_tries += 1 + try: + with urllib.request.urlopen(request, timeout=10) as resp: + result = resp.read() + break + except urllib.error.HTTPError as err: + if err.code == 401 and os.environ.get("GITHUB_TOKEN"): + raise ValueError("GITHUB_TOKEN env is invalid") from err + elif err.code == 403 and "rate limit exceeded" in err.reason: + raise ValueError( + f"use GITHUB_TOKEN env to bypass rate limit ({err})" + ) from err + elif err.code in (404, 503) and num_tries < max_http_tries: + # GitHub sometimes returns this error for valid URLs, so retry + print(f"URL request {num_tries} did not work ({err})") + continue + raise RuntimeError(f"cannot retrieve data from {req_url}") from err + + releases = json.loads(result.decode()) + if not quiet: + print(f"found {len(releases)} releases for {owner}/{repo}") + + avail_releases = ["latest"] + avail_releases.extend(release["tag_name"] for release in releases) + return avail_releases + + +def get_release(owner=None, repo=None, tag="latest", quiet=False) -> dict: + """Get info about a particular release.""" + owner = default_owner if owner is None else owner + repo = default_repo if repo is None else repo + api_url = f"https://api.github.com/repos/{owner}/{repo}" + req_url = ( + f"{api_url}/releases/latest" + if tag == "latest" + else f"{api_url}/releases/tags/{tag}" + ) + request = get_request(req_url) + releases = None + num_tries = 0 + + while True: + num_tries += 1 + try: + with urllib.request.urlopen(request, timeout=10) as resp: + result = resp.read() + remaining = int(resp.headers["x-ratelimit-remaining"]) + if remaining <= 10: + warnings.warn( + f"Only {remaining} GitHub API requests remaining " + "before rate-limiting" + ) + break + except urllib.error.HTTPError as err: + if err.code == 401 and os.environ.get("GITHUB_TOKEN"): + raise ValueError("GITHUB_TOKEN env is invalid") from err + elif err.code == 403 and "rate limit exceeded" in err.reason: + raise ValueError( + f"use GITHUB_TOKEN env to bypass rate limit ({err})" + ) from err + elif err.code == 404: + if releases is None: + releases = get_releases(owner, repo, quiet) + if tag not in releases: + raise ValueError( + f"Release {tag} not found (choose from {', '.join(releases)})" + ) + elif err.code == 503 and num_tries < max_http_tries: + # GitHub sometimes returns this error for valid URLs, so retry + warnings.warn(f"URL request {num_tries} did not work ({err})") + continue + raise RuntimeError(f"cannot retrieve data from {req_url}") from err + + release = json.loads(result.decode()) + tag_name = release["tag_name"] + if not quiet: + print(f"\nFetched release {tag_name!r} info from '{owner}/{repo}'.") + + return release + + +def columns_str(items, line_chars=79) -> str: + """Return str of columns of items, similar to 'ls' command.""" + item_chars = max(len(item) for item in items) + num_cols = line_chars // item_chars + if num_cols == 0: + num_cols = 1 + num_rows = len(items) // num_cols + if len(items) % num_cols != 0: + num_rows += 1 + lines = [] + for row_num in range(num_rows): + row_items = items[row_num::num_rows] + lines.append( + " ".join(item.ljust(item_chars) for item in row_items).rstrip() + ) + return "\n".join(lines) + + +def get_bindir_options(previous=None) -> Dict[str, Tuple[Path, str]]: + """Generate install location options based on platform and filesystem access.""" + options = {} # key is an option name, value is (optpath, optinfo) + if previous is not None and os.access(previous, os.W_OK): + # Make previous bindir as the first option + options[":prev"] = (previous, "previously selected bindir") + if within_pyemu: # don't check is_dir() or access yet + options[":pyemu"] = (pyemu_appdata_path / "bin", "used by pyemu") + # Python bin (same for standard or conda varieties) + py_bin = Path(sys.prefix) / ( + "Scripts" if get_ostag().startswith("win") else "bin" + ) + if py_bin.is_dir() and os.access(py_bin, os.W_OK): + options[":python"] = (py_bin, "used by Python") + home_local_bin = Path.home() / ".local" / "bin" + if home_local_bin.is_dir() and os.access(home_local_bin, os.W_OK): + options[":home"] = (home_local_bin, "user-specific bindir") + local_bin = Path("/usr") / "local" / "bin" + if local_bin.is_dir() and os.access(local_bin, os.W_OK): + options[":system"] = (local_bin, "system local bindir") + # Windows user + windowsapps_dir = Path( + os.path.expandvars(r"%LOCALAPPDATA%\Microsoft\WindowsApps") + ) + if windowsapps_dir.is_dir() and os.access(windowsapps_dir, os.W_OK): + options[":windowsapps"] = (windowsapps_dir, "User App path") + + # any other possible OS-specific hard-coded locations? + if not options: + raise RuntimeError("could not find any installable folders") + + return options + + +def select_bindir(bindir, previous=None, quiet=False, is_cli=False) -> Path: + """Resolve an install location if provided, or prompt interactive user to select one.""" + options = get_bindir_options(previous) + + if len(bindir) > 1: # auto-select mode + # match one option that starts with input, e.g. :Py -> :python + sel = list(opt for opt in options if opt.startswith(bindir.lower())) + if len(sel) != 1: + opt_avail = ", ".join( + f"'{opt}' for '{optpath}'" + for opt, (optpath, _) in options.items() + ) + raise ValueError( + f"invalid option '{bindir}', choose from: {opt_avail}" + ) + if not quiet: + print(f"auto-selecting option {sel[0]!r} for 'bindir'") + return Path(options[sel[0]][0]).resolve() + else: + if not is_cli: + opt_avail = ", ".join( + f"'{opt}' for '{optpath}'" + for opt, (optpath, _) in options.items() + ) + raise ValueError(f"specify the option, choose from: {opt_avail}") + + ioptions = dict(enumerate(options.keys(), 1)) + print("select a number to extract executables to a directory:") + for iopt, opt in ioptions.items(): + optpath, optinfo = options[opt] + print(f" {iopt}: '{optpath}' -- {optinfo} ('{opt}')") + num_tries = 0 + while True: + num_tries += 1 + res = input("> ") + try: + opt = ioptions[int(res)] + print(f"selecting option {opt!r}") + return Path(options[opt][0]).resolve() + except (KeyError, ValueError): + if num_tries < 2: + print("invalid option, try choosing option again") + else: + raise RuntimeError( + "invalid option, too many attempts" + ) from None + + +def run_main( + bindir, + owner=default_owner, + repo=default_repo, + release_id="latest", + ostag=None, + subset=None, + downloads_dir=None, + force=False, + quiet=False, + _is_cli=False, +): + """Run main method to get the PEST++ software suite. + + Parameters + ---------- + bindir : str or Path + Writable path to extract executables. Auto-select options start with a + colon character. See error message or other documentation for further + information on auto-select options. + owner : str, default "usgs" + Name of GitHub repository owner (user or organization). + repo : str, default "pestpp" + Name of GitHub PEST++ repository. + release_id : str, default "latest" + GitHub release ID. + ostag : str, optional + Operating system tag; default is to automatically choose. + subset : list, set or str, optional + Optional subset of executables to extract, specified as a list (e.g.) + ``["pestpp-glm", "pestpp-ies"]`` or a comma-separated string + "pestpp-glm,pestpp-ies". + downloads_dir : str or Path, optional + Manually specify directory to download archives. Default is to use + home Downloads, if available, otherwise a temporary directory. + force : bool, default False + If True, always download archive. Default False will use archive if + previously downloaded in ``downloads_dir``. + quiet : bool, default False + If True, show fewer messages. + _is_cli : bool, default False + Control behavior of method if this is run as a command-line interface + or as a Python function. + """ + meta_path = False + prev_bindir = None + pyemu_bin = False + if within_pyemu: + meta_list = [] + # Store metadata and possibly 'bin' in a user-writable path + if not pyemu_appdata_path.exists(): + pyemu_appdata_path.mkdir(parents=True, exist_ok=True) + pyemu_bin = pyemu_appdata_path / "bin" + meta_path = pyemu_appdata_path / "get_pestpp.json" + meta_path_exists = meta_path.exists() + if meta_path_exists: + del_meta_path = False + try: + meta_list = json.loads(meta_path.read_text()) + except (OSError, json.JSONDecodeError) as err: + print(f"cannot read pyemu metadata file '{meta_path}': {err}") + if isinstance(err, OSError): + meta_path = False + if isinstance(err, json.JSONDecodeError): + del_meta_path = True + try: + prev_bindir = Path(meta_list[-1]["bindir"]) + except (KeyError, IndexError): + del_meta_path = True + if del_meta_path: + try: + meta_path.unlink() + meta_path_exists = False + print(f"removed corrupt pyemu metadata file '{meta_path}'") + except OSError as err: + print(f"cannot remove pyemu metadata file: {err!r}") + meta_path = False + + if ostag is None: + ostag = get_ostag() + + exe_suffix, lib_suffix = get_suffixes(ostag) + + # select bindir if path not provided + if bindir.startswith(":"): + bindir = select_bindir( + bindir, previous=prev_bindir, quiet=quiet, is_cli=_is_cli + ) + elif not isinstance(bindir, (str, Path)): + raise ValueError("Invalid bindir option (expected string or Path)") + bindir = Path(bindir).resolve() + + # make sure bindir exists + if bindir == pyemu_bin: + if not within_pyemu: + raise ValueError("option ':pyemu' is only for pyemu") + elif not pyemu_bin.exists(): + # special case option that can create non-existing directory + pyemu_bin.mkdir(parents=True, exist_ok=True) + if not bindir.is_dir(): + raise OSError(f"extraction directory '{bindir}' does not exist") + elif not os.access(bindir, os.W_OK): + raise OSError(f"extraction directory '{bindir}' is not writable") + + # make sure repo option is valid + if repo not in available_repos: + raise KeyError( + f"repo {repo!r} not supported; choose one of {available_repos}" + ) + + # get the selected release + release = get_release(owner, repo, release_id, quiet) + assets = release.get("assets", []) + + inconsistent_ostag_dict = { + "win": "iwin", + "mac": "imac", + "linux": "linux", + } + + for asset in assets: + if inconsistent_ostag_dict[ostag] in asset["name"]: + break + else: + raise ValueError( + f"could not find ostag {ostag!r} from release {release['tag_name']!r}; " + f"see available assets here:\n{release['html_url']}" + ) + asset_name = asset["name"] + download_url = asset["browser_download_url"] + if repo == "pestpp": + asset_pth = Path(asset_name) + asset_stem = asset_pth.stem + if str(asset_pth).endswith("tar.gz"): + asset_suffix = ".tar.gz" + else: + asset_suffix = asset_pth.suffix + dst_fname = "-".join([repo, release["tag_name"], ostag]) + asset_suffix + else: + # change local download name so it is more unique + dst_fname = "-".join( + [renamed_prefix[repo], release["tag_name"], asset_name] + ) + tmpdir = None + if downloads_dir is None: + downloads_dir = Path.home() / "Downloads" + if not (downloads_dir.is_dir() and os.access(downloads_dir, os.W_OK)): + tmpdir = tempfile.TemporaryDirectory() + downloads_dir = Path(tmpdir.name) + else: # check user-defined + downloads_dir = Path(downloads_dir) + if not downloads_dir.is_dir(): + raise OSError( + f"downloads directory '{downloads_dir}' does not exist" + ) + elif not os.access(downloads_dir, os.W_OK): + raise OSError( + f"downloads directory '{downloads_dir}' is not writable" + ) + download_pth = downloads_dir / dst_fname + if download_pth.is_file() and not force: + if not quiet: + print( + f"using previous download '{download_pth}' (use " + f"{'--force' if _is_cli else 'force=True'!r} to re-download)" + ) + else: + if not quiet: + print(f"\nDownloading '{download_url}' to '{download_pth}'.") + urllib.request.urlretrieve(download_url, download_pth) + + if subset: + if not isinstance(subset, (str, list, tuple)): + raise TypeError( + "subset but be a comma-separated string, list, or tuple" + ) + print(subset) + if isinstance(subset, str): + for rep_text in ("'", '"'): + subset = subset.replace(rep_text, "") + subset = subset.split(sep=",") + if ostag in ("win",): + for idx, entry in enumerate(subset): + if entry.startswith("pestpp") and not entry.endswith( + exe_suffix + ): + subset[idx] = f"{entry}{exe_suffix}" + subset = set(subset) + + # Open archive and extract files + extract = set() + chmod = set() + items = [] + full_path = {} + if meta_path: + from datetime import datetime + + meta = { + "bindir": str(bindir), + "owner": owner, + "repo": repo, + "release_id": release["tag_name"], + "name": asset_name, + "updated_at": asset["updated_at"], + "extracted_at": datetime.now().isoformat(), + } + if subset: + meta["subset"] = sorted(subset) + + if str(download_pth).endswith(".tar.gz"): + zip_path = Path(str(download_pth).replace(".tar.gz", ".zip")) + if not quiet: + print( + f"\nA *.tar.gz file ('{download_pth}') has been downloaded " + + f" and will be converted to a zip file ('{zip_path}')." + ) + + if zip_path.exists(): + zip_path.unlink() + + zipf = zipfile.ZipFile( + file=zip_path, mode="a", compression=zipfile.ZIP_DEFLATED + ) + with tarfile.open(name=download_pth, mode="r|gz") as tarf: + for m in tarf: + f = tarf.extractfile(m) + if f is not None: + fl = f.read() + fn = m.name + zipf.writestr(fn, fl) + zipf.close() + download_pth = zip_path + + with zipfile.ZipFile(download_pth, "r") as zipf: + # First gather files within internal directories named "bin" + for pth in zipf.namelist(): + p = Path(pth) + if p.parent.name == "bin": + full_path[p.name] = pth + files = set(full_path.keys()) + + if not files: + # there was no internal "bin", so assume all files to be extracted + files = set(zipf.namelist()) + + code = False + if "code.json" in files and repo == "pestpp": + code_bytes = zipf.read("code.json") + code = json.loads(code_bytes.decode()) + if meta_path: + import hashlib + + code_md5 = hashlib.md5(code_bytes).hexdigest() + meta["code_json_md5"] = code_md5 + + if "code.json" in files: + # don't extract this file + files.remove("code.json") + + if subset: + nosub = False + subset_keys = files + if code: + subset_keys |= set(code.keys()) + not_found = subset.difference(subset_keys) + if not_found: + raise ValueError( + f"subset item{'s' if len(not_found) != 1 else ''} " + f"not found: {', '.join(sorted(not_found))}\n" + f"available items are:\n{columns_str(sorted(subset_keys))}" + ) + else: + nosub = True + subset = set() + + if code: + + def add_item(key, fname, do_chmod): + if fname in files: + extract.add(fname) + items.append(f"{fname} ({code[key]['version']})") + if do_chmod: + chmod.add(fname) + else: + print(f"file {fname} does not exist") + return + + for key in sorted(code): + if code[key].get("shared_object"): + fname = f"{key}{lib_suffix}" + if nosub or ( + subset and (key in subset or fname in subset) + ): + add_item(key, fname, do_chmod=False) + else: + fname = f"{key}{exe_suffix}" + if nosub or ( + subset and (key in subset or fname in subset) + ): + add_item(key, fname, do_chmod=True) + # check if double version exists + fname = f"{key}dbl{exe_suffix}" + if ( + code[key].get("double_switch", True) + and fname in files + and ( + nosub + or (subset and (key in subset or fname in subset)) + ) + ): + add_item(key, fname, do_chmod=True) + + else: + # releases without code.json + for fname in sorted(files): + if nosub or (subset and fname in subset): + if full_path: + extract.add(full_path[fname]) + else: + extract.add(fname) + items.append(fname) + if not fname.endswith(lib_suffix): + chmod.add(fname) + if not quiet: + print( + f"\nExtracting {len(extract)} " + f"file{'s' if len(extract) != 1 else ''} to '{bindir}'" + ) + + zipf.extractall(bindir, members=extract) + + # If this is a TemporaryDirectory, then delete the directory and files + del tmpdir + + if full_path: + # move files that used a full path to bindir + rmdirs = set() + for fpath in extract: + fpath = Path(fpath) + bindir_path = bindir / fpath + bindir_path.replace(bindir / fpath.name) + rmdirs.add(fpath.parent) + # clean up directories, starting with the longest + for rmdir in reversed(sorted(rmdirs)): + bindir_path = bindir / rmdir + bindir_path.rmdir() + for subdir in rmdir.parents: + bindir_path = bindir / subdir + if bindir_path == bindir: + break + shutil.rmtree(str(bindir_path)) + + if ostag in ["linux", "mac"]: + # similar to "chmod +x fname" for each executable + for fname in chmod: + pth = bindir / fname + pth.chmod(pth.stat().st_mode | 0o111) + + # Show listing + if not quiet: + if any(items): + print(columns_str(items)) + + if not subset: + if full_path: + extract = {Path(fpth).name for fpth in extract} + unexpected = extract.difference(files) + if unexpected: + print(f"unexpected remaining {len(unexpected)} files:") + print(columns_str(sorted(unexpected))) + + # Save metadata, only for pyemu + if meta_path: + if "pytest" in str(bindir) or "pytest" in sys.modules: + # Don't write metadata if this is part of pytest + print("skipping writing pyemu metadata for pytest") + return + meta_list.append(meta) + if not pyemu_appdata_path.exists(): + pyemu_appdata_path.mkdir(parents=True, exist_ok=True) + try: + meta_path.write_text(json.dumps(meta_list, indent=4) + "\n") + except OSError as err: + print(f"cannot write pyemu metadata file: '{meta_path}': {err!r}") + if not quiet: + if meta_path_exists: + print(f"\nUpdated pyemu metadata file: '{meta_path}'") + else: + print(f"\nWrote new pyemu metadata file: '{meta_path}'") + + +def cli_main(): + """Command-line interface.""" + import argparse + + # Show meaningful examples at bottom of help + prog = Path(sys.argv[0]).stem + if sys.platform.startswith("win"): + drv = Path("c:/") + else: + drv = Path("/") + example_bindir = drv / "path" / "to" / "bin" + examples = f"""\ +Examples: + + Install executables into an existing '{example_bindir}' directory: + $ {prog} {example_bindir} + + Install a development snapshot of PEST ++ by choosing a repo: + $ {prog} --repo pestpp {example_bindir} + """ + if within_pyemu: + examples += f"""\ + + PpyEMU users can install executables using a special option: + $ {prog} :pyemu + """ + + parser = argparse.ArgumentParser( + description=__doc__.split("\n")[0], + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=examples, + ) + + bindir_help = ( + "Directory to extract executables. Use ':' to interactively select an " + "option of paths. Other auto-select options are only available if the " + "current user can write files. " + ) + if within_pyemu: + bindir_help += ( + "Option ':prev' is the previously used 'bindir' path selection. " + "Option ':pyemu' will create and install programs for pyEMY. " + ) + if sys.platform.startswith("win"): + bindir_help += ( + "Option ':python' is Python's Scripts directory. " + "Option ':windowsapps' is " + "'%%LOCALAPPDATA%%\\Microsoft\\WindowsApps'." + ) + else: + bindir_help += ( + "Option ':python' is Python's bin directory. " + "Option ':home' is '$HOME/.local/bin'. " + "Option ':system' is '/usr/local/bin'." + ) + parser.add_argument("bindir", help=bindir_help) + parser.add_argument( + "--owner", + type=str, + default=default_owner, + help=f"GitHub repository owner; default is '{default_owner}'.", + ) + parser.add_argument( + "--repo", + choices=available_repos, + default=default_repo, + help=f"Name of GitHub repository; default is '{default_repo}'.", + ) + parser.add_argument( + "--release-id", + default="latest", + help="GitHub release ID; default is 'latest'.", + ) + parser.add_argument( + "--ostag", + choices=available_ostags, + help="Operating system tag; default is to automatically choose.", + ) + parser.add_argument( + "--subset", + help="Subset of executables to extract, specified as a " + "comma-separated string, e.g. 'pestpp-glm,pestpp-ies'.", + ) + parser.add_argument( + "--downloads-dir", + help="Manually specify directory to download archives.", + ) + parser.add_argument( + "--force", + action="store_true", + help="Force re-download archive. Default behavior will use archive if " + "previously downloaded in downloads-dir.", + ) + parser.add_argument( + "--quiet", action="store_true", help="Show fewer messages." + ) + args = vars(parser.parse_args()) + try: + run_main(**args, _is_cli=True) + except (EOFError, KeyboardInterrupt): + sys.exit(f" cancelling '{sys.argv[0]}'") + + +if __name__ == "__main__": + """Run command-line interface, if run as a script.""" + cli_main() diff --git a/pyproject.toml b/pyproject.toml index c08d5f58..4cea4f50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,14 +9,14 @@ build-backend = "setuptools.build_meta" name = "pyemu" dynamic = ["version"] authors = [ - {name = "Jeremy White", email = "jtwhite1000@gmail.com"}, - {name = "Mike Fienen", email = "mnfienen@usgs.gov"}, - {name = "Brioch Hemmings", email = "b.hemmings@gns.cri.nz"}, + { name = "Jeremy White", email = "jtwhite1000@gmail.com" }, + { name = "Mike Fienen", email = "mnfienen@usgs.gov" }, + { name = "Brioch Hemmings", email = "b.hemmings@gns.cri.nz" }, ] description = "pyEMU is a set of python modules for interfacing with PEST and PEST++" readme = "README.md" keywords = ["pest", "pestpp"] -license = {text = "BSD 3-Clause"} +license = { text = "BSD 3-Clause" } classifiers = [ "Intended Audience :: Science/Research", "License :: OSI Approved :: BSD License", @@ -35,22 +35,27 @@ optional = [ "matplotlib", "pyshp", "scipy", - "shapely" + "shapely", ] test = [ "coveralls", "pytest", "pytest-cov", "pytest-xdist", - "nbmake" + "flaky", + "nbmake", + "modflow-devtools", ] docs = [ "pyemu[optional]", - "sphinx <7.2", # sphinx 7.2 challenges with iterating over CSS + "sphinx <7.2", # sphinx 7.2 challenges with iterating over CSS "sphinx-autoapi", "sphinx-rtd-theme >=1.3.0rc1", ] +[project.scripts] +get-pestpp = "pyemu.utils.get_pestpp:cli_main" + [project.urls] documentation = "https://pyemu.readthedocs.io/" repository = "https://github.com/pypest/pyemu"