Merge pull request #457 from jdhughes-usgs/get-pestpp-cli

Thanks @jdhughes-usgs !

README.md

@@ -80,12 +80,22 @@ Related Links
 How to get started with pyEMU
 =============================
 
-pyEMU is available through pyPI:
+pyEMU is available through pyPI and conda. To install pyEMU type:
 
-`>>>pip install pyemu`
+    >>>conda install -c conda-forge pyemu
+
+or
+
+    >>>pip install pyemu
 
 pyEMU needs `numpy` and `pandas`.  For plotting, `matplotloib`, `pyshp`, and `flopy` to take advantage of the auto interface construction
 
+After pyEMU is installed, the PEST++ software suite can be installed for your operating system  using the command:
+
+    get-pestpp :pyemu
+
+See [documentation](get_pestpp.md) for more information.
+
 Found a bug? Got a smart idea? Contributions welcome.
 ====================================================
 Feel free to raise and issue or submit a pull request.

autotest/conftest.py

@@ -1,6 +1,9 @@
+from pathlib import Path
 import pytest
 from pst_from_tests import freybergmf6_2_pstfrom
 
+pytest_plugins = ["modflow_devtools.fixtures"]
+
 
 collect_ignore = [
     # "utils_tests.py",
@@ -16,5 +19,3 @@
     # "mat_tests.py",
     # "da_tests.py"
 ]
-
-

autotest/get_pestpp_tests.py

@@ -0,0 +1,276 @@
+"""Test get-pestpp utility."""
+import os
+import platform
+import sys
+from os.path import expandvars
+from pathlib import Path
+from platform import system
+from urllib.error import HTTPError
+
+import pytest
+from flaky import flaky
+from modflow_devtools.markers import requires_github
+from modflow_devtools.misc import run_py_script
+from pyemu.utils import get_pestpp
+from pyemu.utils.get_pestpp import get_release, get_releases, select_bindir
+
+rate_limit_msg = "rate limit exceeded"
+get_pestpp_script = (
+    Path(__file__).parent.parent / "pyemu" / "utils" / "get_pestpp.py"
+)
+bindir_options = {
+    "pyemu": Path(expandvars(r"%LOCALAPPDATA%\pyemu")) / "bin"
+    if system() == "Windows"
+    else Path.home() / ".local" / "share" / "pyemu" / "bin",
+    "python": Path(sys.prefix)
+    / ("Scripts" if system() == "Windows" else "bin"),
+    "home": Path.home() / ".local" / "bin",
+}
+owner_options = [
+    "usgs",
+]
+repo_options = {
+    "pestpp": [
+        "pestpp-da",
+        "pestpp-glm",
+        "pestpp-ies",
+        "pestpp-mou",
+        "pestpp-opt",
+        "pestpp-sen",
+        "pestpp-sqp",
+        "pestpp-swp",
+    ],
+}
+
+if system() == "Windows":
+    bindir_options["windowsapps"] = Path(
+        expandvars(r"%LOCALAPPDATA%\Microsoft\WindowsApps")
+    )
+else:
+    bindir_options["system"] = Path("/usr") / "local" / "bin"
+
+
+@pytest.fixture
+def downloads_dir(tmp_path_factory):
+    downloads_dir = tmp_path_factory.mktemp("Downloads")
+    return downloads_dir
+
+
+@pytest.fixture(autouse=True)
+def create_home_local_bin():
+    # make sure $HOME/.local/bin exists for :home option
+    home_local = Path.home() / ".local" / "bin"
+    home_local.mkdir(parents=True, exist_ok=True)
+
+
+def run_get_pestpp_script(*args):
+    return run_py_script(get_pestpp_script, *args, verbose=True)
+
+
+def append_ext(path: str):
+    if system() == "Windows":
+        return f"{path}{'.exe'}"
+    elif system() == "Darwin":
+        return f"{path}{''}"
+    elif system() == "Linux":
+        return f"{path}{''}"
+
+
+@pytest.mark.parametrize("per_page", [-1, 0, 101, 1000])
+def test_get_releases_bad_page_size(per_page):
+    with pytest.raises(ValueError):
+        get_releases(repo="pestpp", per_page=per_page)
+
+
+@flaky
+@requires_github
+@pytest.mark.parametrize("repo", repo_options.keys())
+def test_get_releases(repo):
+    releases = get_releases(repo=repo)
+    assert "latest" in releases
+
+
+@flaky
+@requires_github
+@pytest.mark.parametrize("repo", repo_options.keys())
+def test_get_release(repo):
+    tag = "latest"
+    release = get_release(repo=repo, tag=tag)
+    assets = release["assets"]
+    release_tag_name = release["tag_name"]
+
+    expected_assets = [
+        f"pestpp-{release_tag_name}-linux.tar.gz",
+        f"pestpp-{release_tag_name}-imac.tar.gz",
+        f"pestpp-{release_tag_name}-iwin.zip",
+    ]
+    expected_ostags = [a.replace(".zip", "") for a in expected_assets]
+    expected_ostags = [a.replace("tar.gz", "") for a in expected_assets]
+    actual_assets = [asset["name"] for asset in assets]
+
+    if repo == "pestpp":
+        # can remove if modflow6 releases follow asset name conventions followed in executables and nightly build repos
+        assert {a.rpartition("_")[2] for a in actual_assets} >= {
+            a for a in expected_assets if not a.startswith("win")
+        }
+    else:
+        for ostag in expected_ostags:
+            assert any(
+                ostag in a for a in actual_assets
+            ), f"dist not found for {ostag}"
+
+
+@pytest.mark.parametrize("bindir", bindir_options.keys())
+def test_select_bindir(bindir, function_tmpdir):
+    expected_path = bindir_options[bindir]
+    if not os.access(expected_path, os.W_OK):
+        pytest.skip(f"{expected_path} is not writable")
+    selected = select_bindir(f":{bindir}")
+
+    if system() != "Darwin":
+        assert selected == expected_path
+    else:
+        # for some reason sys.prefix can return different python
+        # installs when invoked here and get_modflow.py on macOS
+        #   https://github.com/modflowpy/flopy/actions/runs/3331965840/jobs/5512345032#step:8:1835
+        #
+        # work around by just comparing the end of the bin path
+        # should be .../Python.framework/Versions/<version>/bin
+        assert selected.parts[-4:] == expected_path.parts[-4:]
+
+
+def test_script_help():
+    assert get_pestpp_script.exists()
+    stdout, stderr, returncode = run_get_pestpp_script("-h")
+    assert "usage" in stdout
+    assert len(stderr) == 0
+    assert returncode == 0
+
+
+@flaky
+@requires_github
+def test_script_invalid_options(function_tmpdir, downloads_dir):
+    # try with bindir that doesn't exist
+    bindir = function_tmpdir / "bin1"
+    assert not bindir.exists()
+    stdout, stderr, returncode = run_get_pestpp_script(bindir)
+    if rate_limit_msg in stderr:
+        pytest.skip(f"GitHub {rate_limit_msg}")
+    assert "does not exist" in stderr
+    assert returncode == 1
+
+    # attempt to fetch a non-existing release-id
+    bindir.mkdir()
+    assert bindir.exists()
+    stdout, stderr, returncode = run_get_pestpp_script(
+        bindir, "--release-id", "1.9", "--downloads-dir", downloads_dir
+    )
+    if rate_limit_msg in stderr:
+        pytest.skip(f"GitHub {rate_limit_msg}")
+    assert "Release 1.9 not found" in stderr
+    assert returncode == 1
+
+    # try to select an invalid --subset
+    bindir = function_tmpdir / "bin2"
+    bindir.mkdir()
+    stdout, stderr, returncode = run_get_pestpp_script(
+        bindir, "--subset", "pestpp-opt,mpx", "--downloads-dir", downloads_dir
+    )
+    if rate_limit_msg in stderr:
+        pytest.skip(f"GitHub {rate_limit_msg}")
+    assert "subset item not found: mpx" in stderr
+    assert returncode == 1
+
+
+@flaky
+@requires_github
+def test_script_valid_options(function_tmpdir, downloads_dir):
+    # fetch latest
+    bindir = function_tmpdir / "bin1"
+    bindir.mkdir()
+    stdout, stderr, returncode = run_get_pestpp_script(
+        bindir, "--downloads-dir", downloads_dir
+    )
+    if rate_limit_msg in stderr:
+        pytest.skip(f"GitHub {rate_limit_msg}")
+    assert len(stderr) == returncode == 0
+    files = [item.name for item in bindir.iterdir() if item.is_file()]
+    assert len(files) == 8
+
+    # valid subset
+    bindir = function_tmpdir / "bin2"
+    bindir.mkdir()
+    stdout, stderr, returncode = run_get_pestpp_script(
+        bindir,
+        "--subset",
+        "pestpp-da,pestpp-swp,pestpp-ies",
+        "--downloads-dir",
+        downloads_dir,
+    )
+    if rate_limit_msg in stderr:
+        pytest.skip(f"GitHub {rate_limit_msg}")
+    assert len(stderr) == returncode == 0
+    files = [item.stem for item in bindir.iterdir() if item.is_file()]
+    assert sorted(files) == ["pestpp-da", "pestpp-ies", "pestpp-swp"]
+
+    # similar as before, but also specify a ostag
+    bindir = function_tmpdir / "bin3"
+    bindir.mkdir()
+    stdout, stderr, returncode = run_get_pestpp_script(
+        bindir,
+        "--subset",
+        "pestpp-ies",
+        "--release-id",
+        "5.2.6",
+        "--ostag",
+        "win",
+        "--downloads-dir",
+        downloads_dir,
+    )
+    if rate_limit_msg in stderr:
+        pytest.skip(f"GitHub {rate_limit_msg}")
+    assert len(stderr) == returncode == 0
+    files = [item.name for item in bindir.iterdir() if item.is_file()]
+    assert sorted(files) == ["pestpp-ies.exe"]
+
+
+@flaky
+@requires_github
+@pytest.mark.parametrize("owner", owner_options)
+@pytest.mark.parametrize("repo", repo_options.keys())
+def test_script(function_tmpdir, owner, repo, downloads_dir):
+    bindir = str(function_tmpdir)
+    stdout, stderr, returncode = run_get_pestpp_script(
+        bindir,
+        "--owner",
+        owner,
+        "--repo",
+        repo,
+        "--downloads-dir",
+        downloads_dir,
+    )
+    if rate_limit_msg in stderr:
+        pytest.skip(f"GitHub {rate_limit_msg}")
+
+    paths = list(function_tmpdir.glob("*"))
+    names = [p.name for p in paths]
+    expected_names = [append_ext(p) for p in repo_options[repo]]
+    assert set(names) >= set(expected_names)
+
+
+@flaky
+@requires_github
+@pytest.mark.parametrize("owner", owner_options)
+@pytest.mark.parametrize("repo", repo_options.keys())
+def test_python_api(function_tmpdir, owner, repo, downloads_dir):
+    bindir = str(function_tmpdir)
+    try:
+        get_pestpp(bindir, owner=owner, repo=repo, downloads_dir=downloads_dir)
+    except HTTPError as err:
+        if err.code == 403:
+            pytest.skip(f"GitHub {rate_limit_msg}")
+
+    paths = list(function_tmpdir.glob("*"))
+    names = [p.name for p in paths]
+    expected_names = [append_ext(p) for p in repo_options[repo]]
+    assert set(names) >= set(expected_names)

etc/environment.yml

@@ -5,16 +5,18 @@ dependencies:
   # required
   - python>=3.8
   - numpy>=1.15.0
-  - pandas
+  - pandas<2.1.0
   # optional
   - matplotlib>=1.4.0
   - pyshp
   - jinja2
   # tests
   - coveralls
+  - flaky
   - pytest
   - pytest-cov
   - pytest-xdist
   - nbmake
   - shapely
   - pyproj
+  - modflow-devtools