-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' of https://github.com/OCHA-DAP/hdx-cli-toolkit
- Loading branch information
Showing
11 changed files
with
2,269 additions
and
1,563 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,18 +1,19 @@ | ||
| { | ||
| "pylint.args": ["--rcfile=config/.pylintrc"], | ||
| "flake8.args": [ | ||
| "--config=config/.flake8" | ||
| ], | ||
| "editor.defaultFoldingRangeProvider": "ms-python.black-formatter", | ||
| "editor.defaultFormatter": "ms-python.black-formatter", | ||
| "editor.formatOnSave": true, | ||
| "prettier.enable": false, | ||
| "python.testing.pytestArgs": [ | ||
| "--cov=hdx_cli_toolkit", | ||
| "--cov-config=config/.coveragerc", | ||
| "--cov-report=xml:cov.xml", | ||
| "tests" | ||
| ], | ||
| "python.testing.unittestEnabled": false, | ||
| "python.testing.pytestEnabled": true | ||
| } | ||
| { | ||
| "pylint.args": ["--rcfile=config/.pylintrc"], | ||
| "flake8.args": [ | ||
| "--config=config/.flake8" | ||
| ], | ||
| "editor.defaultFoldingRangeProvider": "ms-python.black-formatter", | ||
| "editor.defaultFormatter": "ms-python.black-formatter", | ||
| "editor.formatOnSave": true, | ||
| "prettier.enable": false, | ||
| "python.testing.pytestArgs": [ | ||
| "--cov=hdx_cli_toolkit", | ||
| "--cov-config=config/.coveragerc", | ||
| "--cov-report=xml:cov.xml", | ||
| "tests" | ||
| ], | ||
| "python.testing.unittestEnabled": false, | ||
| "python.testing.pytestEnabled": true, | ||
| "python.analysis.typeCheckingMode": "basic", | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,115 +1,115 @@ | ||
| # HDX CLI Toolkit | ||
|
|
||
| ## Overview | ||
|
|
||
| This toolkit provides a commandline interface to the [Humanitarian Data Exchange](https://data.humdata.org/) (HDX) to allow for bulk modification operations and other administrative activities such as getting `id` values for users and organization. It is useful for those managing HDX and developers building data pipelines for HDX. The currently supported commands are as follows: | ||
|
|
||
| ``` | ||
| configuration Print configuration information to terminal | ||
| download Download dataset resources from HDX | ||
| get_organization_metadata Get an organization id and other metadata | ||
| get_user_metadata Get user id and other metadata | ||
| list List datasets in HDX | ||
| print Print datasets in HDX to the terminal | ||
| quickcharts Upload QuickChart JSON description to HDX | ||
| showcase Upload showcase to HDX | ||
| update Update datasets in HDX | ||
| update_resource Update a resource in HDX | ||
| ``` | ||
|
|
||
| It is a thin wrapper to the [hdx-python-api](https://github.com/OCHA-DAP/hdx-python-api) library written by Mike Rans. | ||
|
|
||
| The library requires some configuration, described below, to authenticate to the HDX instance. | ||
|
|
||
| ## Installation | ||
| `hdx-cli-toolkit` is a Python application published to the PyPI package repository, therefore it can be installed easily with: | ||
|
|
||
| ```pip install hdx_cli_toolkit``` | ||
|
|
||
| Users may prefer to make a global, isolated installation using [pipx](https://pypi.org/project/pipx/) which will make the `hdx-toolkit` commands available across their projects: | ||
|
|
||
| ```pipx install hdx_cli_toolkit``` | ||
|
|
||
| `hdx-cli-toolkit` uses the `hdx-python-api` library, this requires the following to be added to a file called `.hdx_configuration.yaml` in the user's home directory. | ||
|
|
||
| ``` | ||
| hdx_key_stage: "[hdx_key from the staging HDX site]" | ||
| hdx_key: "[hdx_key from the prod HDX site]" | ||
| ``` | ||
|
|
||
| A user agent (`hdx_cli_toolkit_*`) is specified in the `~/.useragents.yaml` file with the * replaced with the users initials. | ||
| ``` | ||
| hdx-cli-toolkit: | ||
| preprefix: [YOUR_ORGANIZATION] | ||
| user_agent: hdx_cli_toolkit_ih | ||
| ``` | ||
|
|
||
| ## Usage | ||
|
|
||
| The `hdx-toolkit` is built using the Python `click` library. Details of the currently implemented commands can be revealed by running `hdx-toolkit --help`: | ||
|
|
||
| ``` | ||
| $ hdx-toolkit --help | ||
| Usage: hdx-toolkit [OPTIONS] COMMAND [ARGS]... | ||
| Tools for Commandline interactions with HDX | ||
| Options: | ||
| --help Show this message and exit. | ||
| Commands: | ||
| configuration Print configuration information to terminal | ||
| download Download dataset resources from HDX | ||
| get_organization_metadata Get an organization id and other metadata | ||
| get_user_metadata Get user id and other metadata | ||
| list List datasets in HDX | ||
| print Print datasets in HDX to the terminal | ||
| quickcharts Upload QuickChart JSON description to HDX | ||
| remove_extras_key Remove extras key from a dataset | ||
| showcase Upload showcase to HDX | ||
| update Update datasets in HDX | ||
| update_resource Update a resource in HDX | ||
| ``` | ||
|
|
||
| And details of the arguments for a command can be found using: | ||
|
|
||
| ```shell | ||
| hdx-toolkit [COMMAND] --help | ||
| ``` | ||
|
|
||
| A detailed walk through of commands can be found in the [DEMO.md](DEMO.md) file | ||
|
|
||
| ## Contributions | ||
|
|
||
| For developers the code should be cloned installed from the [GitHub repo](https://github.com/OCHA-DAP/hdx-cli-toolkit), and a virtual enviroment created: | ||
|
|
||
| ```shell | ||
| python -m venv venv | ||
| source venv/Scripts/activate | ||
| ``` | ||
|
|
||
| And then an editable installation created: | ||
|
|
||
| ```shell | ||
| pip install -e . | ||
| ``` | ||
|
|
||
| The library is then configured, as described above. | ||
|
|
||
| This project uses a GitHub Action to run tests and linting. It requires the following environment variables/secrets to be set in the `test` environment: | ||
|
|
||
| ``` | ||
| HDX_KEY - secret. Value: fake secret | ||
| HDX_KEY_STAGE - secret. Value: a live API key for the stage server | ||
| HDX_SITE - environment variable. Value: stage | ||
| USER_AGENT - environment variable. Value: hdx_cli_toolkit_gha | ||
| PREPREFIX - - environment variable. Value: [YOUR_organization] | ||
| ``` | ||
|
|
||
| Most tests use mocking in place of HDX, although the `test_integration.py` suite runs against the `stage` server. | ||
|
|
||
| New features should be developed against a GitHub issue on a separate branch with a name starting `GH[issue number]_`. `PULL_REQUEST_TEMPLATE.md` should be used in preparing pull requests. Versioning is updated manually in `pyproject.toml` and is described in the template, in brief it is CalVer `YYYY.MM.Micro`. | ||
|
|
||
| ## Publication | ||
|
|
||
| Publication to PyPI is done automatically when a release is created. | ||
| # HDX CLI Toolkit | ||
|
|
||
| ## Overview | ||
|
|
||
| This toolkit provides a commandline interface to the [Humanitarian Data Exchange](https://data.humdata.org/) (HDX) to allow for bulk modification operations and other administrative activities such as getting `id` values for users and organization. It is useful for those managing HDX and developers building data pipelines for HDX. The currently supported commands are as follows: | ||
|
|
||
| ``` | ||
| configuration Print configuration information to terminal | ||
| download Download dataset resources from HDX | ||
| get_organization_metadata Get an organization id and other metadata | ||
| get_user_metadata Get user id and other metadata | ||
| list List datasets in HDX | ||
| print Print datasets in HDX to the terminal | ||
| quickcharts Upload QuickChart JSON description to HDX | ||
| showcase Upload showcase to HDX | ||
| update Update datasets in HDX | ||
| update_resource Update a resource in HDX | ||
| ``` | ||
|
|
||
| It is a thin wrapper to the [hdx-python-api](https://github.com/OCHA-DAP/hdx-python-api) library written by Mike Rans. | ||
|
|
||
| The library requires some configuration, described below, to authenticate to the HDX instance. | ||
|
|
||
| ## Installation | ||
| `hdx-cli-toolkit` is a Python application published to the PyPI package repository, therefore it can be installed easily with: | ||
|
|
||
| ```pip install hdx_cli_toolkit``` | ||
|
|
||
| Users may prefer to make a global, isolated installation using [pipx](https://pypi.org/project/pipx/) which will make the `hdx-toolkit` commands available across their projects: | ||
|
|
||
| ```pipx install hdx_cli_toolkit``` | ||
|
|
||
| `hdx-cli-toolkit` uses the `hdx-python-api` library, this requires the following to be added to a file called `.hdx_configuration.yaml` in the user's home directory. | ||
|
|
||
| ``` | ||
| hdx_key_stage: "[hdx_key from the staging HDX site]" | ||
| hdx_key: "[hdx_key from the prod HDX site]" | ||
| ``` | ||
|
|
||
| A user agent (`hdx_cli_toolkit_*`) is specified in the `~/.useragents.yaml` file with the * replaced with the users initials. | ||
| ``` | ||
| hdx-cli-toolkit: | ||
| preprefix: [YOUR_ORGANIZATION] | ||
| user_agent: hdx_cli_toolkit_ih | ||
| ``` | ||
|
|
||
| ## Usage | ||
|
|
||
| The `hdx-toolkit` is built using the Python `click` library. Details of the currently implemented commands can be revealed by running `hdx-toolkit --help`: | ||
|
|
||
| ``` | ||
| $ hdx-toolkit --help | ||
| Usage: hdx-toolkit [OPTIONS] COMMAND [ARGS]... | ||
| Tools for Commandline interactions with HDX | ||
| Options: | ||
| --help Show this message and exit. | ||
| Commands: | ||
| configuration Print configuration information to terminal | ||
| download Download dataset resources from HDX | ||
| get_organization_metadata Get an organization id and other metadata | ||
| get_user_metadata Get user id and other metadata | ||
| list List datasets in HDX | ||
| print Print datasets in HDX to the terminal | ||
| quickcharts Upload QuickChart JSON description to HDX | ||
| remove_extras_key Remove extras key from a dataset | ||
| showcase Upload showcase to HDX | ||
| update Update datasets in HDX | ||
| update_resource Update a resource in HDX | ||
| ``` | ||
|
|
||
| And details of the arguments for a command can be found using: | ||
|
|
||
| ```shell | ||
| hdx-toolkit [COMMAND] --help | ||
| ``` | ||
|
|
||
| A detailed walk through of commands can be found in the [DEMO.md](DEMO.md) file | ||
|
|
||
| ## Contributions | ||
|
|
||
| For developers the code should be cloned installed from the [GitHub repo](https://github.com/OCHA-DAP/hdx-cli-toolkit), and a virtual enviroment created: | ||
|
|
||
| ```shell | ||
| python -m venv venv | ||
| source venv/Scripts/activate | ||
| ``` | ||
|
|
||
| And then an editable installation created: | ||
|
|
||
| ```shell | ||
| pip install -e . | ||
| ``` | ||
|
|
||
| The library is then configured, as described above. | ||
|
|
||
| This project uses a GitHub Action to run tests and linting. It requires the following environment variables/secrets to be set in the `test` environment: | ||
|
|
||
| ``` | ||
| HDX_KEY - secret. Value: fake secret | ||
| HDX_KEY_STAGE - secret. Value: a live API key for the stage server | ||
| HDX_SITE - environment variable. Value: stage | ||
| USER_AGENT - environment variable. Value: hdx_cli_toolkit_gha | ||
| PREPREFIX - - environment variable. Value: [YOUR_organization] | ||
| ``` | ||
|
|
||
| Most tests use mocking in place of HDX, although the `test_integration.py` suite runs against the `stage` server. | ||
|
|
||
| New features should be developed against a GitHub issue on a separate branch with a name starting `GH[issue number]_`. `PULL_REQUEST_TEMPLATE.md` should be used in preparing pull requests. Versioning is updated manually in `pyproject.toml` and is described in the template, in brief it is CalVer `YYYY.MM.Micro`. | ||
|
|
||
| ## Publication | ||
|
|
||
| Publication to PyPI is done automatically when a release is created. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,47 +1,47 @@ | ||
| [project] | ||
| name = "hdx_cli_toolkit" | ||
| version = "2024.6.2" | ||
| description = "HDX CLI tool kit for commandline interaction with HDX" | ||
| readme = {file = "README.md", content-type = "text/markdown"} | ||
| license = {file = "LICENSE"} | ||
| requires-python = ">=3.11" | ||
| authors = [ | ||
| {email = "[email protected]"}, | ||
| {name = "Ian Hopkinson"} | ||
| ] | ||
|
|
||
| dependencies = [ | ||
| "hdx-python-api>=3.7.0", | ||
| "hdx-python-country", | ||
| "quantulum3[classifier]", # This stops the UserWarning but has a number of large dependencies | ||
| "click", | ||
| "hatch", | ||
| "pytest", | ||
| "pytest-cov", | ||
| "black==23.10.0", | ||
| "flake8", | ||
| "pylint" | ||
| ] | ||
|
|
||
| [project.scripts] | ||
| hdx-toolkit = "hdx_cli_toolkit.cli:hdx_toolkit" | ||
|
|
||
| [project.urls] | ||
| repository = "https://github.com/OCHA-DAP/hdx-cli-toolkit" | ||
|
|
||
| [build-system] | ||
| requires = ["setuptools >= 61.0.0"] | ||
| build-backend = "setuptools.build_meta" | ||
|
|
||
| [tool.setuptools.packages.find] | ||
| where = ["src"] | ||
|
|
||
| [tool.black] | ||
| line-length = 100 | ||
| target-version = ['py311'] | ||
| include = '\.pyi?$' | ||
| extend-exclude = ''' | ||
| # A regex preceded with ^/ will apply only to files and directories | ||
| # in the root of the project. | ||
| ^/foo.py # exclude a file named foo.py in the root of the project (in addition to the defaults) | ||
| [project] | ||
| name = "hdx_cli_toolkit" | ||
| version = "2024.7.1" | ||
| description = "HDX CLI tool kit for commandline interaction with HDX" | ||
| readme = {file = "README.md", content-type = "text/markdown"} | ||
| license = {file = "LICENSE"} | ||
| requires-python = ">=3.11" | ||
| authors = [ | ||
| {email = "[email protected]"}, | ||
| {name = "Ian Hopkinson"} | ||
| ] | ||
|
|
||
| dependencies = [ | ||
| "hdx-python-api>=3.7.0", | ||
| "hdx-python-country", | ||
| "quantulum3[classifier]", # This stops the UserWarning but has a number of large dependencies | ||
| "click", | ||
| "hatch", | ||
| "pytest", | ||
| "pytest-cov", | ||
| "black==23.10.0", | ||
| "flake8", | ||
| "pylint" | ||
| ] | ||
|
|
||
| [project.scripts] | ||
| hdx-toolkit = "hdx_cli_toolkit.cli:hdx_toolkit" | ||
|
|
||
| [project.urls] | ||
| repository = "https://github.com/OCHA-DAP/hdx-cli-toolkit" | ||
|
|
||
| [build-system] | ||
| requires = ["setuptools >= 61.0.0"] | ||
| build-backend = "setuptools.build_meta" | ||
|
|
||
| [tool.setuptools.packages.find] | ||
| where = ["src"] | ||
|
|
||
| [tool.black] | ||
| line-length = 100 | ||
| target-version = ['py311'] | ||
| include = '\.pyi?$' | ||
| extend-exclude = ''' | ||
| # A regex preceded with ^/ will apply only to files and directories | ||
| # in the root of the project. | ||
| ^/foo.py # exclude a file named foo.py in the root of the project (in addition to the defaults) | ||
| ''' |
Oops, something went wrong.